From 5af0b12061ce9eea660fdf4438489b07bcd80263 Mon Sep 17 00:00:00 2001 From: "Sergey.Konovalov" Date: Wed, 9 Sep 2015 08:06:35 +0000 Subject: [PATCH] add Metrics statistic git-svn-id: svn://192.168.3.15/activex/AVS/Sources/TeamlabOffice/trunk/nodeJSProjects@64575 954022d7-b5bf-4e40-9824-e11837661b57 --- CoAuthoring/sources/DocsCoServer.js | 9 ++ CoAuthoring/sources/canvasservice.js | 2 +- CoAuthoring/sources/converterservice.js | 10 +++ CoAuthoring/sources/fontservice.js | 9 ++ Common/config/default.json | 6 ++ Common/package.json | 3 +- Common/sources/statsdclient.js | 16 ++++ FileConverter/sources/converter.js | 30 +++++++ Metrics/config/config.js | 106 ++++++++++++++++++++++++ Metrics/package.json | 9 ++ 10 files changed, 198 insertions(+), 2 deletions(-) create mode 100644 Common/sources/statsdclient.js create mode 100644 Metrics/config/config.js create mode 100644 Metrics/package.json diff --git a/CoAuthoring/sources/DocsCoServer.js b/CoAuthoring/sources/DocsCoServer.js index 59a5430c..97684618 100644 --- a/CoAuthoring/sources/DocsCoServer.js +++ b/CoAuthoring/sources/DocsCoServer.js @@ -54,6 +54,7 @@ var sockjs = require('sockjs'), constants = require('./../../Common/sources/constants'), utils = require('./../../Common/sources/utils'), commonDefines = require('./../../Common/sources/commondefines'), + statsDClient = require('./../../Common/sources/statsdclient'), config = require('config').get('services.CoAuthoring'), sqlBase = require('./baseConnector'), taskResult = require('./taskresult'); @@ -102,6 +103,7 @@ var connections = [], // Активные соединения redisClient, pubsub, queue; +var clientStatsD = statsDClient.getClient(); var asc_coAuthV = '3.0.8'; // Версия сервера совместного редактирования @@ -696,6 +698,10 @@ exports.install = function(server, callbackFunction) { conn.on('data', function(message) { utils.spawn(function* () { try { + var startDate = null; + if(clientStatsD) { + startDate = new Date(); + } var data = JSON.parse(message); logger.info('data.type = ' + data.type + ' id = ' + conn.docId); switch (data.type) { @@ -730,6 +736,9 @@ exports.install = function(server, callbackFunction) { canvasService.openDocument(conn, data); break; } + if(clientStatsD) { + clientStatsD.timing('data_' + data.type, new Date() - startDate); + } } catch (e) { logger.error("error receiving response: docId = %s type = %s\r\n%s", conn ? conn.docId : 'null', (data && data.type) ? data.type : 'null', e.stack); } diff --git a/CoAuthoring/sources/canvasservice.js b/CoAuthoring/sources/canvasservice.js index 5fb53d14..29d27a8f 100644 --- a/CoAuthoring/sources/canvasservice.js +++ b/CoAuthoring/sources/canvasservice.js @@ -1 +1 @@ -var sqlBase = require('./baseConnector'); var docsCoServer = require('./DocsCoServer'); var taskResult = require('./taskresult'); var logger = require('./../../Common/sources/logger'); var utils = require('./../../Common/sources/utils'); var constants = require('./../../Common/sources/constants'); var commonDefines = require('./../../Common/sources/commondefines'); var storage = require('./../../Common/sources/storage-base'); var formatChecker = require('./../../Common/sources/formatchecker'); var config_server = require('config').get('services.CoAuthoring.server'); var config_utils = require('config').get('services.CoAuthoring.utils'); var cfgAutosaveEnable = config_server.get('editor_settings_autosave_enable'); var cfgAutosaveMininterval = config_server.get('editor_settings_autosave_mininterval'); var cfgCoauthoringUrl = config_server.get('editor_settings_coauthoring_url'); var cfgSpellcheckerUrl = config_server.get('editor_settings_spellchecker_url'); var cfgAnalyticsEnable = config_server.get('editor_settings_analytics_enable'); var cfgActiveconnectionsTrackingInterval = config_server.get('license_activeconnections_tracking_interval'); var cfgReaderformats = config_server.get('editor_settings_readerformats'); var cfgEditorformats = config_server.get('editor_settings_editorformats'); var cfgViewerformats = config_server.get('editor_settings_viewerformats'); var cfgTypesUpload = config_utils.get('limits_image_types_upload'); var cfgTypesCopy = config_utils.get('limits_image_types_copy'); var cfgImageSize = config_server.get('limits_image_size'); var SAVE_TYPE_PART_START = 0; var SAVE_TYPE_PART = 1; var SAVE_TYPE_COMPLETE = 2; var SAVE_TYPE_COMPLETE_ALL = 3; function OutputDataWrap(type, data) { this['type'] = type; this['data'] = data; } OutputDataWrap.prototype = { fromObject: function(data) { this['type'] = data['type']; this['data'] = new OutputData(); this['data'].fromObject(data['data']); }, getType: function() { return this['type']; }, setType: function(data) { this['type'] = data; }, getData: function() { return this['data']; }, setData: function(data) { this['data'] = data; } }; function OutputData(type) { this['type'] = type; this['status'] = undefined; this['data'] = undefined; } OutputData.prototype = { fromObject: function(data) { this['type'] = data['type']; this['status'] = data['status']; this['data'] = data['data']; }, getType: function() { return this['type']; }, setType: function(data) { this['type'] = data; }, getStatus: function() { return this['status']; }, setStatus: function(data) { this['status'] = data; }, getData: function() { return this['data']; }, setData: function(data) { this['data'] = data; } }; function OutputSettingsData(rights, format, trackingInfo) { this['canLicense'] = true; // ToDo это должно браться из лицензии this['canEdit'] = true; this['canDownload'] = true; this['canCoAuthoring'] = true; this['canReaderMode'] = true; this['canAd'] = true; this['canBranding'] = true; this['isAutosaveEnable'] = cfgAutosaveEnable || true; this['AutosaveMinInterval'] = cfgAutosaveMininterval || 300; this['g_cAscCoAuthoringUrl'] = cfgCoauthoringUrl || ''; //так имена переменных написаны в JS this['g_cAscSpellCheckUrl'] = cfgSpellcheckerUrl || ''; //чтобы не светить лишние имена здесь используются такие же. this['isAnalyticsEnable'] = cfgAnalyticsEnable || false; this['trackingInfo'] = undefined; this['TrackingInterval'] = cfgActiveconnectionsTrackingInterval || 300; if (format) { format = format.toLowerCase(); var readerFormats = cfgReaderformats || ''; var readerFormatsArray = readerFormats.split(/[|,;]/); this['canReaderMode'] = -1 != readerFormatsArray.indexOf(format); var editorformats = cfgEditorformats || ''; var editorformatsArray = editorformats.split(/[|,;]/); this['canEdit'] = -1 != editorformatsArray.indexOf(format); var viewerformats = cfgViewerformats || ''; var viewerformatsArray = viewerformats.split(/[|,;]/); this['canDownload'] = -1 != viewerformatsArray.indexOf(format); } if (rights) { //todo } if (trackingInfo) { //todo } } function OutputSfcData() { this['key'] = undefined; this['status'] = undefined; this['url'] = undefined; this['changesurl'] = undefined; this['changeshistory'] = undefined; this['users'] = []; this['mailMerge'] = undefined; } OutputSfcData.prototype = { getKey: function() { return this['key']; }, setKey: function(data) { return this['key'] = data; }, getStatus: function() { return this['status']; }, setStatus: function(data) { return this['status'] = data; }, getUrl: function() { return this['url']; }, setUrl: function(data) { return this['url'] = data; }, getChangeUrl: function() { return this['changesurl']; }, setChangeUrl: function(data) { return this['changesurl'] = data; }, getChangeHistory: function() { return this['changeshistory']; }, setChangeHistory: function(data) { return this['changeshistory'] = data; }, getUsers: function() { return this['users']; }, setUsers: function(data) { return this['users'] = data; }, getMailMerge: function() { return this['mailMerge']; }, setMailMerge: function(data) { return this['mailMerge'] = data; } }; function OutputMailMerge(mailMergeSendData) { if (mailMergeSendData) { this['from'] = mailMergeSendData.getFrom(); this['message'] = mailMergeSendData.getMessage(); this['subject'] = mailMergeSendData.getSubject(); this['title'] = mailMergeSendData.getFileName(); var mailFormat = mailMergeSendData.getMailFormat(); switch (mailFormat) { case constants.AVS_OFFICESTUDIO_FILE_OTHER_HTMLZIP : this['type'] = 0; break; case constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_DOCX : this['type'] = 1; break; case constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF : this['type'] = 2; break; default : this['type'] = 0; break; } this['recordCount'] = mailMergeSendData.getRecordCount(); this['to'] = null; this['recordIndex'] = null; } else { this['from'] = null; this['message'] = null; this['subject'] = null; this['title'] = null; this['to'] = null; this['type'] = null; this['recordCount'] = null; this['recordIndex'] = null; } } OutputMailMerge.prototype = { getRecordIndex: function() { return this['recordIndex']; }, setRecordIndex: function(data) { return this['recordIndex'] = data; }, getTo: function() { return this['to']; }, setTo: function(data) { return this['to'] = data; } }; function getInsertStatisticString(affiliateId, filename, tag) { var dateNow = sqlBase.getDateTime(new Date()); var commandArg = [affiliateId, filename, dateNow, tag]; var commandArgEsc = commandArg.map(function(curVal) { return sqlBase.baseConnector.sqlEscape(curVal) }); return 'INSERT INTO file_statistic2 (fsc_affiliate, fsc_filename, fsc_time, fsc_tag) ' + 'VALUES (' + commandArgEsc.join(', ') + ');'; } function insertStatistic(affiliateId, filename, tag) { return new Promise(function(resolve, reject) { var sqlCommand = getInsertStatisticString(affiliateId, filename, tag); sqlBase.baseConnector.sqlQuery(sqlCommand, function(error, result) { if (error) { reject(error); } else { resolve(result); } }); }); } function* getOutputData(cmd, outputData, key, status, statusInfo, optConn, optAdditionalOutput) { switch (status) { case taskResult.FileStatus.Ok: //todo affilate var tag; var affiliateId = 'affiliateId'; switch (cmd.getEditorId()) { // case (int)LicenseInfo.EditorType.Spreadsheet: tag = 'open_sheet'; break; // case (int)LicenseInfo.EditorType.Presentation: tag = 'open_presentation'; break; default: tag = 'open_word'; break; } yield insertStatistic(affiliateId, cmd.getDocId(), tag); outputData.setStatus('ok'); if (optConn) { outputData.setData(yield storage.getSignedUrls(optConn.baseUrl, key)); } else if (optAdditionalOutput) { optAdditionalOutput.needUrlKey = key; optAdditionalOutput.needUrlMethod = 0; } break; case taskResult.FileStatus.NeedParams: outputData.setStatus('needparams'); var settingsPath = key + '/' + 'settings.json'; if (optConn) { outputData.setData(yield storage.getSignedUrl(optConn.baseUrl, settingsPath)); } else if (optAdditionalOutput) { optAdditionalOutput.needUrlKey = settingsPath; optAdditionalOutput.needUrlMethod = 1; } break; case taskResult.FileStatus.Err: case taskResult.FileStatus.ErrToReload: outputData.setStatus('err'); outputData.setData(statusInfo); break; case taskResult.FileStatus.SaveVersion: case taskResult.FileStatus.UpdateVersion: outputData.setStatus('updateversion'); break; } } function* addRandomKeyTaskCmd(cmd) { var task = yield* taskResult.addRandomKeyTask(cmd.getDocId()); cmd.setSaveKey(task.key); } function* saveParts(cmd) { var result = false; var saveType = cmd.getSaveType(); var filename; if (SAVE_TYPE_COMPLETE_ALL === saveType) { filename = 'Editor.bin'; } else { filename = 'Editor' + (cmd.getSaveIndex() || '') + '.bin'; } if (SAVE_TYPE_PART_START === saveType || SAVE_TYPE_COMPLETE_ALL === saveType) { yield* addRandomKeyTaskCmd(cmd); } if (cmd.getUrl()) { result = true; } else { var buffer = cmd.getData(); yield storage.putObject(cmd.getSaveKey() + '/' + filename, buffer, buffer.length); //delete data to prevent serialize into json cmd.data = null; result = (SAVE_TYPE_COMPLETE_ALL === saveType || SAVE_TYPE_COMPLETE === saveType); } return result; } function getSaveTask(cmd) { cmd.setData(null); var queueData = new commonDefines.TaskQueueData(); queueData.setCmd(cmd); queueData.setToFile('output.' + formatChecker.getStringFromFormat(cmd.getOutputFormat())); //todo paid //if (cmd.vkey) { // bool // bPaid; // Signature.getVKeyParams(cmd.vkey, out bPaid); // oTaskQueueData.m_bPaid = bPaid; //} return queueData; } function getUpdateResponse(cmd) { var updateTask = new taskResult.TaskResultData(); updateTask.key = cmd.getSaveKey() ? cmd.getSaveKey() : cmd.getDocId(); var statusInfo = cmd.getStatusInfo(); if (constants.NO_ERROR == statusInfo) { updateTask.status = taskResult.FileStatus.Ok; } else if (constants.CONVERT_DOWNLOAD == statusInfo) { updateTask.status = taskResult.FileStatus.ErrToReload; } else { updateTask.status = taskResult.FileStatus.Err; } updateTask.statusInfo = statusInfo; if (cmd.getTitle()) { updateTask.title = cmd.getTitle(); } return updateTask; } function* commandOpen(conn, cmd, outputData) { var task = new taskResult.TaskResultData(); task.key = cmd.getDocId(); task.format = cmd.getFormat(); task.status = taskResult.FileStatus.WaitQueue; task.statusInfo = constants.NO_ERROR; task.title = cmd.getTitle(); var upsertRes = yield taskResult.upsert(task); //var bCreate = (upsertRes.affectedRows == 1); var bExist = (upsertRes.affectedRows > 1); if (bExist) { var selectRes = yield taskResult.select(task); if (selectRes.length > 0) { var row = selectRes[0]; yield* getOutputData(cmd, outputData, cmd.getDocId(), row.tr_status, row.tr_status_info, conn); } } else { //add task cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CANVAS); cmd.setEmbeddedFonts(false); var dataQueue = new commonDefines.TaskQueueData(); dataQueue.setCmd(cmd); dataQueue.setToFile('Editor.bin'); yield* docsCoServer.addTask(dataQueue, constants.QUEUE_PRIORITY_HIGH); } } function* commandReopen(cmd) { var task = new taskResult.TaskResultData(); task.status = taskResult.FileStatus.WaitQueue; task.statusInfo = constants.NO_ERROR; var upsertRes = yield taskResult.update(task); if (upsertRes.affectedRows > 1) { //add task cmd.setSaveKey(cmd.getDocId()); cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CANVAS); cmd.setEmbeddedFonts(false); var dataQueue = new commonDefines.TaskQueueData(); dataQueue.setCmd(cmd); dataQueue.setToFile('Editor.bin'); dataQueue.setFromSettings(true); yield* docsCoServer.addTask(dataQueue, constants.QUEUE_PRIORITY_HIGH); } } function* commandSave(cmd, outputData) { var completeParts = yield* saveParts(cmd); if (completeParts) { var queueData = getSaveTask(cmd); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); } outputData.setStatus('ok'); outputData.setData(cmd.getSaveKey()); } function* commandSendMailMerge(cmd, outputData) { var completeParts = yield* saveParts(cmd); var isErr = false; if (completeParts) { isErr = true; var getRes = yield* docsCoServer.getCallback(cmd.getDocId()); if (getRes) { var mailMergeSend = cmd.getMailMergeSend(); mailMergeSend.setUrl(getRes.server.href); mailMergeSend.setBaseUrl(getRes.baseUrl); //меняем JsonKey и SaveKey, новый key нужет потому что за одну конвертацию делается часть, а json нужен всегда mailMergeSend.setJsonKey(cmd.getSaveKey()); yield* addRandomKeyTaskCmd(cmd); var queueData = getSaveTask(cmd); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); isErr = false; } } if (isErr) { outputData.setStatus('err'); outputData.setData(constants.UNKNOWN); } else { outputData.setStatus('ok'); outputData.setData(cmd.getSaveKey()); } } function* commandSfct(cmd, outputData) { yield* addRandomKeyTaskCmd(cmd); var queueData = getSaveTask(cmd); queueData.setFromChanges(true); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); outputData.setStatus('ok'); } function* commandImgurls(conn, cmd, outputData) { var supportedFormats; var urls; if ('imgurl' == cmd.getCommand()) { urls = [cmd.getData()]; supportedFormats = cfgTypesUpload || 'jpg'; } else { urls = cmd.getData(); supportedFormats = cfgTypesCopy || 'jpg'; } //todo Promise.all() var imageCount = 0; var outputUrls = []; for (var i = 0; i < urls.length; ++i) { var url = urls[i]; var data = undefined; if (url.startsWith('data:')) { var delimiterIndex = url.indexOf(','); if (-1 != delimiterIndex && (url.length - (delimiterIndex + 1)) * 0.75 <= cfgImageSize) { data = new Buffer(url.substring(delimiterIndex + 1), 'base64'); } } else { //todo data = yield* utils.downloadUrl(url, undefined, cfgImageSize); } var outputUrl = {url: null, path: null}; if (data) { var format = formatChecker.getFileFormat(data); var formatStr = formatChecker.getStringFromFormat(format); if (-1 !== supportedFormats.indexOf(formatStr)) { var userid = cmd.getUserId(); var imageIndex = cmd.getSaveIndex() + imageCount; imageCount++; var strLocalPath = 'media/' + utils.crc32(userid).toString(16) + '_image' + imageIndex + '.' + formatStr; var strPath = cmd.getDocId() + '/' + strLocalPath; yield storage.putObject(strPath, data, data.length); var imgUrl = yield storage.getSignedUrl(conn.baseUrl, strPath); outputUrl = {url: imgUrl, path: strLocalPath}; } } outputUrls.push(outputUrl); } outputData.setStatus('ok'); outputData.setData(outputUrls); } function* commandSaveFromOrigin(conn, cmd) { yield* addRandomKeyTaskCmd(cmd); cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF); cmd.setUserConnectionId(conn.user.id); var queueData = getSaveTask(cmd); queueData.setFromOrigin(true); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); } function commandGetSettings(cmd, outputData) { //todo license outputData.setStatus('ok'); outputData.setData(new OutputSettingsData(cmd.getFormat())); } function* commandSfcCallback(cmd) { logger.debug('Start commandSfcCallback docId = %s', cmd.getDocId()); var docId = cmd.getDocId(); var saveKey = cmd.getSaveKey(); var statusInfo = cmd.getStatusInfo(); var getRes = yield* docsCoServer.getCallback(docId); if (getRes) { logger.debug('commandSfcCallback docId = %s callback = %s', docId, getRes.server.href); var outputSfc = new OutputSfcData(); outputSfc.setKey(docId); if (cmd.getUserId()) { outputSfc.setUsers([cmd.getUserId()]); } if (constants.NO_ERROR != statusInfo && constants.CONVERT_CORRUPTED != statusInfo) { outputSfc.setStatus(docsCoServer.c_oAscServerStatus.Corrupted); } else { try { var data = yield storage.getObject(saveKey + '/changesHistory.json'); outputSfc.setChangeHistory(data.toString('utf-8')); outputSfc.setUrl(yield storage.getSignedUrl(getRes.baseUrl, saveKey + '/' + cmd.getTitle())); outputSfc.setChangeUrl(yield storage.getSignedUrl(getRes.baseUrl, saveKey + '/changes.zip')); } catch (e) { logger.error('error commandSfcCallback: docId = %s\r\n%s', docId, e.stack); } if (outputSfc.getUrl() && outputSfc.getUsers().length > 0) { outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MustSave); } else { outputSfc.setStatus(docsCoServer.c_oAscServerStatus.Corrupted); } } //if anybody in document stop save var hasEditors = yield* docsCoServer.hasEditors(docId); logger.debug('commandSfcCallback docId = %s hasEditors = %d', docId, hasEditors); if (!hasEditors) { yield docsCoServer.sendServerRequestPromise(getRes.server, JSON.stringify(outputSfc)); yield* docsCoServer.deleteCallback(docId); } } logger.debug('End commandSfcCallback docId = %s', cmd.getDocId()); } function* commandSendMMCallback(cmd) { logger.debug('Start commandSendMMCallback docId = %s', cmd.getDocId()); var docId = cmd.getDocId(); var saveKey = cmd.getSaveKey(); var statusInfo = cmd.getStatusInfo(); var outputSfc = new OutputSfcData(); outputSfc.setKey(docId); if (constants.NO_ERROR == statusInfo) { outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MailMerge); } else { outputSfc.setStatus(docsCoServer.c_oAscServerStatus.Corrupted); } var mailMergeSendData = cmd.getMailMergeSend(); var outputMailMerge = new OutputMailMerge(mailMergeSendData); outputSfc.setMailMerge(outputMailMerge); outputSfc.setUsers([mailMergeSendData.getUserId()]); var data = yield storage.getObject(saveKey + '/' + cmd.getTitle()); var xml = data.toString('utf8'); var files = xml.match(/[< ]file.*?\/>/g); var recordRemain = (mailMergeSendData.getRecordTo() - mailMergeSendData.getRecordFrom() + 1); var recordIndexStart = mailMergeSendData.getRecordCount() - recordRemain; for (var i = 0; i < files.length; ++i) { var file = files[i]; var fieldRes = /field=["'](.*?)["']/.exec(file); outputMailMerge.setTo(fieldRes[1]); outputMailMerge.setRecordIndex(recordIndexStart + i); var pathRes = /path=["'](.*?)["']/.exec(file); var url = yield storage.getSignedUrl(mailMergeSendData.getBaseUrl(), saveKey + '/' + pathRes[1]); outputSfc.setUrl(url); var server = docsCoServer.parseUrl(mailMergeSendData.getUrl()); yield docsCoServer.sendServerRequestPromise(server, JSON.stringify(outputSfc)); } var newRecordFrom = mailMergeSendData.getRecordFrom() + Math.max(files.length, 1); if (newRecordFrom <= mailMergeSendData.getRecordTo()) { mailMergeSendData.setRecordFrom(newRecordFrom); yield* addRandomKeyTaskCmd(cmd); var queueData = getSaveTask(cmd); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); } else { logger.debug('End MailMerge docId = %s', cmd.getDocId()); } logger.debug('End commandSendMMCallback docId = %s', cmd.getDocId()); } exports.openDocument = function(conn, input) { utils.spawn(function* () { var outputData; try { var cmd = new commonDefines.InputCommand(input.message); logger.debug('Start command %s', JSON.stringify(cmd)); outputData = new OutputData(cmd.getCommand()); switch (cmd.getCommand()) { case 'open': yield* commandOpen(conn, cmd, outputData); break; case 'reopen': yield* commandReopen(cmd); break; case 'savefromorigin': yield* commandSaveFromOrigin(conn, cmd); break; case 'imgurl': case 'imgurls': yield* commandImgurls(conn, cmd, outputData); break; case 'getsettings': commandGetSettings(cmd, outputData); break; default: outputData.setStatus('err'); outputData.setData(constants.UNKNOWN); break; } } catch (e) { logger.error('error openDocument: docId = %s type = %s\r\n%s', conn ? conn.docId : 'null', (input && input.type) ? input.type : 'null', e.stack); if (!outputData) { outputData = new OutputData(); } outputData.setStatus('err'); outputData.setData(constants.UNKNOWN); } finally { if (outputData && outputData.getStatus()) { logger.debug('Response command %s', JSON.stringify(outputData)); docsCoServer.sendData(conn, new OutputDataWrap('documentOpen', outputData)); } logger.debug('End command'); } }); }; exports.downloadAs = function(req, res) { utils.spawn(function* () { try { logger.debug('Start downloadAs request'); var strCmd = req.query['cmd']; var cmd = new commonDefines.InputCommand(JSON.parse(strCmd)); logger.debug('downloadAs cmd: %s', strCmd); cmd.setData(req.body); var outputData = new OutputData(cmd.getCommand()); switch (cmd.getCommand()) { case 'save': yield* commandSave(cmd, outputData); break; case 'sendmm': yield* commandSendMailMerge(cmd, outputData); break; case 'sfct': yield* commandSfct(cmd, outputData); break; default: outputData.setStatus('err'); outputData.setData(constants.UNKNOWN); break; } var strRes = JSON.stringify(outputData); res.send(strRes); logger.debug('End downloadAs request: %s', strRes); } catch (e) { logger.error('error downloadAs:\r\n%s', e.stack); res.sendStatus(400); } }); }; exports.saveFromChanges = function(docId, optFormat) { utils.spawn(function* () { try { logger.debug('Start saveFromChanges docId = %s', docId); if (null == optFormat) { optFormat = constants.AVS_OFFICESTUDIO_FILE_OTHER_TEAMLAB_INNER; } var cmd = new commonDefines.InputCommand(); cmd.setCommand('sfc'); cmd.setDocId(docId); cmd.setOutputFormat(optFormat); yield* addRandomKeyTaskCmd(cmd); var queueData = getSaveTask(cmd); queueData.setFromChanges(true); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_NORMAL); logger.debug('addTask saveFromChanges docId = %s', docId); } catch (e) { logger.error('error saveFromChanges: docId = %s\r\n%s', docId, e.stack); } }); }; exports.receiveTask = function(data, dataRaw) { utils.spawn(function* () { try { var task = new commonDefines.TaskQueueData(JSON.parse(data)); if (task) { var cmd = task.getCmd(); logger.debug('Start receiveTask command %s', JSON.stringify(cmd)); var updateTask = getUpdateResponse(cmd); var updateRes = yield taskResult.update(updateTask); if (updateRes.affectedRows > 0) { var outputData = new OutputData(cmd.getCommand()); var command = cmd.getCommand(); var additionalOutput = {needUrlKey: null, needUrlMethod: null}; if ('open' == command || 'reopen' == command) { yield* getOutputData(cmd, outputData, cmd.getDocId(), updateTask.status, updateTask.statusInfo, null, additionalOutput); } else if ('save' == command || 'savefromorigin' == command || 'sfct' == command) { yield* getOutputData(cmd, outputData, cmd.getSaveKey(), updateTask.status, updateTask.statusInfo, null, additionalOutput); } else if ('sfc' == command) { yield* commandSfcCallback(cmd); } else if ('sendmm' == command) { yield* commandSendMMCallback(cmd); } else if ('conv' == command) { //nothing } if (outputData.getStatus()) { logger.debug('send receiveTask %s', JSON.stringify(outputData)); var output = new OutputDataWrap('documentOpen', outputData); yield* docsCoServer.publish({ type: docsCoServer.PublishType.receiveTask, cmd: cmd, output: output, needUrlKey: additionalOutput.needUrlKey, needUrlMethod: additionalOutput.needUrlMethod }); } } yield* docsCoServer.removeResponse(dataRaw); logger.debug('End receiveTask command %s', JSON.stringify(cmd)); } } catch (err) { logger.error(err); } }); }; exports.OutputDataWrap = OutputDataWrap; exports.OutputData = OutputData; \ No newline at end of file +var sqlBase = require('./baseConnector'); var docsCoServer = require('./DocsCoServer'); var taskResult = require('./taskresult'); var logger = require('./../../Common/sources/logger'); var utils = require('./../../Common/sources/utils'); var constants = require('./../../Common/sources/constants'); var commonDefines = require('./../../Common/sources/commondefines'); var storage = require('./../../Common/sources/storage-base'); var formatChecker = require('./../../Common/sources/formatchecker'); var statsDClient = require('./../../Common/sources/statsdclient'); var config_server = require('config').get('services.CoAuthoring.server'); var config_utils = require('config').get('services.CoAuthoring.utils'); var cfgAutosaveEnable = config_server.get('editor_settings_autosave_enable'); var cfgAutosaveMininterval = config_server.get('editor_settings_autosave_mininterval'); var cfgCoauthoringUrl = config_server.get('editor_settings_coauthoring_url'); var cfgSpellcheckerUrl = config_server.get('editor_settings_spellchecker_url'); var cfgAnalyticsEnable = config_server.get('editor_settings_analytics_enable'); var cfgActiveconnectionsTrackingInterval = config_server.get('license_activeconnections_tracking_interval'); var cfgReaderformats = config_server.get('editor_settings_readerformats'); var cfgEditorformats = config_server.get('editor_settings_editorformats'); var cfgViewerformats = config_server.get('editor_settings_viewerformats'); var cfgTypesUpload = config_utils.get('limits_image_types_upload'); var cfgTypesCopy = config_utils.get('limits_image_types_copy'); var cfgImageSize = config_server.get('limits_image_size'); var SAVE_TYPE_PART_START = 0; var SAVE_TYPE_PART = 1; var SAVE_TYPE_COMPLETE = 2; var SAVE_TYPE_COMPLETE_ALL = 3; var clientStatsD = statsDClient.getClient(); function OutputDataWrap(type, data) { this['type'] = type; this['data'] = data; } OutputDataWrap.prototype = { fromObject: function(data) { this['type'] = data['type']; this['data'] = new OutputData(); this['data'].fromObject(data['data']); }, getType: function() { return this['type']; }, setType: function(data) { this['type'] = data; }, getData: function() { return this['data']; }, setData: function(data) { this['data'] = data; } }; function OutputData(type) { this['type'] = type; this['status'] = undefined; this['data'] = undefined; } OutputData.prototype = { fromObject: function(data) { this['type'] = data['type']; this['status'] = data['status']; this['data'] = data['data']; }, getType: function() { return this['type']; }, setType: function(data) { this['type'] = data; }, getStatus: function() { return this['status']; }, setStatus: function(data) { this['status'] = data; }, getData: function() { return this['data']; }, setData: function(data) { this['data'] = data; } }; function OutputSettingsData(rights, format, trackingInfo) { this['canLicense'] = true; // ToDo это должно браться из лицензии this['canEdit'] = true; this['canDownload'] = true; this['canCoAuthoring'] = true; this['canReaderMode'] = true; this['canAd'] = true; this['canBranding'] = true; this['isAutosaveEnable'] = cfgAutosaveEnable || true; this['AutosaveMinInterval'] = cfgAutosaveMininterval || 300; this['g_cAscCoAuthoringUrl'] = cfgCoauthoringUrl || ''; //так имена переменных написаны в JS this['g_cAscSpellCheckUrl'] = cfgSpellcheckerUrl || ''; //чтобы не светить лишние имена здесь используются такие же. this['isAnalyticsEnable'] = cfgAnalyticsEnable || false; this['trackingInfo'] = undefined; this['TrackingInterval'] = cfgActiveconnectionsTrackingInterval || 300; if (format) { format = format.toLowerCase(); var readerFormats = cfgReaderformats || ''; var readerFormatsArray = readerFormats.split(/[|,;]/); this['canReaderMode'] = -1 != readerFormatsArray.indexOf(format); var editorformats = cfgEditorformats || ''; var editorformatsArray = editorformats.split(/[|,;]/); this['canEdit'] = -1 != editorformatsArray.indexOf(format); var viewerformats = cfgViewerformats || ''; var viewerformatsArray = viewerformats.split(/[|,;]/); this['canDownload'] = -1 != viewerformatsArray.indexOf(format); } if (rights) { //todo } if (trackingInfo) { //todo } } function OutputSfcData() { this['key'] = undefined; this['status'] = undefined; this['url'] = undefined; this['changesurl'] = undefined; this['changeshistory'] = undefined; this['users'] = []; this['mailMerge'] = undefined; } OutputSfcData.prototype = { getKey: function() { return this['key']; }, setKey: function(data) { return this['key'] = data; }, getStatus: function() { return this['status']; }, setStatus: function(data) { return this['status'] = data; }, getUrl: function() { return this['url']; }, setUrl: function(data) { return this['url'] = data; }, getChangeUrl: function() { return this['changesurl']; }, setChangeUrl: function(data) { return this['changesurl'] = data; }, getChangeHistory: function() { return this['changeshistory']; }, setChangeHistory: function(data) { return this['changeshistory'] = data; }, getUsers: function() { return this['users']; }, setUsers: function(data) { return this['users'] = data; }, getMailMerge: function() { return this['mailMerge']; }, setMailMerge: function(data) { return this['mailMerge'] = data; } }; function OutputMailMerge(mailMergeSendData) { if (mailMergeSendData) { this['from'] = mailMergeSendData.getFrom(); this['message'] = mailMergeSendData.getMessage(); this['subject'] = mailMergeSendData.getSubject(); this['title'] = mailMergeSendData.getFileName(); var mailFormat = mailMergeSendData.getMailFormat(); switch (mailFormat) { case constants.AVS_OFFICESTUDIO_FILE_OTHER_HTMLZIP : this['type'] = 0; break; case constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_DOCX : this['type'] = 1; break; case constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF : this['type'] = 2; break; default : this['type'] = 0; break; } this['recordCount'] = mailMergeSendData.getRecordCount(); this['to'] = null; this['recordIndex'] = null; } else { this['from'] = null; this['message'] = null; this['subject'] = null; this['title'] = null; this['to'] = null; this['type'] = null; this['recordCount'] = null; this['recordIndex'] = null; } } OutputMailMerge.prototype = { getRecordIndex: function() { return this['recordIndex']; }, setRecordIndex: function(data) { return this['recordIndex'] = data; }, getTo: function() { return this['to']; }, setTo: function(data) { return this['to'] = data; } }; function getInsertStatisticString(affiliateId, filename, tag) { var dateNow = sqlBase.getDateTime(new Date()); var commandArg = [affiliateId, filename, dateNow, tag]; var commandArgEsc = commandArg.map(function(curVal) { return sqlBase.baseConnector.sqlEscape(curVal) }); return 'INSERT INTO file_statistic2 (fsc_affiliate, fsc_filename, fsc_time, fsc_tag) ' + 'VALUES (' + commandArgEsc.join(', ') + ');'; } function insertStatistic(affiliateId, filename, tag) { return new Promise(function(resolve, reject) { var sqlCommand = getInsertStatisticString(affiliateId, filename, tag); sqlBase.baseConnector.sqlQuery(sqlCommand, function(error, result) { if (error) { reject(error); } else { resolve(result); } }); }); } function* getOutputData(cmd, outputData, key, status, statusInfo, optConn, optAdditionalOutput) { switch (status) { case taskResult.FileStatus.Ok: //todo affilate var tag; var affiliateId = 'affiliateId'; switch (cmd.getEditorId()) { // case (int)LicenseInfo.EditorType.Spreadsheet: tag = 'open_sheet'; break; // case (int)LicenseInfo.EditorType.Presentation: tag = 'open_presentation'; break; default: tag = 'open_word'; break; } yield insertStatistic(affiliateId, cmd.getDocId(), tag); outputData.setStatus('ok'); if (optConn) { outputData.setData(yield storage.getSignedUrls(optConn.baseUrl, key)); } else if (optAdditionalOutput) { optAdditionalOutput.needUrlKey = key; optAdditionalOutput.needUrlMethod = 0; } break; case taskResult.FileStatus.NeedParams: outputData.setStatus('needparams'); var settingsPath = key + '/' + 'settings.json'; if (optConn) { outputData.setData(yield storage.getSignedUrl(optConn.baseUrl, settingsPath)); } else if (optAdditionalOutput) { optAdditionalOutput.needUrlKey = settingsPath; optAdditionalOutput.needUrlMethod = 1; } break; case taskResult.FileStatus.Err: case taskResult.FileStatus.ErrToReload: outputData.setStatus('err'); outputData.setData(statusInfo); break; case taskResult.FileStatus.SaveVersion: case taskResult.FileStatus.UpdateVersion: outputData.setStatus('updateversion'); break; } } function* addRandomKeyTaskCmd(cmd) { var task = yield* taskResult.addRandomKeyTask(cmd.getDocId()); cmd.setSaveKey(task.key); } function* saveParts(cmd) { var result = false; var saveType = cmd.getSaveType(); var filename; if (SAVE_TYPE_COMPLETE_ALL === saveType) { filename = 'Editor.bin'; } else { filename = 'Editor' + (cmd.getSaveIndex() || '') + '.bin'; } if (SAVE_TYPE_PART_START === saveType || SAVE_TYPE_COMPLETE_ALL === saveType) { yield* addRandomKeyTaskCmd(cmd); } if (cmd.getUrl()) { result = true; } else { var buffer = cmd.getData(); yield storage.putObject(cmd.getSaveKey() + '/' + filename, buffer, buffer.length); //delete data to prevent serialize into json cmd.data = null; result = (SAVE_TYPE_COMPLETE_ALL === saveType || SAVE_TYPE_COMPLETE === saveType); } return result; } function getSaveTask(cmd) { cmd.setData(null); var queueData = new commonDefines.TaskQueueData(); queueData.setCmd(cmd); queueData.setToFile('output.' + formatChecker.getStringFromFormat(cmd.getOutputFormat())); //todo paid //if (cmd.vkey) { // bool // bPaid; // Signature.getVKeyParams(cmd.vkey, out bPaid); // oTaskQueueData.m_bPaid = bPaid; //} return queueData; } function getUpdateResponse(cmd) { var updateTask = new taskResult.TaskResultData(); updateTask.key = cmd.getSaveKey() ? cmd.getSaveKey() : cmd.getDocId(); var statusInfo = cmd.getStatusInfo(); if (constants.NO_ERROR == statusInfo) { updateTask.status = taskResult.FileStatus.Ok; } else if (constants.CONVERT_DOWNLOAD == statusInfo) { updateTask.status = taskResult.FileStatus.ErrToReload; } else { updateTask.status = taskResult.FileStatus.Err; } updateTask.statusInfo = statusInfo; if (cmd.getTitle()) { updateTask.title = cmd.getTitle(); } return updateTask; } function* commandOpen(conn, cmd, outputData) { var task = new taskResult.TaskResultData(); task.key = cmd.getDocId(); task.format = cmd.getFormat(); task.status = taskResult.FileStatus.WaitQueue; task.statusInfo = constants.NO_ERROR; task.title = cmd.getTitle(); var upsertRes = yield taskResult.upsert(task); //var bCreate = (upsertRes.affectedRows == 1); var bExist = (upsertRes.affectedRows > 1); if (bExist) { var selectRes = yield taskResult.select(task); if (selectRes.length > 0) { var row = selectRes[0]; yield* getOutputData(cmd, outputData, cmd.getDocId(), row.tr_status, row.tr_status_info, conn); } } else { //add task cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CANVAS); cmd.setEmbeddedFonts(false); var dataQueue = new commonDefines.TaskQueueData(); dataQueue.setCmd(cmd); dataQueue.setToFile('Editor.bin'); yield* docsCoServer.addTask(dataQueue, constants.QUEUE_PRIORITY_HIGH); } } function* commandReopen(cmd) { var task = new taskResult.TaskResultData(); task.status = taskResult.FileStatus.WaitQueue; task.statusInfo = constants.NO_ERROR; var upsertRes = yield taskResult.update(task); if (upsertRes.affectedRows > 1) { //add task cmd.setSaveKey(cmd.getDocId()); cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CANVAS); cmd.setEmbeddedFonts(false); var dataQueue = new commonDefines.TaskQueueData(); dataQueue.setCmd(cmd); dataQueue.setToFile('Editor.bin'); dataQueue.setFromSettings(true); yield* docsCoServer.addTask(dataQueue, constants.QUEUE_PRIORITY_HIGH); } } function* commandSave(cmd, outputData) { var completeParts = yield* saveParts(cmd); if (completeParts) { var queueData = getSaveTask(cmd); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); } outputData.setStatus('ok'); outputData.setData(cmd.getSaveKey()); } function* commandSendMailMerge(cmd, outputData) { var completeParts = yield* saveParts(cmd); var isErr = false; if (completeParts) { isErr = true; var getRes = yield* docsCoServer.getCallback(cmd.getDocId()); if (getRes) { var mailMergeSend = cmd.getMailMergeSend(); mailMergeSend.setUrl(getRes.server.href); mailMergeSend.setBaseUrl(getRes.baseUrl); //меняем JsonKey и SaveKey, новый key нужет потому что за одну конвертацию делается часть, а json нужен всегда mailMergeSend.setJsonKey(cmd.getSaveKey()); yield* addRandomKeyTaskCmd(cmd); var queueData = getSaveTask(cmd); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); isErr = false; } } if (isErr) { outputData.setStatus('err'); outputData.setData(constants.UNKNOWN); } else { outputData.setStatus('ok'); outputData.setData(cmd.getSaveKey()); } } function* commandSfct(cmd, outputData) { yield* addRandomKeyTaskCmd(cmd); var queueData = getSaveTask(cmd); queueData.setFromChanges(true); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); outputData.setStatus('ok'); } function* commandImgurls(conn, cmd, outputData) { var supportedFormats; var urls; if ('imgurl' == cmd.getCommand()) { urls = [cmd.getData()]; supportedFormats = cfgTypesUpload || 'jpg'; } else { urls = cmd.getData(); supportedFormats = cfgTypesCopy || 'jpg'; } //todo Promise.all() var imageCount = 0; var outputUrls = []; for (var i = 0; i < urls.length; ++i) { var url = urls[i]; var data = undefined; if (url.startsWith('data:')) { var delimiterIndex = url.indexOf(','); if (-1 != delimiterIndex && (url.length - (delimiterIndex + 1)) * 0.75 <= cfgImageSize) { data = new Buffer(url.substring(delimiterIndex + 1), 'base64'); } } else { //todo data = yield* utils.downloadUrl(url, undefined, cfgImageSize); } var outputUrl = {url: null, path: null}; if (data) { var format = formatChecker.getFileFormat(data); var formatStr = formatChecker.getStringFromFormat(format); if (-1 !== supportedFormats.indexOf(formatStr)) { var userid = cmd.getUserId(); var imageIndex = cmd.getSaveIndex() + imageCount; imageCount++; var strLocalPath = 'media/' + utils.crc32(userid).toString(16) + '_image' + imageIndex + '.' + formatStr; var strPath = cmd.getDocId() + '/' + strLocalPath; yield storage.putObject(strPath, data, data.length); var imgUrl = yield storage.getSignedUrl(conn.baseUrl, strPath); outputUrl = {url: imgUrl, path: strLocalPath}; } } outputUrls.push(outputUrl); } outputData.setStatus('ok'); outputData.setData(outputUrls); } function* commandSaveFromOrigin(conn, cmd) { yield* addRandomKeyTaskCmd(cmd); cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF); cmd.setUserConnectionId(conn.user.id); var queueData = getSaveTask(cmd); queueData.setFromOrigin(true); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); } function commandGetSettings(cmd, outputData) { //todo license outputData.setStatus('ok'); outputData.setData(new OutputSettingsData(cmd.getFormat())); } function* commandSfcCallback(cmd) { logger.debug('Start commandSfcCallback docId = %s', cmd.getDocId()); var docId = cmd.getDocId(); var saveKey = cmd.getSaveKey(); var statusInfo = cmd.getStatusInfo(); var getRes = yield* docsCoServer.getCallback(docId); if (getRes) { logger.debug('commandSfcCallback docId = %s callback = %s', docId, getRes.server.href); var outputSfc = new OutputSfcData(); outputSfc.setKey(docId); if (cmd.getUserId()) { outputSfc.setUsers([cmd.getUserId()]); } if (constants.NO_ERROR != statusInfo && constants.CONVERT_CORRUPTED != statusInfo) { outputSfc.setStatus(docsCoServer.c_oAscServerStatus.Corrupted); } else { try { var data = yield storage.getObject(saveKey + '/changesHistory.json'); outputSfc.setChangeHistory(data.toString('utf-8')); outputSfc.setUrl(yield storage.getSignedUrl(getRes.baseUrl, saveKey + '/' + cmd.getTitle())); outputSfc.setChangeUrl(yield storage.getSignedUrl(getRes.baseUrl, saveKey + '/changes.zip')); } catch (e) { logger.error('error commandSfcCallback: docId = %s\r\n%s', docId, e.stack); } if (outputSfc.getUrl() && outputSfc.getUsers().length > 0) { outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MustSave); } else { outputSfc.setStatus(docsCoServer.c_oAscServerStatus.Corrupted); } } //if anybody in document stop save var hasEditors = yield* docsCoServer.hasEditors(docId); logger.debug('commandSfcCallback docId = %s hasEditors = %d', docId, hasEditors); if (!hasEditors) { yield docsCoServer.sendServerRequestPromise(getRes.server, JSON.stringify(outputSfc)); yield* docsCoServer.deleteCallback(docId); } } logger.debug('End commandSfcCallback docId = %s', cmd.getDocId()); } function* commandSendMMCallback(cmd) { logger.debug('Start commandSendMMCallback docId = %s', cmd.getDocId()); var docId = cmd.getDocId(); var saveKey = cmd.getSaveKey(); var statusInfo = cmd.getStatusInfo(); var outputSfc = new OutputSfcData(); outputSfc.setKey(docId); if (constants.NO_ERROR == statusInfo) { outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MailMerge); } else { outputSfc.setStatus(docsCoServer.c_oAscServerStatus.Corrupted); } var mailMergeSendData = cmd.getMailMergeSend(); var outputMailMerge = new OutputMailMerge(mailMergeSendData); outputSfc.setMailMerge(outputMailMerge); outputSfc.setUsers([mailMergeSendData.getUserId()]); var data = yield storage.getObject(saveKey + '/' + cmd.getTitle()); var xml = data.toString('utf8'); var files = xml.match(/[< ]file.*?\/>/g); var recordRemain = (mailMergeSendData.getRecordTo() - mailMergeSendData.getRecordFrom() + 1); var recordIndexStart = mailMergeSendData.getRecordCount() - recordRemain; for (var i = 0; i < files.length; ++i) { var file = files[i]; var fieldRes = /field=["'](.*?)["']/.exec(file); outputMailMerge.setTo(fieldRes[1]); outputMailMerge.setRecordIndex(recordIndexStart + i); var pathRes = /path=["'](.*?)["']/.exec(file); var url = yield storage.getSignedUrl(mailMergeSendData.getBaseUrl(), saveKey + '/' + pathRes[1]); outputSfc.setUrl(url); var server = docsCoServer.parseUrl(mailMergeSendData.getUrl()); yield docsCoServer.sendServerRequestPromise(server, JSON.stringify(outputSfc)); } var newRecordFrom = mailMergeSendData.getRecordFrom() + Math.max(files.length, 1); if (newRecordFrom <= mailMergeSendData.getRecordTo()) { mailMergeSendData.setRecordFrom(newRecordFrom); yield* addRandomKeyTaskCmd(cmd); var queueData = getSaveTask(cmd); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); } else { logger.debug('End MailMerge docId = %s', cmd.getDocId()); } logger.debug('End commandSendMMCallback docId = %s', cmd.getDocId()); } exports.openDocument = function(conn, input) { utils.spawn(function* () { var outputData; try { var cmd = new commonDefines.InputCommand(input.message); logger.debug('Start command %s', JSON.stringify(cmd)); outputData = new OutputData(cmd.getCommand()); switch (cmd.getCommand()) { case 'open': yield* commandOpen(conn, cmd, outputData); break; case 'reopen': yield* commandReopen(cmd); break; case 'savefromorigin': yield* commandSaveFromOrigin(conn, cmd); break; case 'imgurl': case 'imgurls': yield* commandImgurls(conn, cmd, outputData); break; case 'getsettings': commandGetSettings(cmd, outputData); break; default: outputData.setStatus('err'); outputData.setData(constants.UNKNOWN); break; } } catch (e) { logger.error('error openDocument: docId = %s type = %s\r\n%s', conn ? conn.docId : 'null', (input && input.type) ? input.type : 'null', e.stack); if (!outputData) { outputData = new OutputData(); } outputData.setStatus('err'); outputData.setData(constants.UNKNOWN); } finally { if (outputData && outputData.getStatus()) { logger.debug('Response command %s', JSON.stringify(outputData)); docsCoServer.sendData(conn, new OutputDataWrap('documentOpen', outputData)); } logger.debug('End command'); } }); }; exports.downloadAs = function(req, res) { utils.spawn(function* () { try { var startDate = null; if(clientStatsD) { startDate = new Date(); } logger.debug('Start downloadAs request'); var strCmd = req.query['cmd']; var cmd = new commonDefines.InputCommand(JSON.parse(strCmd)); logger.debug('downloadAs cmd: %s', strCmd); cmd.setData(req.body); var outputData = new OutputData(cmd.getCommand()); switch (cmd.getCommand()) { case 'save': yield* commandSave(cmd, outputData); break; case 'sendmm': yield* commandSendMailMerge(cmd, outputData); break; case 'sfct': yield* commandSfct(cmd, outputData); break; default: outputData.setStatus('err'); outputData.setData(constants.UNKNOWN); break; } var strRes = JSON.stringify(outputData); res.send(strRes); logger.debug('End downloadAs request: %s', strRes); if(clientStatsD) { clientStatsD.timing('downloadAs_' + cmd.getCommand(), new Date() - startDate); } } catch (e) { logger.error('error downloadAs:\r\n%s', e.stack); res.sendStatus(400); } }); }; exports.saveFromChanges = function(docId, optFormat) { utils.spawn(function* () { try { var startDate = null; if(clientStatsD) { startDate = new Date(); } logger.debug('Start saveFromChanges docId = %s', docId); if (null == optFormat) { optFormat = constants.AVS_OFFICESTUDIO_FILE_OTHER_TEAMLAB_INNER; } var cmd = new commonDefines.InputCommand(); cmd.setCommand('sfc'); cmd.setDocId(docId); cmd.setOutputFormat(optFormat); yield* addRandomKeyTaskCmd(cmd); var queueData = getSaveTask(cmd); queueData.setFromChanges(true); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_NORMAL); logger.debug('addTask saveFromChanges docId = %s', docId); if(clientStatsD) { clientStatsD.timing('saveFromChanges', new Date() - startDate); } } catch (e) { logger.error('error saveFromChanges: docId = %s\r\n%s', docId, e.stack); } }); }; exports.receiveTask = function(data, dataRaw) { utils.spawn(function* () { try { var task = new commonDefines.TaskQueueData(JSON.parse(data)); if (task) { var cmd = task.getCmd(); logger.debug('Start receiveTask command %s', JSON.stringify(cmd)); var updateTask = getUpdateResponse(cmd); var updateRes = yield taskResult.update(updateTask); if (updateRes.affectedRows > 0) { var outputData = new OutputData(cmd.getCommand()); var command = cmd.getCommand(); var additionalOutput = {needUrlKey: null, needUrlMethod: null}; if ('open' == command || 'reopen' == command) { yield* getOutputData(cmd, outputData, cmd.getDocId(), updateTask.status, updateTask.statusInfo, null, additionalOutput); } else if ('save' == command || 'savefromorigin' == command || 'sfct' == command) { yield* getOutputData(cmd, outputData, cmd.getSaveKey(), updateTask.status, updateTask.statusInfo, null, additionalOutput); } else if ('sfc' == command) { yield* commandSfcCallback(cmd); } else if ('sendmm' == command) { yield* commandSendMMCallback(cmd); } else if ('conv' == command) { //nothing } if (outputData.getStatus()) { logger.debug('send receiveTask %s', JSON.stringify(outputData)); var output = new OutputDataWrap('documentOpen', outputData); yield* docsCoServer.publish({ type: docsCoServer.PublishType.receiveTask, cmd: cmd, output: output, needUrlKey: additionalOutput.needUrlKey, needUrlMethod: additionalOutput.needUrlMethod }); } } yield* docsCoServer.removeResponse(dataRaw); logger.debug('End receiveTask command %s', JSON.stringify(cmd)); } } catch (err) { logger.error(err); } }); }; exports.OutputDataWrap = OutputDataWrap; exports.OutputData = OutputData; \ No newline at end of file diff --git a/CoAuthoring/sources/converterservice.js b/CoAuthoring/sources/converterservice.js index 7ae9cca1..2f829994 100644 --- a/CoAuthoring/sources/converterservice.js +++ b/CoAuthoring/sources/converterservice.js @@ -6,11 +6,14 @@ var commonDefines = require('./../../Common/sources/commondefines'); var docsCoServer = require('./DocsCoServer'); var storage = require('./../../Common/sources/storage-base'); var formatChecker = require('./../../Common/sources/formatchecker'); +var statsDClient = require('./../../Common/sources/statsdclient'); //todo var CONVERT_TIMEOUT = 6 * 60 * 1000; var CONVERT_ASYNC_DELAY = 1000; +var clientStatsD = statsDClient.getClient(); + function* getConvertStatus(cmd, selectRes, req) { var status = {url: undefined, err: constants.NO_ERROR}; if (selectRes.length > 0) { @@ -36,6 +39,10 @@ function* getConvertStatus(cmd, selectRes, req) { exports.convert = function(req, res) { utils.spawn(function* () { try { + var startDate = null; + if(clientStatsD) { + startDate = new Date(); + } logger.debug('Start convert request'); var cmd = new commonDefines.InputCommand(); cmd.setCommand('conv'); @@ -92,6 +99,9 @@ exports.convert = function(req, res) { } utils.fillXmlResponse(res, status.url, status.err); logger.debug('End convert request'); + if(clientStatsD) { + clientStatsD.timing('convertservice', new Date() - startDate); + } } catch (e) { logger.error('error convert:\r\n%s', e.stack); diff --git a/CoAuthoring/sources/fontservice.js b/CoAuthoring/sources/fontservice.js index aa8147b1..5ed24e56 100644 --- a/CoAuthoring/sources/fontservice.js +++ b/CoAuthoring/sources/fontservice.js @@ -6,6 +6,7 @@ var base64 = require('base64-stream'); var utils = require('./../../Common/sources/utils'); var formatChecker = require('./../../Common/sources/formatchecker'); var logger = require('./../../Common/sources/logger'); +var statsDClient = require('./../../Common/sources/statsdclient'); var config = require('config').get('services.CoAuthoring.utils'); var cfgFontDir = config.get('utils_common_fontdir'); @@ -17,6 +18,7 @@ var BYTE_MAX_VALUE = 255; var GUID_ODTTF = [0xA0, 0x66, 0xD6, 0x20, 0x14, 0x96, 0x47, 0xfa, 0x95, 0x69, 0xB8, 0x50, 0xB0, 0x41, 0x49, 0x48]; var fontNameToFullPath = {}; +var clientStatsD = statsDClient.getClient(); function ZBase32Encoder() { this.encodingTable = 'ybndrfg8ejkmcpqxot1uwisza345h769'; @@ -175,6 +177,10 @@ var zBase32Encoder = new ZBase32Encoder(); exports.getFont = function(req, res) { utils.spawn(function* () { try { + var startDate = null; + if(clientStatsD) { + startDate = new Date(); + } logger.debug('Start getFont request'); var fontname = req.params.fontname; logger.debug('fontname:' + fontname); @@ -230,6 +236,9 @@ exports.getFont = function(req, res) { res.sendStatus(404); } logger.debug('End getFont request'); + if(clientStatsD) { + clientStatsD.timing('getFont', new Date() - startDate); + } } catch (e) { logger.error('error getFont:\r\n%s', e.stack); diff --git a/Common/config/default.json b/Common/config/default.json index d11e50f8..88e389f1 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -1,4 +1,10 @@ { + "statsd": { + "useMetrics": false, + "host": "localhost", + "port": "8125", + "prefix": "ds." + }, "log": { "appenders": [{ "type": "console" diff --git a/Common/package.json b/Common/package.json index a4c4ddf0..a72c7fd7 100644 --- a/Common/package.json +++ b/Common/package.json @@ -11,7 +11,8 @@ "log4js": "0.6.26", "mime": "^1.3.4", "mkdirp": "^0.5.1", + "node-statsd": "^0.1.1", "node-zip": "^1.1.1", - "config": "^1.15.0" + "config": "^1.15.0" } } diff --git a/Common/sources/statsdclient.js b/Common/sources/statsdclient.js new file mode 100644 index 00000000..327a19ae --- /dev/null +++ b/Common/sources/statsdclient.js @@ -0,0 +1,16 @@ +var statsD = require('node-statsd'); +var configStatsD = require('config').get('statsd'); + +var cfgStatsDUseMetrics = configStatsD.get('useMetrics'); +var cfgStatsDHost = configStatsD.get('host'); +var cfgStatsDPort = configStatsD.get('port'); +var cfgStatsDPrefix = configStatsD.get('prefix'); + +var clientStatsD = null; +if(cfgStatsDUseMetrics) { + clientStatsD = new statsD({host: cfgStatsDHost, port:cfgStatsDPort, prefix: cfgStatsDPrefix}); +} + +exports.getClient = function() { + return clientStatsD; +}; diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index 3029beb6..8727d44f 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -17,6 +17,7 @@ var logger = require('./../../Common/sources/logger'); var constants = require('./../../Common/sources/constants'); var nodeZip = require('./../../Common/node_modules/node-zip'); var baseConnector = require('./../../CoAuthoring/sources/baseConnector'); +var statsDClient = require('./../../Common/sources/statsdclient'); var queueService = require('./../../Common/sources/' + config.get('queue.name')); var cfgMaxDownloadBytes = configConverter.has('maxDownloadBytes') ? configConverter.get('maxDownloadBytes') : 100000000; @@ -29,6 +30,7 @@ var cfgErrorFiles = configConverter.get('errorfiles'); var TEMP_PREFIX = 'ASC_CONVERT'; var queue = null; +var clientStatsD = statsDClient.getClient(); function TaskQueueDataConvert(task) { var cmd = task.getCmd(); @@ -458,6 +460,11 @@ function deleteFolderRecursive(strPath) { } function* ExecuteTask(task) { + var startDate = null; + var curDate = null; + if(clientStatsD) { + startDate = curDate = new Date(); + } var resData; var tempDirs; var getTaskTime = new Date(); @@ -476,12 +483,20 @@ function* ExecuteTask(task) { error = constants.CONVERT_DOWNLOAD; logger.error(err); } + if(clientStatsD) { + clientStatsD.timing('downloadFile', new Date() - curDate); + curDate = new Date(); + } if (constants.NO_ERROR === error) { error = processDownloadFile(dataConvert, cmd.format); } } else if (cmd.getSaveKey()) { yield* downloadFileFromStorage(cmd.getDocId(), cmd.getDocId(), tempDirs.source); logger.debug('downloadFileFromStorage complete(id=%s)', dataConvert.key); + if(clientStatsD) { + clientStatsD.timing('downloadFileFromStorage', new Date() - curDate); + curDate = new Date(); + } yield* processDownloadFromStorage(dataConvert, cmd, task, tempDirs); } else { error = constants.UNKNOWN; @@ -502,12 +517,27 @@ function* ExecuteTask(task) { var waitMS = (task.getVisibilityTimeout() || 600) * 1000 - (new Date().getTime() - getTaskTime.getTime()); childRes = childProcess.spawnSync(cfgFilePath, childArgs, {timeout: waitMS}); logger.debug('ExitCode (code=%d;signal=%s;id=%s)', childRes.status, childRes.signal, dataConvert.key); + if(clientStatsD) { + clientStatsD.timing('spawnSync', new Date() - curDate); + curDate = new Date(); + } } resData = yield* postProcess(cmd, dataConvert, tempDirs, childRes, error); logger.debug('postProcess (id=%s)', dataConvert.key); + if(clientStatsD) { + clientStatsD.timing('postProcess', new Date() - curDate); + curDate = new Date(); + } if (tempDirs) { deleteFolderRecursive(tempDirs.temp); logger.debug('deleteFolderRecursive (id=%s)', dataConvert.key); + if(clientStatsD) { + clientStatsD.timing('deleteFolderRecursive', new Date() - curDate); + curDate = new Date(); + } + } + if(clientStatsD) { + clientStatsD.timing('convert', new Date() - startDate); } return resData; } diff --git a/Metrics/config/config.js b/Metrics/config/config.js new file mode 100644 index 00000000..d9151a2e --- /dev/null +++ b/Metrics/config/config.js @@ -0,0 +1,106 @@ +/* + +Required Variables: + + port: StatsD listening port [default: 8125] + +Graphite Required Variables: + +(Leave these unset to avoid sending stats to Graphite. + Set debug flag and leave these unset to run in 'dry' debug mode - + useful for testing statsd clients without a Graphite server.) + + graphiteHost: hostname or IP of Graphite server + graphitePort: port of Graphite server + +Optional Variables: + + backends: an array of backends to load. Each backend must exist + by name in the directory backends/. If not specified, + the default graphite backend will be loaded. + debug: debug flag [default: false] + address: address to listen on over UDP [default: 0.0.0.0] + address_ipv6: defines if the address is an IPv4 or IPv6 address [true or false, default: false] + port: port to listen for messages on over UDP [default: 8125] + mgmt_address: address to run the management TCP interface on + [default: 0.0.0.0] + mgmt_port: port to run the management TCP interface on [default: 8126] + title : Allows for overriding the process title. [default: statsd] + if set to false, will not override the process title and let the OS set it. + The length of the title has to be less than or equal to the binary name + cli arguments + NOTE: This does not work on Mac's with node versions prior to v0.10 + + healthStatus: default health status to be returned and statsd process starts ['up' or 'down', default: 'up'] + dumpMessages: log all incoming messages + flushInterval: interval (in ms) to flush to Graphite + percentThreshold: for time information, calculate the Nth percentile(s) + (can be a single value or list of floating-point values) + negative values mean to use "top" Nth percentile(s) values + [%, default: 90] + flush_counts: send stats_counts metrics [default: true] + + keyFlush: log the most frequently sent keys [object, default: undefined] + interval: how often to log frequent keys [ms, default: 0] + percent: percentage of frequent keys to log [%, default: 100] + log: location of log file for frequent keys [default: STDOUT] + deleteIdleStats: don't send values to graphite for inactive counters, sets, gauges, or timeers + as opposed to sending 0. For gauges, this unsets the gauge (instead of sending + the previous value). Can be individually overriden. [default: false] + deleteGauges : don't send values to graphite for inactive gauges, as opposed to sending the previous value [default: false] + deleteTimers: don't send values to graphite for inactive timers, as opposed to sending 0 [default: false] + deleteSets: don't send values to graphite for inactive sets, as opposed to sending 0 [default: false] + deleteCounters: don't send values to graphite for inactive counters, as opposed to sending 0 [default: false] + prefixStats: prefix to use for the statsd statistics data for this running instance of statsd [default: statsd] + applies to both legacy and new namespacing + + console: + prettyprint: whether to prettyprint the console backend + output [true or false, default: true] + + log: log settings [object, default: undefined] + backend: where to log: stdout or syslog [string, default: stdout] + application: name of the application for syslog [string, default: statsd] + level: log level for [node-]syslog [string, default: LOG_INFO] + + graphite: + legacyNamespace: use the legacy namespace [default: true] + globalPrefix: global prefix to use for sending stats to graphite [default: "stats"] + prefixCounter: graphite prefix for counter metrics [default: "counters"] + prefixTimer: graphite prefix for timer metrics [default: "timers"] + prefixGauge: graphite prefix for gauge metrics [default: "gauges"] + prefixSet: graphite prefix for set metrics [default: "sets"] + globalSuffix: global suffix to use for sending stats to graphite [default: ""] + This is particularly useful for sending per host stats by + settings this value to: require('os').hostname().split('.')[0] + + repeater: an array of hashes of the for host: and port: + that details other statsd servers to which the received + packets should be "repeated" (duplicated to). + e.g. [ { host: '10.10.10.10', port: 8125 }, + { host: 'observer', port: 88125 } ] + + repeaterProtocol: whether to use udp4 or udp6 for repeaters. + ["udp4" or "udp6", default: "udp4"] + + histogram: for timers, an array of mappings of strings (to match metrics) and + corresponding ordered non-inclusive upper limits of bins. + For all matching metrics, histograms are maintained over + time by writing the frequencies for all bins. + 'inf' means infinity. A lower limit of 0 is assumed. + default: [], meaning no histograms for any timer. + First match wins. examples: + * histogram to only track render durations, with unequal + class intervals and catchall for outliers: + [ { metric: 'render', bins: [ 0.01, 0.1, 1, 10, 'inf'] } ] + * histogram for all timers except 'foo' related, + equal class interval and catchall for outliers: + [ { metric: 'foo', bins: [] }, + { metric: '', bins: [ 50, 100, 150, 200, 'inf'] } ] + +*/ +{ + port: 8125 +, deleteIdleStats: true +, flushInterval: 600000 +, backends: [ "./backends/console" ] +} diff --git a/Metrics/package.json b/Metrics/package.json new file mode 100644 index 00000000..59e0f407 --- /dev/null +++ b/Metrics/package.json @@ -0,0 +1,9 @@ +{ + "name": "metrics", + "version": "1.0.0", + "homepage": "http://www.onlyoffice.com", + "private": true, + "dependencies": { + "statsd": "^0.7.2" + } +}