From 46b5f04600daaef202df84500637459ade6bb188 Mon Sep 17 00:00:00 2001 From: konovalovsergey Date: Wed, 20 Apr 2016 11:45:34 +0300 Subject: [PATCH] join auth and open command, userIndex move from redis to mysql. --- Common/config/default.json | 1 - Common/sources/commondefines.js | 6 ---- Common/sources/constants.js | 1 - DocService/sources/DocsCoServer.js | 33 +++++++++-------- DocService/sources/canvasservice.js | 2 +- DocService/sources/converterservice.js | 4 +-- DocService/sources/taskresult.js | 49 ++++++++++++++++++-------- schema/createdb.sql | 2 ++ 8 files changed, 56 insertions(+), 42 deletions(-) diff --git a/Common/config/default.json b/Common/config/default.json index d9407fc7..1b685f78 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -85,7 +85,6 @@ "maxChanges": 1000 }, "expire": { - "userindex": 604800, "saveLock": 60, "presence": 300, "locks": 604800, diff --git a/Common/sources/commondefines.js b/Common/sources/commondefines.js index 596b1a4d..e43c0498 100644 --- a/Common/sources/commondefines.js +++ b/Common/sources/commondefines.js @@ -100,12 +100,6 @@ InputCommand.prototype = { setData: function(data) { this['data'] = data; }, - getEditorId: function() { - return this['editorid']; - }, - setEditorId: function(data) { - this['editorid'] = data; - }, getFormat: function() { return this['format']; }, diff --git a/Common/sources/constants.js b/Common/sources/constants.js index a33a125a..5d68299b 100644 --- a/Common/sources/constants.js +++ b/Common/sources/constants.js @@ -121,7 +121,6 @@ exports.EDITOR_TYPE_PRESENTATION = 2; exports.EDITOR_TYPE_CONVERTATION = 3; exports.REDIS_KEY_PUBSUB = 'pubsub'; -exports.REDIS_KEY_USER_INDEX = 'userindex:'; exports.REDIS_KEY_SAVE_LOCK = 'savelock:'; exports.REDIS_KEY_PRESENCE_HASH = 'presence:hash:'; exports.REDIS_KEY_PRESENCE_SET = 'presence:set:'; diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 2a902628..c0525a38 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -66,7 +66,6 @@ var cfgCallbackRequestTimeout = config.get('server.callbackRequestTimeout'); var cfgPubSubMaxChanges = config.get('pubsub.maxChanges'); var cfgRedisPrefix = config.get('redis.prefix'); -var cfgExpUserIndex = config.get('expire.userindex'); var cfgExpSaveLock = config.get('expire.saveLock'); var cfgExpPresence = config.get('expire.presence'); var cfgExpLocks = config.get('expire.locks'); @@ -78,7 +77,6 @@ var cfgExpForceSave = config.get('expire.forcesave'); var cfgExpSaved = config.get('expire.saved'); var cfgSockjsUrl = config.get('server.sockjsUrl'); -var redisKeyUserIndex = cfgRedisPrefix + constants.REDIS_KEY_USER_INDEX; var redisKeySaveLock = cfgRedisPrefix + constants.REDIS_KEY_SAVE_LOCK; var redisKeyPresenceHash = cfgRedisPrefix + constants.REDIS_KEY_PRESENCE_HASH; var redisKeyPresenceSet = cfgRedisPrefix + constants.REDIS_KEY_PRESENCE_SET; @@ -706,13 +704,10 @@ function* bindEvents(docId, callback, baseUrl, opt_userAction) { yield* sendStatusDocument(docId, bChangeBase, userAction, oCallbackUrl, baseUrl); } -function* cleanDocumentOnExit(docId, deleteChanges, deleteUserIndex) { +function* cleanDocumentOnExit(docId, deleteChanges) { //clean redis var redisArgs = [redisClient, redisClient.del, redisKeyLocks + docId, redisKeyPresenceSet + docId, redisKeyPresenceHash + docId, redisKeyMessage + docId, redisKeyChangeIndex + docId, redisKeyForceSave + docId, redisKeyLastSave + docId]; - if (deleteUserIndex) { - redisArgs.push(redisKeyUserIndex + docId); - } utils.promiseRedis.apply(this, redisArgs); //remove callback yield* deleteCallback(docId); @@ -727,7 +722,7 @@ function* cleanDocumentOnExitNoChanges(docId, opt_userId) { yield* sendStatusDocument(docId, c_oAscChangeBase.All, userAction); //если пользователь зашел в документ, соединение порвалось, на сервере удалилась вся информация, //при восстановлении соединения userIndex сохранится и он совпадет с userIndex следующего пользователя - yield* cleanDocumentOnExit(docId, false, false); + yield* cleanDocumentOnExit(docId, false); } function* _createSaveTimer(docId, opt_userId, opt_queue, opt_noDelay) { @@ -839,7 +834,8 @@ exports.install = function(server, callbackFunction) { yield* closeDocument(conn, false); break; case 'openDocument' : - canvasService.openDocument(conn, data); + var cmd = new commonDefines.InputCommand(data.message); + yield canvasService.openDocument(conn, cmd); break; case 'changesError': logger.error("changesError %s", data.stack); @@ -1235,20 +1231,20 @@ exports.install = function(server, callbackFunction) { } var bIsRestore = null != data.sessionId; - + var upsertRes = null; + var cmd = null; // Если восстанавливаем, индекс тоже восстанавливаем var curIndexUser; if (bIsRestore) { curIndexUser = user.indexUser; } else { - curIndexUser = 1; - var multi = redisClient.multi([ - ['incr', redisKeyUserIndex + docId], - ['expire', redisKeyUserIndex + docId, cfgExpUserIndex] - ]); - var replies = yield utils.promiseRedis(multi, multi.exec); - if(replies){ - curIndexUser = replies[0]; + cmd = new commonDefines.InputCommand(data.openCmd); + upsertRes = yield canvasService.commandOpenStartPromise(cmd, true); + var bCreate = upsertRes.affectedRows == 1; + if (bCreate) { + curIndexUser = 1; + } else { + curIndexUser = upsertRes.insertId; } } @@ -1350,6 +1346,9 @@ exports.install = function(server, callbackFunction) { } else { conn.sessionId = conn.id; yield* endAuth(conn, false, data.documentCallbackUrl); + if (upsertRes && cmd) { + yield canvasService.openDocument(conn, cmd, upsertRes); + } } } } diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index 6fd467dd..b5c5fc2d 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -1 +1 @@ -var pathModule = require('path'); var urlModule = require('url'); var co = require('co'); var sqlBase = require('./baseConnector'); var docsCoServer = require('./DocsCoServer'); var taskResult = require('./taskresult'); var logger = require('./../../Common/sources/logger'); var utils = require('./../../Common/sources/utils'); var constants = require('./../../Common/sources/constants'); var commonDefines = require('./../../Common/sources/commondefines'); var storage = require('./../../Common/sources/storage-base'); var formatChecker = require('./../../Common/sources/formatchecker'); var statsDClient = require('./../../Common/sources/statsdclient'); var config = require('config'); var config_server = config.get('services.CoAuthoring.server'); var config_utils = config.get('services.CoAuthoring.utils'); var pubsubRedis = require('./pubsubRedis'); var cfgTypesUpload = config_utils.get('limits_image_types_upload'); var cfgTypesCopy = config_utils.get('limits_image_types_copy'); var cfgImageSize = config_server.get('limits_image_size'); var cfgImageDownloadTimeout = config_server.get('limits_image_download_timeout'); var cfgRedisPrefix = config.get('services.CoAuthoring.redis.prefix'); var SAVE_TYPE_PART_START = 0; var SAVE_TYPE_PART = 1; var SAVE_TYPE_COMPLETE = 2; var SAVE_TYPE_COMPLETE_ALL = 3; var clientStatsD = statsDClient.getClient(); var redisClient = pubsubRedis.getClientRedis(); var redisKeySaved = cfgRedisPrefix + constants.REDIS_KEY_SAVED; var redisKeyShutdown = cfgRedisPrefix + constants.REDIS_KEY_SHUTDOWN; function OutputDataWrap(type, data) { this['type'] = type; this['data'] = data; } OutputDataWrap.prototype = { fromObject: function(data) { this['type'] = data['type']; this['data'] = new OutputData(); this['data'].fromObject(data['data']); }, getType: function() { return this['type']; }, setType: function(data) { this['type'] = data; }, getData: function() { return this['data']; }, setData: function(data) { this['data'] = data; } }; function OutputData(type) { this['type'] = type; this['status'] = undefined; this['data'] = undefined; } OutputData.prototype = { fromObject: function(data) { this['type'] = data['type']; this['status'] = data['status']; this['data'] = data['data']; }, getType: function() { return this['type']; }, setType: function(data) { this['type'] = data; }, getStatus: function() { return this['status']; }, setStatus: function(data) { this['status'] = data; }, getData: function() { return this['data']; }, setData: function(data) { this['data'] = data; } }; function* getOutputData(cmd, outputData, key, status, statusInfo, optConn, optAdditionalOutput) { var docId = cmd.getDocId(); switch (status) { case taskResult.FileStatus.SaveVersion: case taskResult.FileStatus.UpdateVersion: case taskResult.FileStatus.Ok: if(taskResult.FileStatus.Ok == status) { outputData.setStatus('ok'); } else if(taskResult.FileStatus.SaveVersion == status) { if (optConn && optConn.user.view) { outputData.setStatus('updateversion'); } else { var updateMask = new taskResult.TaskResultData(); updateMask.key = docId; updateMask.status = status; updateMask.statusInfo = statusInfo; var updateTask = new taskResult.TaskResultData(); updateTask.status = taskResult.FileStatus.Ok; updateTask.statusInfo = constants.NO_ERROR; var updateIfRes = yield taskResult.updateIf(updateTask, updateMask); if (updateIfRes.affectedRows > 0) { outputData.setStatus('ok'); } else { outputData.setStatus('updateversion'); } } } else { outputData.setStatus('updateversion'); } var command = cmd.getCommand(); if ('open' != command && 'reopen' != command) { var strPath = key + '/' + cmd.getOutputPath(); if (optConn) { var contentDisposition = cmd.getInline() ? constants.CONTENT_DISPOSITION_INLINE : constants.CONTENT_DISPOSITION_ATTACHMENT; outputData.setData(yield storage.getSignedUrl(optConn.baseUrl, strPath, null, cmd.getTitle(), contentDisposition)); } else if (optAdditionalOutput) { optAdditionalOutput.needUrlKey = strPath; optAdditionalOutput.needUrlMethod = 2; } } else { if (optConn) { outputData.setData(yield storage.getSignedUrls(optConn.baseUrl, key)); } else if (optAdditionalOutput) { optAdditionalOutput.needUrlKey = key; optAdditionalOutput.needUrlMethod = 0; } } break; case taskResult.FileStatus.NeedParams: outputData.setStatus('needparams'); var settingsPath = key + '/' + 'settings.json'; if (optConn) { outputData.setData(yield storage.getSignedUrl(optConn.baseUrl, settingsPath)); } else if (optAdditionalOutput) { optAdditionalOutput.needUrlKey = settingsPath; optAdditionalOutput.needUrlMethod = 1; } break; case taskResult.FileStatus.Err: case taskResult.FileStatus.ErrToReload: outputData.setStatus('err'); outputData.setData(statusInfo); if (taskResult.FileStatus.ErrToReload == status) { yield cleanupCache(key); } break; } } function* addRandomKeyTaskCmd(cmd) { var task = yield* taskResult.addRandomKeyTask(cmd.getDocId()); cmd.setSaveKey(task.key); } function* saveParts(cmd) { var result = false; var saveType = cmd.getSaveType(); var filename; if (SAVE_TYPE_COMPLETE_ALL === saveType) { filename = 'Editor.bin'; } else { filename = 'Editor' + (cmd.getSaveIndex() || '') + '.bin'; } if (SAVE_TYPE_PART_START === saveType || SAVE_TYPE_COMPLETE_ALL === saveType) { yield* addRandomKeyTaskCmd(cmd); } if (cmd.getUrl()) { result = true; } else { var buffer = cmd.getData(); yield storage.putObject(cmd.getSaveKey() + '/' + filename, buffer, buffer.length); //delete data to prevent serialize into json cmd.data = null; result = (SAVE_TYPE_COMPLETE_ALL === saveType || SAVE_TYPE_COMPLETE === saveType); } return result; } function getSaveTask(cmd) { cmd.setData(null); var queueData = new commonDefines.TaskQueueData(); queueData.setCmd(cmd); queueData.setToFile(constants.OUTPUT_NAME + '.' + formatChecker.getStringFromFormat(cmd.getOutputFormat())); //todo paid //if (cmd.vkey) { // bool // bPaid; // Signature.getVKeyParams(cmd.vkey, out bPaid); // oTaskQueueData.m_bPaid = bPaid; //} return queueData; } function getUpdateResponse(cmd) { var updateTask = new taskResult.TaskResultData(); updateTask.key = cmd.getSaveKey() ? cmd.getSaveKey() : cmd.getDocId(); var statusInfo = cmd.getStatusInfo(); if (constants.NO_ERROR == statusInfo) { updateTask.status = taskResult.FileStatus.Ok; } else if (constants.CONVERT_DOWNLOAD == statusInfo) { updateTask.status = taskResult.FileStatus.ErrToReload; } else if (constants.CONVERT_NEED_PARAMS == statusInfo) { updateTask.status = taskResult.FileStatus.NeedParams; } else { updateTask.status = taskResult.FileStatus.Err; } updateTask.statusInfo = statusInfo; if (cmd.getTitle()) { updateTask.title = cmd.getTitle(); } return updateTask; } var cleanupCache = co.wrap(function* (docId) { //todo redis ? var res = false; var removeRes = yield taskResult.remove(docId); if (removeRes.affectedRows > 0) { yield storage.deletePath(docId); res = true; } return res; }); function* commandOpen(conn, cmd, outputData) { var task = new taskResult.TaskResultData(); task.key = cmd.getDocId(); task.format = cmd.getFormat(); task.status = taskResult.FileStatus.WaitQueue; task.statusInfo = constants.NO_ERROR; task.title = cmd.getTitle(); var upsertRes = yield taskResult.upsert(task); //if CLIENT_FOUND_ROWS don't specify 1 row is inserted , 2 row is updated, and 0 row is set to its current values //http://dev.mysql.com/doc/refman/5.7/en/insert-on-duplicate.html var bCreate = upsertRes.affectedRows == 1; if (!bCreate) { var selectRes = yield taskResult.select(task); if (selectRes.length > 0) { var row = selectRes[0]; yield* getOutputData(cmd, outputData, cmd.getDocId(), row.tr_status, row.tr_status_info, conn); } } else { //add task cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CANVAS); cmd.setEmbeddedFonts(false); var dataQueue = new commonDefines.TaskQueueData(); dataQueue.setCmd(cmd); dataQueue.setToFile('Editor.bin'); var priority = constants.QUEUE_PRIORITY_HIGH; var formatIn = formatChecker.getFormatFromString(cmd.getFormat()); //decrease pdf, djvu, xps convert priority becase long open time if (constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === formatIn || constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_DJVU === formatIn || constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_XPS === formatIn) { priority = constants.QUEUE_PRIORITY_LOW; } yield* docsCoServer.addTask(dataQueue, priority); } } function* commandReopen(cmd) { var task = new taskResult.TaskResultData(); task.key = cmd.getDocId(); task.status = taskResult.FileStatus.WaitQueue; task.statusInfo = constants.NO_ERROR; var upsertRes = yield taskResult.update(task); if (upsertRes.affectedRows > 0) { //add task cmd.setUrl(null);//url may expire cmd.setSaveKey(cmd.getDocId()); cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CANVAS); cmd.setEmbeddedFonts(false); var dataQueue = new commonDefines.TaskQueueData(); dataQueue.setCmd(cmd); dataQueue.setToFile('Editor.bin'); dataQueue.setFromSettings(true); yield* docsCoServer.addTask(dataQueue, constants.QUEUE_PRIORITY_HIGH); } } function* commandSave(cmd, outputData) { var completeParts = yield* saveParts(cmd); if (completeParts) { var queueData = getSaveTask(cmd); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); } outputData.setStatus('ok'); outputData.setData(cmd.getSaveKey()); } function* commandSendMailMerge(cmd, outputData) { var completeParts = yield* saveParts(cmd); var isErr = false; if (completeParts) { isErr = true; var getRes = yield* docsCoServer.getCallback(cmd.getDocId()); if (getRes) { var mailMergeSend = cmd.getMailMergeSend(); mailMergeSend.setUrl(getRes.server.href); mailMergeSend.setBaseUrl(getRes.baseUrl); //меняем JsonKey и SaveKey, новый key нужет потому что за одну конвертацию делается часть, а json нужен всегда mailMergeSend.setJsonKey(cmd.getSaveKey()); yield* addRandomKeyTaskCmd(cmd); var queueData = getSaveTask(cmd); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); isErr = false; } } if (isErr) { outputData.setStatus('err'); outputData.setData(constants.UNKNOWN); } else { outputData.setStatus('ok'); outputData.setData(cmd.getSaveKey()); } } function* commandSfctByCmd(cmd) { yield* addRandomKeyTaskCmd(cmd); var queueData = getSaveTask(cmd); queueData.setFromChanges(true); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); } function* commandSfct(cmd, outputData) { yield* commandSfctByCmd(cmd); outputData.setStatus('ok'); } function isDisplayedImage(strName) { var res = 0; if (strName) { //шаблон display[N]image.ext var findStr = constants.DISPLAY_PREFIX; var index = strName.indexOf(findStr); if (-1 != index) { if (index + findStr.length < strName.length) { var displayN = parseInt(strName[index + findStr.length]); if (1 <= displayN && displayN <= 6) { var imageIndex = index + findStr.length + 1; if (imageIndex == strName.indexOf("image", imageIndex)) res = displayN; } } } } return res; } function* commandImgurls(conn, cmd, outputData) { var supportedFormats; var urls; var errorCode = constants.NO_ERROR; var isImgUrl = 'imgurl' == cmd.getCommand(); if (isImgUrl) { urls = [cmd.getData()]; supportedFormats = cfgTypesUpload || 'jpg'; } else { urls = cmd.getData(); supportedFormats = cfgTypesCopy || 'jpg'; } //todo Promise.all() var displayedImageMap = {};//to make one imageIndex for ole object urls var imageCount = 0; var outputUrls = []; for (var i = 0; i < urls.length; ++i) { var urlSource = urls[i]; var urlParsed; var data = undefined; if (urlSource.startsWith('data:')) { var delimiterIndex = urlSource.indexOf(','); if (-1 != delimiterIndex && (urlSource.length - (delimiterIndex + 1)) * 0.75 <= cfgImageSize) { data = new Buffer(urlSource.substring(delimiterIndex + 1), 'base64'); } } else if(urlSource) { //todo stream data = yield utils.downloadUrlPromise(urlSource, cfgImageDownloadTimeout * 1000, cfgImageSize); urlParsed = urlModule.parse(urlSource); } var outputUrl = {url: 'error', path: 'error'}; if (data) { var format = formatChecker.getImageFormat(data); var formatStr; if (constants.AVS_OFFICESTUDIO_FILE_UNKNOWN == format && urlParsed) { //bin, txt occur in ole object case var ext = pathModule.extname(urlParsed.pathname); if ('.bin' == ext || '.txt' == ext) { formatStr = ext.substring(1); } } else { formatStr = formatChecker.getStringFromFormat(format); } if (formatStr && -1 !== supportedFormats.indexOf(formatStr)) { var userid = cmd.getUserId(); var imageIndex = cmd.getSaveIndex() + imageCount; imageCount++; var strLocalPath = 'media/' + utils.crc32(userid).toString(16) + '_'; if (urlParsed) { var urlBasename = pathModule.basename(urlParsed.pathname); var displayN = isDisplayedImage(urlBasename); if (displayN > 0) { var displayedImageName = urlBasename.substring(0, urlBasename.length - formatStr.length - 1); var tempIndex = displayedImageMap[displayedImageName]; if (null != tempIndex) { imageIndex = tempIndex; imageCount--; } else { displayedImageMap[displayedImageName] = imageIndex; } strLocalPath += constants.DISPLAY_PREFIX + displayN; } } strLocalPath += 'image' + imageIndex + '.' + formatStr; var strPath = cmd.getDocId() + '/' + strLocalPath; yield storage.putObject(strPath, data, data.length); var imgUrl = yield storage.getSignedUrl(conn.baseUrl, strPath); outputUrl = {url: imgUrl, path: strLocalPath}; } } if (isImgUrl && ('error' === outputUrl.url || 'error' === outputUrl.path)) { errorCode = constants.UPLOAD_EXTENSION; break; } outputUrls.push(outputUrl); } if (constants.NO_ERROR !== errorCode) { outputData.setStatus('err'); outputData.setData(errorCode); } else { outputData.setStatus('ok'); outputData.setData(outputUrls); } } function* commandSaveFromOrigin(cmd, outputData) { yield* addRandomKeyTaskCmd(cmd); var queueData = getSaveTask(cmd); queueData.setFromOrigin(true); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); outputData.setStatus('ok'); outputData.setData(cmd.getSaveKey()); } function* commandSfcCallback(cmd, isSfcm) { var docId = cmd.getDocId(); logger.debug('Start commandSfcCallback: docId = %s', docId); var saveKey = cmd.getSaveKey(); var statusInfo = cmd.getStatusInfo(); var isError = constants.NO_ERROR != statusInfo && constants.CONVERT_CORRUPTED != statusInfo; var savePathDoc = saveKey + '/' + cmd.getOutputPath(); var savePathChanges = saveKey + '/changes.zip'; var savePathHistory = saveKey + '/changesHistory.json'; var getRes = yield* docsCoServer.getCallback(docId); if (getRes) { logger.debug('Callback commandSfcCallback: docId = %s callback = %s', docId, getRes.server.href); var outputSfc = new commonDefines.OutputSfcData(); outputSfc.setKey(docId); if (cmd.getUserId()) { outputSfc.setUsers([cmd.getUserId()]); } if (isSfcm) { outputSfc.setActions(undefined); outputSfc.setUserData(cmd.getUserData()); } else { //use UserId case UserActionId miss in gc convertion var userActionId = cmd.getUserActionId() || cmd.getUserId(); if (userActionId) { outputSfc.setActions([new commonDefines.OutputAction(commonDefines.c_oAscUserAction.Out, userActionId)]); } } if (!isError) { try { var data = yield storage.getObject(savePathHistory); outputSfc.setChangeHistory(data.toString('utf-8')); outputSfc.setUrl(yield storage.getSignedUrl(getRes.baseUrl, savePathDoc)); outputSfc.setChangeUrl(yield storage.getSignedUrl(getRes.baseUrl, savePathChanges)); } catch (e) { logger.error('Error commandSfcCallback: docId = %s\r\n%s', docId, e.stack); } if (outputSfc.getUrl() && outputSfc.getUsers().length > 0) { if (isSfcm) { outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MustSaveForce); } else { outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MustSave); } } else { isError = true; } } if (isError) { if (isSfcm) { outputSfc.setStatus(docsCoServer.c_oAscServerStatus.CorruptedForce); } else { outputSfc.setStatus(docsCoServer.c_oAscServerStatus.Corrupted); } } var uri = getRes.server.href; var postData = JSON.stringify(outputSfc); if (isSfcm) { var lastSave = cmd.getLastSave(); if (lastSave && !isError) { yield* docsCoServer.setForceSave(docId, lastSave, savePathDoc); } try { yield* docsCoServer.sendServerRequest(docId, uri, postData); } catch (err) { logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack); } } else { //if anybody in document stop save var hasEditors = yield* docsCoServer.hasEditors(docId); logger.debug('hasEditors commandSfcCallback: docId = %s hasEditors = %d', docId, hasEditors); if (!hasEditors) { var updateMask = new taskResult.TaskResultData(); updateMask.key = docId; updateMask.status = taskResult.FileStatus.SaveVersion; updateMask.statusInfo = cmd.getData(); var updateIfTask = new taskResult.TaskResultData(); updateIfTask.status = taskResult.FileStatus.UpdateVersion; updateIfTask.statusInfo = constants.NO_ERROR; var updateIfRes = yield taskResult.updateIf(updateIfTask, updateMask); if (updateIfRes.affectedRows > 0) { var replyStr = null; try { replyStr = yield* docsCoServer.sendServerRequest(docId, uri, postData); } catch (err) { replyStr = null; logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack); } var requestRes = false; var replyData = docsCoServer.parseReplyData(docId, replyStr); if (replyData && commonDefines.c_oAscServerCommandErrors.NoError == replyData.error) { //в случае comunity server придет запрос в CommandService проверяем результат var multi = redisClient.multi([ ['get', redisKeySaved + docId], ['del', redisKeySaved + docId] ]); var execRes = yield utils.promiseRedis(multi, multi.exec); var savedVal = execRes[0]; requestRes = (null == savedVal || '1' === savedVal); } if (requestRes) { yield docsCoServer.cleanDocumentOnExitPromise(docId, true, true); } else { var updateTask = new taskResult.TaskResultData(); updateTask.key = docId; updateTask.status = taskResult.FileStatus.Ok; updateTask.statusInfo = constants.NO_ERROR; yield taskResult.update(updateTask); } } } } } if (docsCoServer.getIsShutdown()) { yield utils.promiseRedis(redisClient, redisClient.decr, redisKeyShutdown); } logger.debug('End commandSfcCallback: docId = %s', docId); } function* commandSendMMCallback(cmd) { var docId = cmd.getDocId(); logger.debug('Start commandSendMMCallback: docId = %s', docId); var saveKey = cmd.getSaveKey(); var statusInfo = cmd.getStatusInfo(); var outputSfc = new commonDefines.OutputSfcData(); outputSfc.setKey(docId); if (constants.NO_ERROR == statusInfo) { outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MailMerge); } else { outputSfc.setStatus(docsCoServer.c_oAscServerStatus.Corrupted); } var mailMergeSendData = cmd.getMailMergeSend(); var outputMailMerge = new commonDefines.OutputMailMerge(mailMergeSendData); outputSfc.setMailMerge(outputMailMerge); outputSfc.setUsers([mailMergeSendData.getUserId()]); var data = yield storage.getObject(saveKey + '/' + cmd.getOutputPath()); var xml = data.toString('utf8'); var files = xml.match(/[< ]file.*?\/>/g); var recordRemain = (mailMergeSendData.getRecordTo() - mailMergeSendData.getRecordFrom() + 1); var recordIndexStart = mailMergeSendData.getRecordCount() - recordRemain; for (var i = 0; i < files.length; ++i) { var file = files[i]; var fieldRes = /field=["'](.*?)["']/.exec(file); outputMailMerge.setTo(fieldRes[1]); outputMailMerge.setRecordIndex(recordIndexStart + i); var pathRes = /path=["'](.*?)["']/.exec(file); var signedUrl = yield storage.getSignedUrl(mailMergeSendData.getBaseUrl(), saveKey + '/' + pathRes[1]); outputSfc.setUrl(signedUrl); var uri = mailMergeSendData.getUrl(); var postData = JSON.stringify(outputSfc); try { yield* docsCoServer.sendServerRequest(docId, uri, postData); } catch (err) { logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack); } } var newRecordFrom = mailMergeSendData.getRecordFrom() + Math.max(files.length, 1); if (newRecordFrom <= mailMergeSendData.getRecordTo()) { mailMergeSendData.setRecordFrom(newRecordFrom); yield* addRandomKeyTaskCmd(cmd); var queueData = getSaveTask(cmd); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); } else { logger.debug('End MailMerge: docId = %s', docId); } logger.debug('End commandSendMMCallback: docId = %s', docId); } exports.openDocument = function(conn, input) { return co(function* () { var outputData; var docId = conn ? conn.docId : 'null'; try { var startDate = null; if(clientStatsD) { startDate = new Date(); } var cmd = new commonDefines.InputCommand(input.message); logger.debug('Start command: docId = %s %s', docId, JSON.stringify(cmd)); outputData = new OutputData(cmd.getCommand()); switch (cmd.getCommand()) { case 'open': //yield utils.sleep(5000); yield* commandOpen(conn, cmd, outputData); break; case 'reopen': yield* commandReopen(cmd); break; case 'imgurl': case 'imgurls': yield* commandImgurls(conn, cmd, outputData); break; default: outputData.setStatus('err'); outputData.setData(constants.UNKNOWN); break; } if(clientStatsD) { clientStatsD.timing('coauth.openDocument.' + cmd.getCommand(), new Date() - startDate); } } catch (e) { logger.error('Error openDocument: docId = %s\r\n%s', docId, e.stack); if (!outputData) { outputData = new OutputData(); } outputData.setStatus('err'); outputData.setData(constants.UNKNOWN); } finally { if (outputData && outputData.getStatus()) { logger.debug('Response command: docId = %s %s', docId, JSON.stringify(outputData)); docsCoServer.sendData(conn, new OutputDataWrap('documentOpen', outputData)); } logger.debug('End command: docId = %s', docId); } }); }; exports.downloadAs = function(req, res) { return co(function* () { var docId = 'null'; try { var startDate = null; if(clientStatsD) { startDate = new Date(); } var strCmd = req.query['cmd']; var cmd = new commonDefines.InputCommand(JSON.parse(strCmd)); docId = cmd.getDocId(); logger.debug('Start downloadAs: docId = %s %s', docId, strCmd); cmd.setData(req.body); var outputData = new OutputData(cmd.getCommand()); switch (cmd.getCommand()) { case 'save': yield* commandSave(cmd, outputData); break; case 'savefromorigin': yield* commandSaveFromOrigin(cmd, outputData); break; case 'sendmm': yield* commandSendMailMerge(cmd, outputData); break; case 'sfct': yield* commandSfct(cmd, outputData); break; default: outputData.setStatus('err'); outputData.setData(constants.UNKNOWN); break; } var strRes = JSON.stringify(outputData); res.send(strRes); logger.debug('End downloadAs: docId = %s %s', docId, strRes); if(clientStatsD) { clientStatsD.timing('coauth.downloadAs.' + cmd.getCommand(), new Date() - startDate); } } catch (e) { logger.error('Error downloadAs: docId = %s\r\n%s', docId, e.stack); res.sendStatus(400); } }); }; exports.saveFromChanges = function(docId, statusInfo, optFormat, opt_userId, opt_queue) { return co(function* () { try { var startDate = null; if(clientStatsD) { startDate = new Date(); } logger.debug('Start saveFromChanges: docId = %s', docId); var task = new taskResult.TaskResultData(); task.key = docId; //делаем select, потому что за время timeout информация могла измениться var selectRes = yield taskResult.select(task); var row = selectRes.length > 0 ? selectRes[0] : null; if (row && row.tr_status == taskResult.FileStatus.SaveVersion && row.tr_status_info == statusInfo) { if (null == optFormat) { optFormat = constants.AVS_OFFICESTUDIO_FILE_OTHER_TEAMLAB_INNER; } var cmd = new commonDefines.InputCommand(); cmd.setCommand('sfc'); cmd.setDocId(docId); cmd.setOutputFormat(optFormat); cmd.setData(statusInfo); cmd.setUserActionId(opt_userId); yield* addRandomKeyTaskCmd(cmd); var queueData = getSaveTask(cmd); queueData.setFromChanges(true); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_NORMAL, opt_queue); if (docsCoServer.getIsShutdown()) { yield utils.promiseRedis(redisClient, redisClient.incr, redisKeyShutdown); } logger.debug('AddTask saveFromChanges: docId = %s', docId); } else { if (row) { logger.debug('saveFromChanges status mismatch: docId = %s; row: %d; %d; expected: %d', docId, row.tr_status, row.tr_status_info, statusInfo); } } if (clientStatsD) { clientStatsD.timing('coauth.saveFromChanges', new Date() - startDate); } } catch (e) { logger.error('Error saveFromChanges: docId = %s\r\n%s', docId, e.stack); } }); }; exports.receiveTask = function(data, dataRaw) { return co(function* () { var docId = 'null'; try { var task = new commonDefines.TaskQueueData(JSON.parse(data)); if (task) { var cmd = task.getCmd(); docId = cmd.getDocId(); logger.debug('Start receiveTask: docId = %s %s', docId, data); var updateTask = getUpdateResponse(cmd); var updateRes = yield taskResult.update(updateTask); if (updateRes.affectedRows > 0) { var outputData = new OutputData(cmd.getCommand()); var command = cmd.getCommand(); var additionalOutput = {needUrlKey: null, needUrlMethod: null}; if ('open' == command || 'reopen' == command) { //yield utils.sleep(5000); yield* getOutputData(cmd, outputData, cmd.getDocId(), updateTask.status, updateTask.statusInfo, null, additionalOutput); } else if ('save' == command || 'savefromorigin' == command || 'sfct' == command) { yield* getOutputData(cmd, outputData, cmd.getSaveKey(), updateTask.status, updateTask.statusInfo, null, additionalOutput); } else if ('sfcm' == command) { yield* commandSfcCallback(cmd, true); } else if ('sfc' == command) { yield* commandSfcCallback(cmd, false); } else if ('sendmm' == command) { yield* commandSendMMCallback(cmd); } else if ('conv' == command) { //nothing } if (outputData.getStatus()) { logger.debug('Send receiveTask: docId = %s %s', docId, JSON.stringify(outputData)); var output = new OutputDataWrap('documentOpen', outputData); yield* docsCoServer.publish({ type: commonDefines.c_oPublishType.receiveTask, cmd: cmd, output: output, needUrlKey: additionalOutput.needUrlKey, needUrlMethod: additionalOutput.needUrlMethod }); } } yield* docsCoServer.removeResponse(dataRaw); logger.debug('End receiveTask: docId = %s', docId); } } catch (err) { logger.debug('Error receiveTask: docId = %s\r\n%s', docId, err.stack); } }); }; exports.cleanupCache = cleanupCache; exports.commandSfctByCmd = commandSfctByCmd; exports.OutputDataWrap = OutputDataWrap; exports.OutputData = OutputData; +var pathModule = require('path'); var urlModule = require('url'); var co = require('co'); var sqlBase = require('./baseConnector'); var docsCoServer = require('./DocsCoServer'); var taskResult = require('./taskresult'); var logger = require('./../../Common/sources/logger'); var utils = require('./../../Common/sources/utils'); var constants = require('./../../Common/sources/constants'); var commonDefines = require('./../../Common/sources/commondefines'); var storage = require('./../../Common/sources/storage-base'); var formatChecker = require('./../../Common/sources/formatchecker'); var statsDClient = require('./../../Common/sources/statsdclient'); var config = require('config'); var config_server = config.get('services.CoAuthoring.server'); var config_utils = config.get('services.CoAuthoring.utils'); var pubsubRedis = require('./pubsubRedis'); var cfgTypesUpload = config_utils.get('limits_image_types_upload'); var cfgTypesCopy = config_utils.get('limits_image_types_copy'); var cfgImageSize = config_server.get('limits_image_size'); var cfgImageDownloadTimeout = config_server.get('limits_image_download_timeout'); var cfgRedisPrefix = config.get('services.CoAuthoring.redis.prefix'); var SAVE_TYPE_PART_START = 0; var SAVE_TYPE_PART = 1; var SAVE_TYPE_COMPLETE = 2; var SAVE_TYPE_COMPLETE_ALL = 3; var clientStatsD = statsDClient.getClient(); var redisClient = pubsubRedis.getClientRedis(); var redisKeySaved = cfgRedisPrefix + constants.REDIS_KEY_SAVED; var redisKeyShutdown = cfgRedisPrefix + constants.REDIS_KEY_SHUTDOWN; function OutputDataWrap(type, data) { this['type'] = type; this['data'] = data; } OutputDataWrap.prototype = { fromObject: function(data) { this['type'] = data['type']; this['data'] = new OutputData(); this['data'].fromObject(data['data']); }, getType: function() { return this['type']; }, setType: function(data) { this['type'] = data; }, getData: function() { return this['data']; }, setData: function(data) { this['data'] = data; } }; function OutputData(type) { this['type'] = type; this['status'] = undefined; this['data'] = undefined; } OutputData.prototype = { fromObject: function(data) { this['type'] = data['type']; this['status'] = data['status']; this['data'] = data['data']; }, getType: function() { return this['type']; }, setType: function(data) { this['type'] = data; }, getStatus: function() { return this['status']; }, setStatus: function(data) { this['status'] = data; }, getData: function() { return this['data']; }, setData: function(data) { this['data'] = data; } }; function* getOutputData(cmd, outputData, key, status, statusInfo, optConn, optAdditionalOutput) { var docId = cmd.getDocId(); switch (status) { case taskResult.FileStatus.SaveVersion: case taskResult.FileStatus.UpdateVersion: case taskResult.FileStatus.Ok: if(taskResult.FileStatus.Ok == status) { outputData.setStatus('ok'); } else if(taskResult.FileStatus.SaveVersion == status) { if (optConn && optConn.user.view) { outputData.setStatus('updateversion'); } else { var updateMask = new taskResult.TaskResultData(); updateMask.key = docId; updateMask.status = status; updateMask.statusInfo = statusInfo; var updateTask = new taskResult.TaskResultData(); updateTask.status = taskResult.FileStatus.Ok; updateTask.statusInfo = constants.NO_ERROR; var updateIfRes = yield taskResult.updateIf(updateTask, updateMask); if (updateIfRes.affectedRows > 0) { outputData.setStatus('ok'); } else { outputData.setStatus('updateversion'); } } } else { outputData.setStatus('updateversion'); } var command = cmd.getCommand(); if ('open' != command && 'reopen' != command) { var strPath = key + '/' + cmd.getOutputPath(); if (optConn) { var contentDisposition = cmd.getInline() ? constants.CONTENT_DISPOSITION_INLINE : constants.CONTENT_DISPOSITION_ATTACHMENT; outputData.setData(yield storage.getSignedUrl(optConn.baseUrl, strPath, null, cmd.getTitle(), contentDisposition)); } else if (optAdditionalOutput) { optAdditionalOutput.needUrlKey = strPath; optAdditionalOutput.needUrlMethod = 2; } } else { if (optConn) { outputData.setData(yield storage.getSignedUrls(optConn.baseUrl, key)); } else if (optAdditionalOutput) { optAdditionalOutput.needUrlKey = key; optAdditionalOutput.needUrlMethod = 0; } } break; case taskResult.FileStatus.NeedParams: outputData.setStatus('needparams'); var settingsPath = key + '/' + 'settings.json'; if (optConn) { outputData.setData(yield storage.getSignedUrl(optConn.baseUrl, settingsPath)); } else if (optAdditionalOutput) { optAdditionalOutput.needUrlKey = settingsPath; optAdditionalOutput.needUrlMethod = 1; } break; case taskResult.FileStatus.Err: case taskResult.FileStatus.ErrToReload: outputData.setStatus('err'); outputData.setData(statusInfo); if (taskResult.FileStatus.ErrToReload == status) { yield cleanupCache(key); } break; } } function* addRandomKeyTaskCmd(cmd) { var task = yield* taskResult.addRandomKeyTask(cmd.getDocId()); cmd.setSaveKey(task.key); } function* saveParts(cmd) { var result = false; var saveType = cmd.getSaveType(); var filename; if (SAVE_TYPE_COMPLETE_ALL === saveType) { filename = 'Editor.bin'; } else { filename = 'Editor' + (cmd.getSaveIndex() || '') + '.bin'; } if (SAVE_TYPE_PART_START === saveType || SAVE_TYPE_COMPLETE_ALL === saveType) { yield* addRandomKeyTaskCmd(cmd); } if (cmd.getUrl()) { result = true; } else { var buffer = cmd.getData(); yield storage.putObject(cmd.getSaveKey() + '/' + filename, buffer, buffer.length); //delete data to prevent serialize into json cmd.data = null; result = (SAVE_TYPE_COMPLETE_ALL === saveType || SAVE_TYPE_COMPLETE === saveType); } return result; } function getSaveTask(cmd) { cmd.setData(null); var queueData = new commonDefines.TaskQueueData(); queueData.setCmd(cmd); queueData.setToFile(constants.OUTPUT_NAME + '.' + formatChecker.getStringFromFormat(cmd.getOutputFormat())); //todo paid //if (cmd.vkey) { // bool // bPaid; // Signature.getVKeyParams(cmd.vkey, out bPaid); // oTaskQueueData.m_bPaid = bPaid; //} return queueData; } function getUpdateResponse(cmd) { var updateTask = new taskResult.TaskResultData(); updateTask.key = cmd.getSaveKey() ? cmd.getSaveKey() : cmd.getDocId(); var statusInfo = cmd.getStatusInfo(); if (constants.NO_ERROR == statusInfo) { updateTask.status = taskResult.FileStatus.Ok; } else if (constants.CONVERT_DOWNLOAD == statusInfo) { updateTask.status = taskResult.FileStatus.ErrToReload; } else if (constants.CONVERT_NEED_PARAMS == statusInfo) { updateTask.status = taskResult.FileStatus.NeedParams; } else { updateTask.status = taskResult.FileStatus.Err; } updateTask.statusInfo = statusInfo; if (cmd.getTitle()) { updateTask.title = cmd.getTitle(); } return updateTask; } var cleanupCache = co.wrap(function* (docId) { //todo redis ? var res = false; var removeRes = yield taskResult.remove(docId); if (removeRes.affectedRows > 0) { yield storage.deletePath(docId); res = true; } return res; }); function commandOpenStartPromise(cmd, opt_updateUserIndex) { var task = new taskResult.TaskResultData(); task.key = cmd.getDocId(); task.format = cmd.getFormat(); task.status = taskResult.FileStatus.WaitQueue; task.statusInfo = constants.NO_ERROR; task.title = cmd.getTitle(); return taskResult.upsert(task, opt_updateUserIndex); } function* commandOpen(conn, cmd, outputData, opt_upsertRes) { var upsertRes; if (opt_upsertRes) { upsertRes = opt_upsertRes; } else { upsertRes = yield commandOpenStartPromise(cmd); } //if CLIENT_FOUND_ROWS don't specify 1 row is inserted , 2 row is updated, and 0 row is set to its current values //http://dev.mysql.com/doc/refman/5.7/en/insert-on-duplicate.html var bCreate = upsertRes.affectedRows == 1; if (!bCreate) { var selectRes = yield taskResult.select(cmd.getDocId()); if (selectRes.length > 0) { var row = selectRes[0]; yield* getOutputData(cmd, outputData, cmd.getDocId(), row.tr_status, row.tr_status_info, conn); } } else { //add task cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CANVAS); cmd.setEmbeddedFonts(false); var dataQueue = new commonDefines.TaskQueueData(); dataQueue.setCmd(cmd); dataQueue.setToFile('Editor.bin'); var priority = constants.QUEUE_PRIORITY_HIGH; var formatIn = formatChecker.getFormatFromString(cmd.getFormat()); //decrease pdf, djvu, xps convert priority becase long open time if (constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === formatIn || constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_DJVU === formatIn || constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_XPS === formatIn) { priority = constants.QUEUE_PRIORITY_LOW; } yield* docsCoServer.addTask(dataQueue, priority); } } function* commandReopen(cmd) { var task = new taskResult.TaskResultData(); task.key = cmd.getDocId(); task.status = taskResult.FileStatus.WaitQueue; task.statusInfo = constants.NO_ERROR; var upsertRes = yield taskResult.update(task); if (upsertRes.affectedRows > 0) { //add task cmd.setUrl(null);//url may expire cmd.setSaveKey(cmd.getDocId()); cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CANVAS); cmd.setEmbeddedFonts(false); var dataQueue = new commonDefines.TaskQueueData(); dataQueue.setCmd(cmd); dataQueue.setToFile('Editor.bin'); dataQueue.setFromSettings(true); yield* docsCoServer.addTask(dataQueue, constants.QUEUE_PRIORITY_HIGH); } } function* commandSave(cmd, outputData) { var completeParts = yield* saveParts(cmd); if (completeParts) { var queueData = getSaveTask(cmd); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); } outputData.setStatus('ok'); outputData.setData(cmd.getSaveKey()); } function* commandSendMailMerge(cmd, outputData) { var completeParts = yield* saveParts(cmd); var isErr = false; if (completeParts) { isErr = true; var getRes = yield* docsCoServer.getCallback(cmd.getDocId()); if (getRes) { var mailMergeSend = cmd.getMailMergeSend(); mailMergeSend.setUrl(getRes.server.href); mailMergeSend.setBaseUrl(getRes.baseUrl); //меняем JsonKey и SaveKey, новый key нужет потому что за одну конвертацию делается часть, а json нужен всегда mailMergeSend.setJsonKey(cmd.getSaveKey()); yield* addRandomKeyTaskCmd(cmd); var queueData = getSaveTask(cmd); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); isErr = false; } } if (isErr) { outputData.setStatus('err'); outputData.setData(constants.UNKNOWN); } else { outputData.setStatus('ok'); outputData.setData(cmd.getSaveKey()); } } function* commandSfctByCmd(cmd) { yield* addRandomKeyTaskCmd(cmd); var queueData = getSaveTask(cmd); queueData.setFromChanges(true); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); } function* commandSfct(cmd, outputData) { yield* commandSfctByCmd(cmd); outputData.setStatus('ok'); } function isDisplayedImage(strName) { var res = 0; if (strName) { //шаблон display[N]image.ext var findStr = constants.DISPLAY_PREFIX; var index = strName.indexOf(findStr); if (-1 != index) { if (index + findStr.length < strName.length) { var displayN = parseInt(strName[index + findStr.length]); if (1 <= displayN && displayN <= 6) { var imageIndex = index + findStr.length + 1; if (imageIndex == strName.indexOf("image", imageIndex)) res = displayN; } } } } return res; } function* commandImgurls(conn, cmd, outputData) { var supportedFormats; var urls; var errorCode = constants.NO_ERROR; var isImgUrl = 'imgurl' == cmd.getCommand(); if (isImgUrl) { urls = [cmd.getData()]; supportedFormats = cfgTypesUpload || 'jpg'; } else { urls = cmd.getData(); supportedFormats = cfgTypesCopy || 'jpg'; } //todo Promise.all() var displayedImageMap = {};//to make one imageIndex for ole object urls var imageCount = 0; var outputUrls = []; for (var i = 0; i < urls.length; ++i) { var urlSource = urls[i]; var urlParsed; var data = undefined; if (urlSource.startsWith('data:')) { var delimiterIndex = urlSource.indexOf(','); if (-1 != delimiterIndex && (urlSource.length - (delimiterIndex + 1)) * 0.75 <= cfgImageSize) { data = new Buffer(urlSource.substring(delimiterIndex + 1), 'base64'); } } else if(urlSource) { //todo stream data = yield utils.downloadUrlPromise(urlSource, cfgImageDownloadTimeout * 1000, cfgImageSize); urlParsed = urlModule.parse(urlSource); } var outputUrl = {url: 'error', path: 'error'}; if (data) { var format = formatChecker.getImageFormat(data); var formatStr; if (constants.AVS_OFFICESTUDIO_FILE_UNKNOWN == format && urlParsed) { //bin, txt occur in ole object case var ext = pathModule.extname(urlParsed.pathname); if ('.bin' == ext || '.txt' == ext) { formatStr = ext.substring(1); } } else { formatStr = formatChecker.getStringFromFormat(format); } if (formatStr && -1 !== supportedFormats.indexOf(formatStr)) { var userid = cmd.getUserId(); var imageIndex = cmd.getSaveIndex() + imageCount; imageCount++; var strLocalPath = 'media/' + utils.crc32(userid).toString(16) + '_'; if (urlParsed) { var urlBasename = pathModule.basename(urlParsed.pathname); var displayN = isDisplayedImage(urlBasename); if (displayN > 0) { var displayedImageName = urlBasename.substring(0, urlBasename.length - formatStr.length - 1); var tempIndex = displayedImageMap[displayedImageName]; if (null != tempIndex) { imageIndex = tempIndex; imageCount--; } else { displayedImageMap[displayedImageName] = imageIndex; } strLocalPath += constants.DISPLAY_PREFIX + displayN; } } strLocalPath += 'image' + imageIndex + '.' + formatStr; var strPath = cmd.getDocId() + '/' + strLocalPath; yield storage.putObject(strPath, data, data.length); var imgUrl = yield storage.getSignedUrl(conn.baseUrl, strPath); outputUrl = {url: imgUrl, path: strLocalPath}; } } if (isImgUrl && ('error' === outputUrl.url || 'error' === outputUrl.path)) { errorCode = constants.UPLOAD_EXTENSION; break; } outputUrls.push(outputUrl); } if (constants.NO_ERROR !== errorCode) { outputData.setStatus('err'); outputData.setData(errorCode); } else { outputData.setStatus('ok'); outputData.setData(outputUrls); } } function* commandSaveFromOrigin(cmd, outputData) { yield* addRandomKeyTaskCmd(cmd); var queueData = getSaveTask(cmd); queueData.setFromOrigin(true); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); outputData.setStatus('ok'); outputData.setData(cmd.getSaveKey()); } function* commandSfcCallback(cmd, isSfcm) { var docId = cmd.getDocId(); logger.debug('Start commandSfcCallback: docId = %s', docId); var saveKey = cmd.getSaveKey(); var statusInfo = cmd.getStatusInfo(); var isError = constants.NO_ERROR != statusInfo && constants.CONVERT_CORRUPTED != statusInfo; var savePathDoc = saveKey + '/' + cmd.getOutputPath(); var savePathChanges = saveKey + '/changes.zip'; var savePathHistory = saveKey + '/changesHistory.json'; var getRes = yield* docsCoServer.getCallback(docId); if (getRes) { logger.debug('Callback commandSfcCallback: docId = %s callback = %s', docId, getRes.server.href); var outputSfc = new commonDefines.OutputSfcData(); outputSfc.setKey(docId); if (cmd.getUserId()) { outputSfc.setUsers([cmd.getUserId()]); } if (isSfcm) { outputSfc.setActions(undefined); outputSfc.setUserData(cmd.getUserData()); } else { //use UserId case UserActionId miss in gc convertion var userActionId = cmd.getUserActionId() || cmd.getUserId(); if (userActionId) { outputSfc.setActions([new commonDefines.OutputAction(commonDefines.c_oAscUserAction.Out, userActionId)]); } } if (!isError) { try { var data = yield storage.getObject(savePathHistory); outputSfc.setChangeHistory(data.toString('utf-8')); outputSfc.setUrl(yield storage.getSignedUrl(getRes.baseUrl, savePathDoc)); outputSfc.setChangeUrl(yield storage.getSignedUrl(getRes.baseUrl, savePathChanges)); } catch (e) { logger.error('Error commandSfcCallback: docId = %s\r\n%s', docId, e.stack); } if (outputSfc.getUrl() && outputSfc.getUsers().length > 0) { if (isSfcm) { outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MustSaveForce); } else { outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MustSave); } } else { isError = true; } } if (isError) { if (isSfcm) { outputSfc.setStatus(docsCoServer.c_oAscServerStatus.CorruptedForce); } else { outputSfc.setStatus(docsCoServer.c_oAscServerStatus.Corrupted); } } var uri = getRes.server.href; var postData = JSON.stringify(outputSfc); if (isSfcm) { var lastSave = cmd.getLastSave(); if (lastSave && !isError) { yield* docsCoServer.setForceSave(docId, lastSave, savePathDoc); } try { yield* docsCoServer.sendServerRequest(docId, uri, postData); } catch (err) { logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack); } } else { //if anybody in document stop save var hasEditors = yield* docsCoServer.hasEditors(docId); logger.debug('hasEditors commandSfcCallback: docId = %s hasEditors = %d', docId, hasEditors); if (!hasEditors) { var updateMask = new taskResult.TaskResultData(); updateMask.key = docId; updateMask.status = taskResult.FileStatus.SaveVersion; updateMask.statusInfo = cmd.getData(); var updateIfTask = new taskResult.TaskResultData(); updateIfTask.status = taskResult.FileStatus.UpdateVersion; updateIfTask.statusInfo = constants.NO_ERROR; var updateIfRes = yield taskResult.updateIf(updateIfTask, updateMask); if (updateIfRes.affectedRows > 0) { var replyStr = null; try { replyStr = yield* docsCoServer.sendServerRequest(docId, uri, postData); } catch (err) { replyStr = null; logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack); } var requestRes = false; var replyData = docsCoServer.parseReplyData(docId, replyStr); if (replyData && commonDefines.c_oAscServerCommandErrors.NoError == replyData.error) { //в случае comunity server придет запрос в CommandService проверяем результат var multi = redisClient.multi([ ['get', redisKeySaved + docId], ['del', redisKeySaved + docId] ]); var execRes = yield utils.promiseRedis(multi, multi.exec); var savedVal = execRes[0]; requestRes = (null == savedVal || '1' === savedVal); } if (requestRes) { yield docsCoServer.cleanDocumentOnExitPromise(docId, true); } else { var updateTask = new taskResult.TaskResultData(); updateTask.key = docId; updateTask.status = taskResult.FileStatus.Ok; updateTask.statusInfo = constants.NO_ERROR; yield taskResult.update(updateTask); } } } } } if (docsCoServer.getIsShutdown()) { yield utils.promiseRedis(redisClient, redisClient.decr, redisKeyShutdown); } logger.debug('End commandSfcCallback: docId = %s', docId); } function* commandSendMMCallback(cmd) { var docId = cmd.getDocId(); logger.debug('Start commandSendMMCallback: docId = %s', docId); var saveKey = cmd.getSaveKey(); var statusInfo = cmd.getStatusInfo(); var outputSfc = new commonDefines.OutputSfcData(); outputSfc.setKey(docId); if (constants.NO_ERROR == statusInfo) { outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MailMerge); } else { outputSfc.setStatus(docsCoServer.c_oAscServerStatus.Corrupted); } var mailMergeSendData = cmd.getMailMergeSend(); var outputMailMerge = new commonDefines.OutputMailMerge(mailMergeSendData); outputSfc.setMailMerge(outputMailMerge); outputSfc.setUsers([mailMergeSendData.getUserId()]); var data = yield storage.getObject(saveKey + '/' + cmd.getOutputPath()); var xml = data.toString('utf8'); var files = xml.match(/[< ]file.*?\/>/g); var recordRemain = (mailMergeSendData.getRecordTo() - mailMergeSendData.getRecordFrom() + 1); var recordIndexStart = mailMergeSendData.getRecordCount() - recordRemain; for (var i = 0; i < files.length; ++i) { var file = files[i]; var fieldRes = /field=["'](.*?)["']/.exec(file); outputMailMerge.setTo(fieldRes[1]); outputMailMerge.setRecordIndex(recordIndexStart + i); var pathRes = /path=["'](.*?)["']/.exec(file); var signedUrl = yield storage.getSignedUrl(mailMergeSendData.getBaseUrl(), saveKey + '/' + pathRes[1]); outputSfc.setUrl(signedUrl); var uri = mailMergeSendData.getUrl(); var postData = JSON.stringify(outputSfc); try { yield* docsCoServer.sendServerRequest(docId, uri, postData); } catch (err) { logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack); } } var newRecordFrom = mailMergeSendData.getRecordFrom() + Math.max(files.length, 1); if (newRecordFrom <= mailMergeSendData.getRecordTo()) { mailMergeSendData.setRecordFrom(newRecordFrom); yield* addRandomKeyTaskCmd(cmd); var queueData = getSaveTask(cmd); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); } else { logger.debug('End MailMerge: docId = %s', docId); } logger.debug('End commandSendMMCallback: docId = %s', docId); } exports.openDocument = function(conn, cmd, opt_upsertRes) { return co(function* () { var outputData; var docId = conn ? conn.docId : 'null'; try { var startDate = null; if(clientStatsD) { startDate = new Date(); } logger.debug('Start command: docId = %s %s', docId, JSON.stringify(cmd)); outputData = new OutputData(cmd.getCommand()); switch (cmd.getCommand()) { case 'open': //yield utils.sleep(5000); yield* commandOpen(conn, cmd, outputData, opt_upsertRes); break; case 'reopen': yield* commandReopen(cmd); break; case 'imgurl': case 'imgurls': yield* commandImgurls(conn, cmd, outputData); break; default: outputData.setStatus('err'); outputData.setData(constants.UNKNOWN); break; } if(clientStatsD) { clientStatsD.timing('coauth.openDocument.' + cmd.getCommand(), new Date() - startDate); } } catch (e) { logger.error('Error openDocument: docId = %s\r\n%s', docId, e.stack); if (!outputData) { outputData = new OutputData(); } outputData.setStatus('err'); outputData.setData(constants.UNKNOWN); } finally { if (outputData && outputData.getStatus()) { logger.debug('Response command: docId = %s %s', docId, JSON.stringify(outputData)); docsCoServer.sendData(conn, new OutputDataWrap('documentOpen', outputData)); } logger.debug('End command: docId = %s', docId); } }); }; exports.downloadAs = function(req, res) { return co(function* () { var docId = 'null'; try { var startDate = null; if(clientStatsD) { startDate = new Date(); } var strCmd = req.query['cmd']; var cmd = new commonDefines.InputCommand(JSON.parse(strCmd)); docId = cmd.getDocId(); logger.debug('Start downloadAs: docId = %s %s', docId, strCmd); cmd.setData(req.body); var outputData = new OutputData(cmd.getCommand()); switch (cmd.getCommand()) { case 'save': yield* commandSave(cmd, outputData); break; case 'savefromorigin': yield* commandSaveFromOrigin(cmd, outputData); break; case 'sendmm': yield* commandSendMailMerge(cmd, outputData); break; case 'sfct': yield* commandSfct(cmd, outputData); break; default: outputData.setStatus('err'); outputData.setData(constants.UNKNOWN); break; } var strRes = JSON.stringify(outputData); res.send(strRes); logger.debug('End downloadAs: docId = %s %s', docId, strRes); if(clientStatsD) { clientStatsD.timing('coauth.downloadAs.' + cmd.getCommand(), new Date() - startDate); } } catch (e) { logger.error('Error downloadAs: docId = %s\r\n%s', docId, e.stack); res.sendStatus(400); } }); }; exports.saveFromChanges = function(docId, statusInfo, optFormat, opt_userId, opt_queue) { return co(function* () { try { var startDate = null; if(clientStatsD) { startDate = new Date(); } logger.debug('Start saveFromChanges: docId = %s', docId); var task = new taskResult.TaskResultData(); task.key = docId; //делаем select, потому что за время timeout информация могла измениться var selectRes = yield taskResult.select(docId); var row = selectRes.length > 0 ? selectRes[0] : null; if (row && row.tr_status == taskResult.FileStatus.SaveVersion && row.tr_status_info == statusInfo) { if (null == optFormat) { optFormat = constants.AVS_OFFICESTUDIO_FILE_OTHER_TEAMLAB_INNER; } var cmd = new commonDefines.InputCommand(); cmd.setCommand('sfc'); cmd.setDocId(docId); cmd.setOutputFormat(optFormat); cmd.setData(statusInfo); cmd.setUserActionId(opt_userId); yield* addRandomKeyTaskCmd(cmd); var queueData = getSaveTask(cmd); queueData.setFromChanges(true); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_NORMAL, opt_queue); if (docsCoServer.getIsShutdown()) { yield utils.promiseRedis(redisClient, redisClient.incr, redisKeyShutdown); } logger.debug('AddTask saveFromChanges: docId = %s', docId); } else { if (row) { logger.debug('saveFromChanges status mismatch: docId = %s; row: %d; %d; expected: %d', docId, row.tr_status, row.tr_status_info, statusInfo); } } if (clientStatsD) { clientStatsD.timing('coauth.saveFromChanges', new Date() - startDate); } } catch (e) { logger.error('Error saveFromChanges: docId = %s\r\n%s', docId, e.stack); } }); }; exports.receiveTask = function(data, dataRaw) { return co(function* () { var docId = 'null'; try { var task = new commonDefines.TaskQueueData(JSON.parse(data)); if (task) { var cmd = task.getCmd(); docId = cmd.getDocId(); logger.debug('Start receiveTask: docId = %s %s', docId, data); var updateTask = getUpdateResponse(cmd); var updateRes = yield taskResult.update(updateTask); if (updateRes.affectedRows > 0) { var outputData = new OutputData(cmd.getCommand()); var command = cmd.getCommand(); var additionalOutput = {needUrlKey: null, needUrlMethod: null}; if ('open' == command || 'reopen' == command) { //yield utils.sleep(5000); yield* getOutputData(cmd, outputData, cmd.getDocId(), updateTask.status, updateTask.statusInfo, null, additionalOutput); } else if ('save' == command || 'savefromorigin' == command || 'sfct' == command) { yield* getOutputData(cmd, outputData, cmd.getSaveKey(), updateTask.status, updateTask.statusInfo, null, additionalOutput); } else if ('sfcm' == command) { yield* commandSfcCallback(cmd, true); } else if ('sfc' == command) { yield* commandSfcCallback(cmd, false); } else if ('sendmm' == command) { yield* commandSendMMCallback(cmd); } else if ('conv' == command) { //nothing } if (outputData.getStatus()) { logger.debug('Send receiveTask: docId = %s %s', docId, JSON.stringify(outputData)); var output = new OutputDataWrap('documentOpen', outputData); yield* docsCoServer.publish({ type: commonDefines.c_oPublishType.receiveTask, cmd: cmd, output: output, needUrlKey: additionalOutput.needUrlKey, needUrlMethod: additionalOutput.needUrlMethod }); } } yield* docsCoServer.removeResponse(dataRaw); logger.debug('End receiveTask: docId = %s', docId); } } catch (err) { logger.debug('Error receiveTask: docId = %s\r\n%s', docId, err.stack); } }); }; exports.cleanupCache = cleanupCache; exports.commandSfctByCmd = commandSfctByCmd; exports.commandOpenStartPromise = commandOpenStartPromise; exports.OutputDataWrap = OutputDataWrap; exports.OutputData = OutputData; diff --git a/DocService/sources/converterservice.js b/DocService/sources/converterservice.js index 32e0713d..576e653f 100644 --- a/DocService/sources/converterservice.js +++ b/DocService/sources/converterservice.js @@ -72,7 +72,7 @@ function* convertByCmd(cmd, async, baseUrl, opt_healthcheck) { var selectRes; var status; if (!bCreate && !opt_healthcheck) { - selectRes = yield taskResult.select(task); + selectRes = yield taskResult.select(docId); status = yield* getConvertStatus(cmd, selectRes, baseUrl); } else { var queueData = new commonDefines.TaskQueueData(); @@ -92,7 +92,7 @@ function* convertByCmd(cmd, async, baseUrl, opt_healthcheck) { break; } yield utils.sleep(CONVERT_ASYNC_DELAY); - selectRes = yield taskResult.select(task); + selectRes = yield taskResult.select(docId); status = yield* getConvertStatus(cmd, selectRes, baseUrl); waitTime += CONVERT_ASYNC_DELAY; if (waitTime > CONVERT_TIMEOUT) { diff --git a/DocService/sources/taskresult.js b/DocService/sources/taskresult.js index 3b4f3d1b..9e2d40ac 100644 --- a/DocService/sources/taskresult.js +++ b/DocService/sources/taskresult.js @@ -24,6 +24,8 @@ function TaskResultData() { this.statusInfo = null; this.lastOpenDate = null; this.title = null; + this.userIndex = null; + this.changeId = null; } TaskResultData.prototype.completeDefaults = function() { if (!this.key) { @@ -44,23 +46,36 @@ TaskResultData.prototype.completeDefaults = function() { if (!this.title) { this.title = ''; } + if (!this.userIndex) { + this.userIndex = 1; + } + if (!this.changeId) { + this.changeId = 0; + } }; -function getUpsertString(task) { +function getUpsertString(task, opt_updateUserIndex) { task.completeDefaults(); var dateNow = sqlBase.getDateTime(new Date()); - var commandArg = [task.key, task.format, task.status, task.statusInfo, dateNow, task.title]; + var commandArg = [task.key, task.format, task.status, task.statusInfo, dateNow, task.title, task.userIndex, task.changeId]; var commandArgEsc = commandArg.map(function(curVal) { return sqlBase.baseConnector.sqlEscape(curVal) }); - return 'INSERT INTO task_result ( tr_key, tr_format, tr_status, tr_status_info, tr_last_open_date, tr_title )' + - ' VALUES (' + commandArgEsc.join(', ') + ')' + - 'ON DUPLICATE KEY UPDATE tr_last_open_date = ' + sqlBase.baseConnector.sqlEscape(dateNow) + ';'; + var sql = 'INSERT INTO task_result ( tr_key, tr_format, tr_status, tr_status_info, tr_last_open_date, tr_title,' + + ' tr_user_index, tr_change_id ) VALUES (' + commandArgEsc.join(', ') + ') ON DUPLICATE KEY UPDATE' + + ' tr_last_open_date = ' + sqlBase.baseConnector.sqlEscape(dateNow); + if (opt_updateUserIndex) { + //todo LAST_INSERT_ID in posgresql - RETURNING + sql += ', tr_user_index = LAST_INSERT_ID(tr_user_index + 1);'; + } else { + sql += ';'; + } + return sql; } -function upsert(task) { +function upsert(task, opt_updateUserIndex) { return new Promise(function(resolve, reject) { - var sqlCommand = getUpsertString(task); + var sqlCommand = getUpsertString(task, opt_updateUserIndex); sqlBase.baseConnector.sqlQuery(sqlCommand, function(error, result) { if (error) { reject(error); @@ -71,13 +86,13 @@ function upsert(task) { }); } -function getSelectString(task) { - return 'SELECT * FROM task_result WHERE tr_key=' + sqlBase.baseConnector.sqlEscape(task.key) + ';'; +function getSelectString(docId) { + return 'SELECT * FROM task_result WHERE tr_key=' + sqlBase.baseConnector.sqlEscape(docId) + ';'; } -function select(task) { +function select(docId) { return new Promise(function(resolve, reject) { - var sqlCommand = getSelectString(task); + var sqlCommand = getSelectString(docId); sqlBase.baseConnector.sqlQuery(sqlCommand, function(error, result) { if (error) { reject(error); @@ -104,6 +119,12 @@ function toUpdateArray(task, updateTime) { if (null != task.title) { res.push('tr_title=' + sqlBase.baseConnector.sqlEscape(task.title)); } + if (null != task.indexUser) { + res.push('tr_index_user=' + sqlBase.baseConnector.sqlEscape(task.indexUser)); + } + if (null != task.changeId) { + res.push('tr_change_id=' + sqlBase.baseConnector.sqlEscape(task.changeId)); + } return res; } function getUpdateString(task) { @@ -148,12 +169,12 @@ function updateIf(task, mask) { function getInsertString(task) { var dateNow = sqlBase.getDateTime(new Date()); task.completeDefaults(); - var commandArg = [task.key, task.format, task.status, task.statusInfo, dateNow, task.title]; + var commandArg = [task.key, task.format, task.status, task.statusInfo, dateNow, task.title, task.userIndex, task.changeId]; var commandArgEsc = commandArg.map(function(curVal) { return sqlBase.baseConnector.sqlEscape(curVal) }); - return 'INSERT INTO task_result ( tr_key, tr_format, tr_status, tr_status_info, tr_last_open_date, tr_title )' + - ' VALUES (' + commandArgEsc.join(', ') + ');'; + return 'INSERT INTO task_result ( tr_key, tr_format, tr_status, tr_status_info, tr_last_open_date, tr_title,'+ + ' tr_user_index, tr_change_id) VALUES (' + commandArgEsc.join(', ') + ');'; } function addRandomKey(task) { return new Promise(function(resolve, reject) { diff --git a/schema/createdb.sql b/schema/createdb.sql index f949e2d9..e27b13ee 100644 --- a/schema/createdb.sql +++ b/schema/createdb.sql @@ -78,6 +78,8 @@ CREATE TABLE IF NOT EXISTS `task_result` ( `tr_status_info` int(10) NOT NULL, `tr_last_open_date` datetime NOT NULL, `tr_title` varchar(255) NOT NULL, + `tr_user_index` int(10) unsigned NOT NULL DEFAULT 1, + `tr_change_id` int(10) unsigned NOT NULL DEFAULT 0, PRIMARY KEY (`tr_key`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8;