From 412ae51ebb1e982495099bb152a71d5bff93ea1b Mon Sep 17 00:00:00 2001 From: konovalovsergey Date: Thu, 7 Sep 2017 18:46:25 +0300 Subject: [PATCH] store files that are assembled, but not sent --- Common/config/default.json | 4 +++- Common/sources/commondefines.js | 8 +++++++ DocService/sources/DocsCoServer.js | 30 ++++++++++++++++++++------ DocService/sources/canvasservice.js | 33 ++++++++++++++++++++++++++--- FileConverter/sources/converter.js | 9 ++++++++ 5 files changed, 74 insertions(+), 10 deletions(-) diff --git a/Common/config/default.json b/Common/config/default.json index 274bf052..ee43d744 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -60,7 +60,9 @@ "callbackRequestTimeout": 120, "healthcheckfilepath": "../public/healthcheck.docx", "savetimeoutdelay": 5000, - "edit_singleton": false + "edit_singleton": false, + "forgottenfiles": "forgotten", + "forgottenfilesname": "output" }, "autoAssembly": { "enable": false, diff --git a/Common/sources/commondefines.js b/Common/sources/commondefines.js index 5a94e222..7d6d7ac3 100644 --- a/Common/sources/commondefines.js +++ b/Common/sources/commondefines.js @@ -83,6 +83,7 @@ function InputCommand(data) { this['rediskey'] = data['rediskey']; this['nobase64'] = data['nobase64']; this['savexfile'] = data['savexfile']; + this['forgotten'] = data['forgotten']; } else { this['c'] = undefined;//string command this['id'] = undefined;//string document id @@ -123,6 +124,7 @@ function InputCommand(data) { this['rediskey'] = undefined; this['nobase64'] = true; this['savexfile'] = undefined; + this['forgotten'] = undefined; } } InputCommand.prototype = { @@ -249,6 +251,12 @@ InputCommand.prototype = { setSaveKey: function(data) { this['savekey'] = data; }, + getForgotten: function() { + return this['forgotten']; + }, + setForgotten: function(data) { + this['forgotten'] = data; + }, getUserConnectionId: function() { return this['userconnectionid']; }, diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 46c516b2..091667e2 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -136,6 +136,7 @@ const cfgForceSaveEnable = config.get('autoAssembly.enable'); const cfgForceSaveInterval = ms(config.get('autoAssembly.interval')); const cfgForceSaveStep = ms(config.get('autoAssembly.step')); const cfgQueueRetentionPeriod = configCommon.get('queue.retentionPeriod'); +const cfgForgottenFiles = config.get('server.forgottenfiles'); const redisKeySaveLock = cfgRedisPrefix + constants.REDIS_KEY_SAVE_LOCK; const redisKeyPresenceHash = cfgRedisPrefix + constants.REDIS_KEY_PRESENCE_HASH; @@ -855,6 +856,7 @@ function* sendStatusDocument(docId, bChangeBase, userAction, callback, baseUrl, logger.error('postData error: docId = %s;url = %s;data = %j\r\n%s', docId, uri, sendData, err.stack); } yield* onReplySendStatusDocument(docId, replyData); + return callback; } function parseReplyData(docId, replyData) { var res = null; @@ -935,6 +937,8 @@ function* cleanDocumentOnExit(docId, deleteChanges) { //remove changes if (deleteChanges) { sqlBase.deleteChanges(docId, null); + //delete forgotten after successful send on callbackUrl + yield storage.deletePath(cfgForgottenFiles + '/' + docId); } } function* cleanDocumentOnExitNoChanges(docId, opt_userId) { @@ -1259,8 +1263,15 @@ exports.install = function(server, callbackFunction) { // На всякий случай снимаем lock yield utils.promiseRedis(redisClient, redisClient.del, redisKeySaveLock + docId); - // Send changes to save server - if (bHasChanges) { + let needSaveChanges = bHasChanges; + if (!needSaveChanges) { + //start save changes if forgotten file exists. + //more effective to send file without sfc, but this method is simpler by code + let forgotten = yield storage.listObjects(cfgForgottenFiles + '/' + docId); + needSaveChanges = forgotten.length > 0; + } + if (needSaveChanges) { + // Send changes to save server yield* _createSaveTimer(docId, tmpUser.idOriginal); } else { yield* cleanDocumentOnExitNoChanges(docId, tmpUser.idOriginal); @@ -1884,6 +1895,7 @@ exports.install = function(server, callbackFunction) { var res = true; var docId = conn.docId; var tmpUser = conn.user; + let hasForgotten; if (constants.CONN_CLOSED === conn.readyState) { //closing could happen during async action return false; @@ -1909,7 +1921,12 @@ exports.install = function(server, callbackFunction) { if (documentCallbackUrl) { bindEventsRes = yield* bindEvents(docId, documentCallbackUrl, conn.baseUrl, userAction); } else { - yield* sendStatusDocument(docId, c_oAscChangeBase.No, userAction); + let callback = yield* sendStatusDocument(docId, c_oAscChangeBase.No, userAction); + if (!callback && !bIsRestore) { + //check forgotten file + let forgotten = yield storage.listObjects(cfgForgottenFiles + '/' + docId); + hasForgotten = forgotten.length > 0; + } } } @@ -1944,10 +1961,10 @@ exports.install = function(server, callbackFunction) { sendData(conn, sendObject);//Or 0 if fails } else { if (bIsRestore) { - yield* sendAuthInfo(undefined, undefined, conn, participantsMap); + yield* sendAuthInfo(undefined, undefined, conn, participantsMap, hasForgotten); } else { var objChangesDocument = yield* getDocumentChanges(docId); - yield* sendAuthInfo(objChangesDocument.arrChanges, objChangesDocument.getLength(), conn, participantsMap); + yield* sendAuthInfo(objChangesDocument.arrChanges, objChangesDocument.getLength(), conn, participantsMap, hasForgotten); } } yield* publish({type: commonDefines.c_oPublishType.participantsState, docId: docId, user: tmpUser, state: true}, docId, tmpUser.id); @@ -1958,7 +1975,7 @@ exports.install = function(server, callbackFunction) { return res; } - function* sendAuthInfo(objChangesDocument, changesIndex, conn, participantsMap) { + function* sendAuthInfo(objChangesDocument, changesIndex, conn, participantsMap, opt_hasForgotten) { const docId = conn.docId; let docLock; if(EditorTypes.document == conn.editorType){ @@ -1990,6 +2007,7 @@ exports.install = function(server, callbackFunction) { changes: objChangesDocument, changesIndex: changesIndex, indexUser: conn.user.indexUser, + hasForgotten: opt_hasForgotten, jwt: cfgTokenEnableBrowser ? {token: fillJwtByConnection(conn), expires: cfgTokenSessionExpires} : undefined, g_cAscSpellCheckUrl: cfgSpellcheckerUrl, buildVersion: commonDefines.buildVersion, diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index 492d9c06..9b510fa7 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -57,6 +57,8 @@ var cfgImageSize = config_server.get('limits_image_size'); var cfgImageDownloadTimeout = config_server.get('limits_image_download_timeout'); var cfgRedisPrefix = config.get('services.CoAuthoring.redis.prefix'); var cfgTokenEnableBrowser = config.get('services.CoAuthoring.token.enable.browser'); +const cfgForgottenFiles = config_server.get('forgottenfiles'); +const cfgForgottenFilesName = config_server.get('forgottenfilesname'); var SAVE_TYPE_PART_START = 0; var SAVE_TYPE_PART = 1; @@ -296,6 +298,13 @@ function* commandOpen(conn, cmd, outputData, opt_upsertRes) { yield* getOutputData(cmd, outputData, cmd.getDocId(), row.status, row.status_info, conn); } } else { + let forgottenId = cfgForgottenFiles + '/' + cmd.getDocId(); + let forgotten = yield storage.listObjects(forgottenId); + //replace url with forgotten file because it absorbed all lost changes + if (forgotten.length > 0) { + cmd.setUrl(undefined); + cmd.setForgotten(forgottenId); + } //add task cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CANVAS); cmd.setEmbeddedFonts(false); @@ -547,6 +556,7 @@ function* commandSfcCallback(cmd, isSfcm) { var forceSave = cmd.getForceSave(); var forceSaveType = forceSave ? forceSave.getType() : commonDefines.c_oAscForceSaveTypes.Command; var isSfcmSuccess = false; + let storeForgotten = false; var statusOk; var statusErr; if (isSfcm) { @@ -577,10 +587,15 @@ function* commandSfcCallback(cmd, isSfcm) { outputSfc.setUserData(cmd.getUserData()); if (!isError || isErrorCorrupted) { try { - var data = yield storage.getObject(savePathHistory); - outputSfc.setChangeHistory(JSON.parse(data.toString('utf-8'))); + let forgottenId = cfgForgottenFiles + '/' + docId; + let forgotten = yield storage.listObjects(forgottenId); + if (0 === forgotten.length) { + //don't send history info because changes isn't from file in storage + var data = yield storage.getObject(savePathHistory); + outputSfc.setChangeHistory(JSON.parse(data.toString('utf-8'))); + outputSfc.setChangeUrl(yield storage.getSignedUrl(getRes.baseUrl, savePathChanges)); + } outputSfc.setUrl(yield storage.getSignedUrl(getRes.baseUrl, savePathDoc)); - outputSfc.setChangeUrl(yield storage.getSignedUrl(getRes.baseUrl, savePathChanges)); } catch (e) { logger.error('Error commandSfcCallback: docId = %s\r\n%s', docId, e.stack); } @@ -656,12 +671,24 @@ function* commandSfcCallback(cmd, isSfcm) { updateTask.status = taskResult.FileStatus.Ok; updateTask.statusInfo = constants.NO_ERROR; yield taskResult.update(updateTask); + storeForgotten = true; } } } } } else { logger.error('Empty Callback commandSfcCallback: docId = %s', docId); + storeForgotten = true; + } + if (storeForgotten && (!isError || isErrorCorrupted)) { + try { + //todo implement storage.copy + let data = yield storage.getObject(savePathDoc); + let forgottenName = cfgForgottenFilesName + pathModule.extname(cmd.getOutputPath()); + yield storage.putObject(cfgForgottenFiles + '/' + docId + '/' + forgottenName, data, data.length); + } catch (err) { + logger.error('Empty storeForgotten: docId = %s\r\n%s', docId, err.stack); + } } if (forceSave) { yield* docsCoServer.setForceSave(docId, forceSave, cmd, isSfcmSuccess && !isError); diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index a44758aa..30708ded 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -517,6 +517,15 @@ function* ExecuteTask(task) { curDate = new Date(); } yield* processDownloadFromStorage(dataConvert, cmd, task, tempDirs); + } else if (cmd.getForgotten()) { + yield* downloadFileFromStorage(cmd.getDocId(), cmd.getForgotten(), tempDirs.source); + logger.debug('downloadFileFromStorage complete(id=%s)', dataConvert.key); + let list = yield utils.listObjects(tempDirs.source, false); + if (list.length > 0) { + dataConvert.fileFrom = list[0]; + } else { + error = constants.UNKNOWN; + } } else { error = constants.UNKNOWN; }