From 8904a33e594929067e4f0261e53638173dea1d30 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 7 Dec 2020 20:01:49 +0300 Subject: [PATCH] [sql] Add creation_date, additional_params columns creation_date - to get non-expiring permanent url additional_params - for future needs --- Common/sources/storage-base.js | 8 +-- Common/sources/storage-fs.js | 11 ++-- Common/sources/storage-s3.js | 2 +- DocService/sources/DocsCoServer.js | 6 +-- DocService/sources/canvasservice.js | 52 +++++++++++-------- DocService/sources/mySqlBaseConnector.js | 16 +++--- DocService/sources/postgreSqlBaseConnector.js | 16 +++--- DocService/sources/taskresult.js | 24 ++++++--- schema/mysql/createdb.sql | 3 ++ schema/postgresql/createdb.sql | 3 ++ 10 files changed, 85 insertions(+), 56 deletions(-) diff --git a/Common/sources/storage-base.js b/Common/sources/storage-base.js index 9cab0d8b..785e6e9a 100644 --- a/Common/sources/storage-base.js +++ b/Common/sources/storage-base.js @@ -78,13 +78,13 @@ exports.deletePath = function(strPath) { return exports.deleteObjects(list); }); }; -exports.getSignedUrl = function(baseUrl, strPath, urlType, optFilename, opt_type) { - return storage.getSignedUrl(baseUrl, getStoragePath(strPath), urlType, optFilename, opt_type); +exports.getSignedUrl = function(baseUrl, strPath, urlType, optFilename, opt_type, opt_creationDate) { + return storage.getSignedUrl(baseUrl, getStoragePath(strPath), urlType, optFilename, opt_type, opt_creationDate); }; -exports.getSignedUrls = function(baseUrl, strPath, urlType) { +exports.getSignedUrls = function(baseUrl, strPath, urlType, opt_creationDate) { return exports.listObjects(getStoragePath(strPath)).then(function(list) { return Promise.all(list.map(function(curValue) { - return exports.getSignedUrl(baseUrl, curValue, urlType); + return exports.getSignedUrl(baseUrl, curValue, urlType, undefined, undefined, opt_creationDate); })).then(function(urls) { var outputMap = {}; for (var i = 0; i < list.length && i < urls.length; ++i) { diff --git a/Common/sources/storage-fs.js b/Common/sources/storage-fs.js index 6e20b795..5d99ea5f 100644 --- a/Common/sources/storage-fs.js +++ b/Common/sources/storage-fs.js @@ -155,16 +155,19 @@ exports.deleteObject = function(strPath) { exports.deleteObjects = function(strPaths) { return Promise.all(strPaths.map(exports.deleteObject)); }; -exports.getSignedUrl = function(baseUrl, strPath, urlType, optFilename, opt_type) { +exports.getSignedUrl = function(baseUrl, strPath, urlType, optFilename, opt_type, opt_creationDate) { return new Promise(function(resolve, reject) { //replace '/' with %2f before encodeURIComponent becase nginx determine %2f as '/' and get wrong system path var userFriendlyName = optFilename ? encodeURIComponent(optFilename.replace(/\//g, "%2f")) : path.basename(strPath); var uri = '/' + cfgBucketName + '/' + cfgStorageFolderName + '/' + strPath + '/' + userFriendlyName; var url = (cfgStorageExternalHost ? cfgStorageExternalHost : baseUrl) + uri; - var date = new Date(); - var expires = Math.ceil(date.getTime() / 1000); - expires += (commonDefines.c_oAscUrlTypes.Session === urlType ? (cfgExpSessionAbsolute / 1000) : cfgStorageUrlExpires) || 31536000; + var date = Date.now(); + let creationDate = opt_creationDate || date; + let expiredAfter = (commonDefines.c_oAscUrlTypes.Session === urlType ? (cfgExpSessionAbsolute / 1000) : cfgStorageUrlExpires) || 31536000; + var expires = creationDate + Math.ceil((date - creationDate)/expiredAfter) * expiredAfter; + expires /= 1000; + expires += expiredAfter; var md5 = crypto.createHash('md5').update(expires + decodeURIComponent(uri) + cfgStorageSecretString).digest("base64"); md5 = md5.replace(/\+/g, "-").replace(/\//g, "_").replace(/=/g, ""); diff --git a/Common/sources/storage-s3.js b/Common/sources/storage-s3.js index 69fc35d8..5a651fe2 100644 --- a/Common/sources/storage-s3.js +++ b/Common/sources/storage-s3.js @@ -204,7 +204,7 @@ exports.deleteObjects = function(strPaths) { } return Promise.all(deletePromises); }; -exports.getSignedUrl = function(baseUrl, strPath, urlType, optFilename, opt_type) { +exports.getSignedUrl = function(baseUrl, strPath, urlType, optFilename, opt_type, opt_creationDate) { return new Promise(function(resolve, reject) { var expires = (commonDefines.c_oAscUrlTypes.Session === urlType ? cfgExpSessionAbsolute : cfgStorageUrlExpires) || 31536000; var userFriendlyName = optFilename ? optFilename.replace(/\//g, "%2f") : path.basename(strPath); diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 0c669f96..e868758f 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -2849,12 +2849,12 @@ exports.install = function(server, callbackFunction) { participant = participants[i]; if (data.needUrlKey) { if (0 == data.needUrlMethod) { - outputData.setData(yield storage.getSignedUrls(participant.baseUrl, data.needUrlKey, data.needUrlType)); + outputData.setData(yield storage.getSignedUrls(participant.baseUrl, data.needUrlKey, data.needUrlType, data.creationDate)); } else if (1 == data.needUrlMethod) { - outputData.setData(yield storage.getSignedUrl(participant.baseUrl, data.needUrlKey, data.needUrlType)); + outputData.setData(yield storage.getSignedUrl(participant.baseUrl, data.needUrlKey, data.needUrlType, undefined, undefined, data.creationDate)); } else { var contentDisposition = cmd.getInline() ? constants.CONTENT_DISPOSITION_INLINE : constants.CONTENT_DISPOSITION_ATTACHMENT; - outputData.setData(yield storage.getSignedUrl(participant.baseUrl, data.needUrlKey, data.needUrlType, cmd.getTitle(), contentDisposition)); + outputData.setData(yield storage.getSignedUrl(participant.baseUrl, data.needUrlKey, data.needUrlType, cmd.getTitle(), contentDisposition, data.creationDate)); } modifyConnectionForPassword(participant, data.needUrlIsCorrectPassword); } diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index f3b98968..6214c243 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -128,8 +128,16 @@ OutputData.prototype = { } }; -function* getOutputData(cmd, outputData, key, status, statusInfo, password, optConn, optAdditionalOutput, opt_bIsRestore) { - var docId = cmd.getDocId(); +function* getOutputData(cmd, outputData, key, optConn, optAdditionalOutput, opt_bIsRestore) { + let status, statusInfo, password, creationDate; + let selectRes = yield taskResult.select(key); + if (selectRes.length > 0) { + let row = selectRes[0]; + status = row.status; + statusInfo = row.status_info; + password = row.password; + creationDate = row.creation_date.getTime(); + } switch (status) { case taskResult.FileStatus.SaveVersion: case taskResult.FileStatus.UpdateVersion: @@ -143,10 +151,10 @@ function* getOutputData(cmd, outputData, key, status, statusInfo, password, optC outputData.setStatus(constants.FILE_STATUS_UPDATE_VERSION); } else { if (taskResult.FileStatus.UpdateVersion === status) { - logger.warn("UpdateVersion expired: docId = %s", docId); + logger.warn("UpdateVersion expired: docId = %s", key); } var updateMask = new taskResult.TaskResultData(); - updateMask.key = docId; + updateMask.key = key; updateMask.status = status; updateMask.statusInfo = statusInfo; var updateTask = new taskResult.TaskResultData(); @@ -187,7 +195,7 @@ function* getOutputData(cmd, outputData, key, status, statusInfo, password, optC isCorrectPassword = decryptedPassword === userPassword; } if(password && !isCorrectPassword) { - logger.debug("getOutputData password mismatch: docId = %s", docId); + logger.debug("getOutputData password mismatch: docId = %s", key); if(encryptedUserPassword) { outputData.setStatus('needpassword'); outputData.setData(constants.CONVERT_PASSWORD); @@ -196,12 +204,13 @@ function* getOutputData(cmd, outputData, key, status, statusInfo, password, optC outputData.setData(constants.CONVERT_DRM); } } else if (optConn) { - outputData.setData(yield storage.getSignedUrls(optConn.baseUrl, key, commonDefines.c_oAscUrlTypes.Session)); + outputData.setData(yield storage.getSignedUrls(optConn.baseUrl, key, commonDefines.c_oAscUrlTypes.Session, creationDate)); } else if (optAdditionalOutput) { optAdditionalOutput.needUrlKey = key; optAdditionalOutput.needUrlMethod = 0; optAdditionalOutput.needUrlType = commonDefines.c_oAscUrlTypes.Session; optAdditionalOutput.needUrlIsCorrectPassword = isCorrectPassword; + optAdditionalOutput.creationDate = creationDate; } } break; @@ -229,6 +238,13 @@ function* getOutputData(cmd, outputData, key, status, statusInfo, password, optC yield cleanupCache(key); } break; + case taskResult.FileStatus.None: + outputData.setStatus('none'); + break; + default: + outputData.setStatus('err'); + outputData.setData(constants.UNKNOWN); + break; } } function* addRandomKeyTaskCmd(cmd) { @@ -383,17 +399,8 @@ function* commandOpen(conn, cmd, outputData, opt_upsertRes, opt_bIsRestore) { } } function* commandOpenFillOutput(conn, cmd, outputData, opt_bIsRestore) { - let needAddTask = false; - let selectRes = yield taskResult.select(cmd.getDocId()); - if (selectRes.length > 0) { - let row = selectRes[0]; - if (taskResult.FileStatus.None === row.status) { - needAddTask = true; - } else { - yield* getOutputData(cmd, outputData, cmd.getDocId(), row.status, row.status_info, row.password, conn, undefined, opt_bIsRestore); - } - } - return needAddTask; + yield* getOutputData(cmd, outputData, cmd.getDocId(), conn, undefined, opt_bIsRestore); + return 'none' === outputData.getStatus(); } function* commandReopen(conn, cmd, outputData) { let res = true; @@ -1295,13 +1302,11 @@ exports.receiveTask = function(data, ack) { if (updateRes.affectedRows > 0) { var outputData = new OutputData(cmd.getCommand()); var command = cmd.getCommand(); - var additionalOutput = {needUrlKey: null, needUrlMethod: null, needUrlType: null, needUrlIsCorrectPassword: undefined}; + var additionalOutput = {needUrlKey: null, needUrlMethod: null, needUrlType: null, needUrlIsCorrectPassword: undefined, creationDate: undefined}; if ('open' == command || 'reopen' == command) { - yield* getOutputData(cmd, outputData, cmd.getDocId(), updateTask.status, - updateTask.statusInfo, updateTask.password, null, additionalOutput); + yield* getOutputData(cmd, outputData, cmd.getDocId(), null, additionalOutput); } else if ('save' == command || 'savefromorigin' == command || 'sfct' == command) { - yield* getOutputData(cmd, outputData, cmd.getSaveKey(), updateTask.status, - updateTask.statusInfo, updateTask.password, null, additionalOutput); + yield* getOutputData(cmd, outputData, cmd.getSaveKey(), null, additionalOutput); } else if ('sfcm' == command) { yield* commandSfcCallback(cmd, true); } else if ('sfc' == command) { @@ -1319,7 +1324,8 @@ exports.receiveTask = function(data, ack) { needUrlKey: additionalOutput.needUrlKey, needUrlMethod: additionalOutput.needUrlMethod, needUrlType: additionalOutput.needUrlType, - needUrlIsCorrectPassword: additionalOutput.needUrlIsCorrectPassword + needUrlIsCorrectPassword: additionalOutput.needUrlIsCorrectPassword, + creationDate: additionalOutput.creationDate }); } } diff --git a/DocService/sources/mySqlBaseConnector.js b/DocService/sources/mySqlBaseConnector.js index 231fc691..e0007711 100644 --- a/DocService/sources/mySqlBaseConnector.js +++ b/DocService/sources/mySqlBaseConnector.js @@ -97,14 +97,16 @@ exports.upsert = function(task, opt_updateUserIndex) { let p2 = addSqlParam(task.status, values); let p3 = addSqlParam(task.statusInfo, values); let p4 = addSqlParam(dateNow, values); - let p5 = addSqlParam(task.userIndex, values); - let p6 = addSqlParam(task.changeId, values); - let p7 = addSqlParam(cbInsert, values); - let p8 = addSqlParam(task.baseurl, values); - let p9 = addSqlParam(task.password, values); + let p5 = addSqlParam(dateNow, values); + let p6 = addSqlParam(task.userIndex, values); + let p7 = addSqlParam(task.changeId, values); + let p8 = addSqlParam(cbInsert, values); + let p9 = addSqlParam(task.baseurl, values); + let p10 = addSqlParam(task.password, values); + let p11 = addSqlParam(task.additionalParams, values); let pDate = addSqlParam(dateNow, values); - var sqlCommand = 'INSERT INTO task_result (id, status, status_info, last_open_date, user_index, change_id, callback, baseurl, password)'+ - ` VALUES (${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8}, ${p9}) ON DUPLICATE KEY UPDATE` + + var sqlCommand = 'INSERT INTO task_result (id, status, status_info, creation_date, last_open_date, user_index, change_id, callback, baseurl, password, additional_params)'+ + ` VALUES (${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8}, ${p9}, ${p10}, ${p11}) ON DUPLICATE KEY UPDATE` + ` last_open_date = ${pDate}`; if (task.callback) { let pCallback = addSqlParam(JSON.stringify(task.callback), values); diff --git a/DocService/sources/postgreSqlBaseConnector.js b/DocService/sources/postgreSqlBaseConnector.js index bcb1d654..ccc4121e 100644 --- a/DocService/sources/postgreSqlBaseConnector.js +++ b/DocService/sources/postgreSqlBaseConnector.js @@ -114,16 +114,18 @@ function getUpsertString(task, values) { let p2 = addSqlParam(task.status, values); let p3 = addSqlParam(task.statusInfo, values); let p4 = addSqlParam(dateNow, values); - let p5 = addSqlParam(task.userIndex, values); - let p6 = addSqlParam(task.changeId, values); - let p7 = addSqlParam(cbInsert, values); - let p8 = addSqlParam(task.baseurl, values); - let p9 = addSqlParam(task.password, values); + let p5 = addSqlParam(dateNow, values); + let p6 = addSqlParam(task.userIndex, values); + let p7 = addSqlParam(task.changeId, values); + let p8 = addSqlParam(cbInsert, values); + let p9 = addSqlParam(task.baseurl, values); + let p10 = addSqlParam(task.password, values); + let p11 = addSqlParam(task.additionalParams, values); if (isSupportOnConflict) { let pDate = addSqlParam(dateNow, values); //http://stackoverflow.com/questions/34762732/how-to-find-out-if-an-upsert-was-an-update-with-postgresql-9-5-upsert - let sqlCommand = "INSERT INTO task_result (id, status, status_info, last_open_date, user_index, change_id, callback, baseurl, password)"; - sqlCommand += ` VALUES (${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8}, ${p9})`; + let sqlCommand = "INSERT INTO task_result (id, status, status_info, creation_date, last_open_date, user_index, change_id, callback, baseurl, password, additional_params)"; + sqlCommand += ` VALUES (${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8}, ${p9}, ${p10}, ${p11})`; sqlCommand += ` ON CONFLICT (id) DO UPDATE SET last_open_date = ${pDate}`; if (task.callback) { let pCallback = addSqlParam(JSON.stringify(task.callback), values); diff --git a/DocService/sources/taskresult.js b/DocService/sources/taskresult.js index c6df9381..c4c16f78 100644 --- a/DocService/sources/taskresult.js +++ b/DocService/sources/taskresult.js @@ -60,11 +60,13 @@ function TaskResultData() { this.status = null; this.statusInfo = null; this.lastOpenDate = null; + this.creationDate = null; this.userIndex = null; this.changeId = null; this.callback = null; this.baseurl = null; this.password = null; + this.additionalParams = null; } TaskResultData.prototype.completeDefaults = function() { if (!this.key) { @@ -79,6 +81,9 @@ TaskResultData.prototype.completeDefaults = function() { if (!this.lastOpenDate) { this.lastOpenDate = new Date(); } + if (!this.creationDate) { + this.creationDate = new Date(); + } if (!this.userIndex) { this.userIndex = 1; } @@ -94,6 +99,9 @@ TaskResultData.prototype.completeDefaults = function() { if (!this.password) { this.password = ''; } + if (!this.additionalParams) { + this.additionalParams = ''; + } }; function upsert(task, opt_updateUserIndex) { @@ -206,13 +214,15 @@ function addRandomKey(task, opt_prefix, opt_size) { let p2 = addSqlParam(task.status, values); let p3 = addSqlParam(task.statusInfo, values); let p4 = addSqlParam(new Date(), values); - let p5 = addSqlParam(task.userIndex, values); - let p6 = addSqlParam(task.changeId, values); - let p7 = addSqlParam(task.callback, values); - let p8 = addSqlParam(task.baseurl, values); - let p9 = addSqlParam(task.password, values); - let sqlCommand = 'INSERT INTO task_result (id, status, status_info, last_open_date, user_index, change_id, callback, baseurl, password)' + - ` VALUES (${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8}, ${p9});`; + let p5 = addSqlParam(new Date(), values); + let p6 = addSqlParam(task.userIndex, values); + let p7 = addSqlParam(task.changeId, values); + let p8 = addSqlParam(task.callback, values); + let p9 = addSqlParam(task.baseurl, values); + let p10 = addSqlParam(task.password, values); + let p11 = addSqlParam(task.additionalParams, values); + let sqlCommand = 'INSERT INTO task_result (id, status, status_info, creation_date, last_open_date, user_index, change_id, callback, baseurl, password, additional_params)' + + ` VALUES (${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8}, ${p9}, ${p10}, ${p11});`; sqlBase.baseConnector.sqlQuery(sqlCommand, function(error, result) { if (error) { reject(error); diff --git a/schema/mysql/createdb.sql b/schema/mysql/createdb.sql index 235b75ba..2b5f4aad 100644 --- a/schema/mysql/createdb.sql +++ b/schema/mysql/createdb.sql @@ -51,11 +51,14 @@ CREATE TABLE IF NOT EXISTS `task_result` ( `id` varchar(255) NOT NULL, `status` tinyint(3) NOT NULL, `status_info` int(10) NOT NULL, + `creation_date` datetime NOT NULL, `last_open_date` datetime NOT NULL, `user_index` int(10) unsigned NOT NULL DEFAULT 1, `change_id` int(10) unsigned NOT NULL DEFAULT 0, `callback` longtext NOT NULL, `baseurl` text NOT NULL, + `password` longtext NOT NULL, + `additional_params` longtext NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/schema/postgresql/createdb.sql b/schema/postgresql/createdb.sql index 96b553a8..0cea33e2 100644 --- a/schema/postgresql/createdb.sql +++ b/schema/postgresql/createdb.sql @@ -26,11 +26,14 @@ CREATE TABLE IF NOT EXISTS "public"."task_result" ( "id" varchar(255) COLLATE "default" NOT NULL, "status" int2 NOT NULL, "status_info" int4 NOT NULL, +"creation_date" timestamp without time zone NOT NULL, "last_open_date" timestamp without time zone NOT NULL, "user_index" int4 NOT NULL DEFAULT 1, "change_id" int4 NOT NULL DEFAULT 0, "callback" text COLLATE "default" NOT NULL, "baseurl" text COLLATE "default" NOT NULL, +"password" text COLLATE "default" NOT NULL, +"additional_params" text COLLATE "default" NOT NULL, PRIMARY KEY ("id") ) WITH (OIDS=FALSE);