From 593a68b8179d0d6ff15585173bdd0546c681f000 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 11 Dec 2023 19:34:45 +0300 Subject: [PATCH] [feature] Use WOPISrc param to make local urls # Conflicts: # Common/sources/storage-fs.js # DocService/sources/DocsCoServer.js # DocService/sources/connectorUtilities.js # DocService/sources/gc.js --- Common/sources/constants.js | 1 + Common/sources/operationContext.js | 21 ++++++++++++++------- Common/sources/storage-fs.js | 4 ++++ Common/sources/utils.js | 8 ++++++++ DocService/sources/DocsCoServer.js | 19 ++++++++++++------- DocService/sources/canvasservice.js | 21 ++++++++++++++++----- DocService/sources/connectorUtilities.js | 19 ++++++++++++++++++- DocService/sources/converterservice.js | 4 ++-- DocService/sources/gc.js | 10 +++++++--- 9 files changed, 82 insertions(+), 25 deletions(-) diff --git a/Common/sources/constants.js b/Common/sources/constants.js index f2e97507..36471cad 100644 --- a/Common/sources/constants.js +++ b/Common/sources/constants.js @@ -50,6 +50,7 @@ exports.VIEWER_ONLY = /^(?:(pdf|djvu|xps|oxps))$/; exports.DEFAULT_DOC_ID = 'docId'; exports.DEFAULT_USER_ID = 'userId'; exports.ALLOWED_PROTO = /^https?$/i; +exports.SHARED_KEY_NAME = 'WOPISrc'; exports.RIGHTS = { None : 0, diff --git a/Common/sources/operationContext.js b/Common/sources/operationContext.js index ab2643a5..0eeb11a5 100644 --- a/Common/sources/operationContext.js +++ b/Common/sources/operationContext.js @@ -41,17 +41,18 @@ function Context(){ this.logger = logger.getLogger('nodeJS'); this.initDefault(); } -Context.prototype.init = function(tenant, docId, userId) { +Context.prototype.init = function(tenant, docId, userId, opt_shardKey) { this.setTenant(tenant); this.setDocId(docId); this.setUserId(userId); + this.setShardKey(opt_shardKey); this.config = null; this.secret = null; this.license = null; }; Context.prototype.initDefault = function() { - this.init(tenantManager.getDefautTenant(), constants.DEFAULT_DOC_ID, constants.DEFAULT_USER_ID); + this.init(tenantManager.getDefautTenant(), constants.DEFAULT_DOC_ID, constants.DEFAULT_USER_ID, undefined); }; Context.prototype.initFromConnection = function(conn) { let tenant = tenantManager.getTenantByConnection(this, conn); @@ -64,19 +65,21 @@ Context.prototype.initFromConnection = function(conn) { } } let userId = conn.user?.id; - this.init(tenant, docId || this.docId, userId || this.userId); + let shardKey = utils.getShardByConnection(this, conn); + this.init(tenant, docId || this.docId, userId || this.userId, shardKey); }; Context.prototype.initFromRequest = function(req) { let tenant = tenantManager.getTenantByRequest(this, req); - this.init(tenant, this.docId, this.userId); + let shardKey = utils.getShardKeyByRequest(this, req); + this.init(tenant, this.docId, this.userId, shardKey); }; Context.prototype.initFromTaskQueueData = function(task) { let ctx = task.getCtx(); - this.init(ctx.tenant, ctx.docId, ctx.userId); + this.init(ctx.tenant, ctx.docId, ctx.userId, ctx.shardKey); }; Context.prototype.initFromPubSub = function(data) { let ctx = data.ctx; - this.init(ctx.tenant, ctx.docId, ctx.userId); + this.init(ctx.tenant, ctx.docId, ctx.userId, ctx.shardKey); }; Context.prototype.initTenantCache = async function() { this.config = await tenantManager.getTenantConfig(this); @@ -95,11 +98,15 @@ Context.prototype.setUserId = function(userId) { this.userId = userId; this.logger.addContext('USERID', userId); }; +Context.prototype.setShardKey = function(shardKey) { + this.shardKey = shardKey; +}; Context.prototype.toJSON = function() { return { tenant: this.tenant, docId: this.docId, - userId: this.userId + userId: this.userId, + shardKey: this.shardKey } }; Context.prototype.getCfg = function(property, defaultValue) { diff --git a/Common/sources/storage-fs.js b/Common/sources/storage-fs.js index efbfd3ed..e4feee43 100644 --- a/Common/sources/storage-fs.js +++ b/Common/sources/storage-fs.js @@ -40,6 +40,7 @@ var utils = require("./utils"); var crypto = require('crypto'); const ms = require('ms'); const commonDefines = require('./../../Common/sources/commondefines'); +const constants = require('./../../Common/sources/constants'); var config = require('config'); var configStorage = config.get('storage'); @@ -202,6 +203,9 @@ exports.getSignedUrl = function(ctx, baseUrl, strPath, urlType, optFilename, opt url += '?md5=' + encodeURIComponent(md5); url += '&expires=' + encodeURIComponent(expires); + if (ctx.shardKey) { + url += `&${constants.SHARED_KEY_NAME}=${encodeURIComponent(ctx.shardKey)}`; + } url += '&filename=' + userFriendlyName; resolve(url); }); diff --git a/Common/sources/utils.js b/Common/sources/utils.js index 42e33ac2..3b02a5f9 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -731,6 +731,14 @@ function getDomainByRequest(ctx, req) { } exports.getDomainByConnection = getDomainByConnection; exports.getDomainByRequest = getDomainByRequest; +function getShardByConnection(ctx, conn) { + return conn?.handshake?.query?.[constants.SHARED_KEY_NAME]; +} +function getShardKeyByRequest(ctx, req) { + return req.query[constants.SHARED_KEY_NAME]; +} +exports.getShardByConnection = getShardByConnection; +exports.getShardKeyByRequest = getShardKeyByRequest; function stream2Buffer(stream) { return new Promise(function(resolve, reject) { if (!stream.readable) { diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 46574086..594760ee 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -910,7 +910,7 @@ async function applyForceSaveCache(ctx, docId, forceSave, type, opt_userConnecti } return res; } -async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_userId, opt_userConnectionId, opt_userConnectionDocId, opt_userIndex, opt_responseKey, opt_baseUrl, opt_queue, opt_pubsub, opt_conn) { +async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_userId, opt_userConnectionId, opt_userConnectionDocId, opt_userIndex, opt_responseKey, opt_baseUrl, opt_queue, opt_pubsub, opt_conn, opt_initShardKey) { ctx.logger.debug('startForceSave start'); let res = {code: commonDefines.c_oAscServerCommandErrors.NoError, time: null, inProgress: false}; let startedForceSave; @@ -975,7 +975,7 @@ async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_ //start new convert let status = await converterService.convertFromChanges(ctx, docId, baseUrl, forceSave, startedForceSave.changeInfo, opt_userdata, opt_formdata, opt_userConnectionId, opt_userConnectionDocId, opt_responseKey, priority, expiration, - opt_queue); + opt_queue, undefined, opt_initShardKey); if (constants.NO_ERROR === status.err) { res.time = forceSave.getTime(); } else { @@ -1319,7 +1319,7 @@ function* cleanDocumentOnExitNoChanges(ctx, docId, opt_userId, opt_userIndex, op yield* cleanDocumentOnExit(ctx, docId, false, opt_userIndex); } -function createSaveTimer(ctx, docId, opt_userId, opt_userIndex, opt_queue, opt_noDelay) { +function createSaveTimer(ctx, docId, opt_userId, opt_userIndex, opt_queue, opt_noDelay, opt_initShardKey) { return co(function*(){ const tenAscSaveTimeOutDelay = ctx.getCfg('services.CoAuthoring.server.savetimeoutdelay', cfgAscSaveTimeOutDelay); @@ -1337,7 +1337,7 @@ function createSaveTimer(ctx, docId, opt_userId, opt_userIndex, opt_queue, opt_n } while (true) { if (!sqlBase.isLockCriticalSection(docId)) { - canvasService.saveFromChanges(ctx, docId, updateTask.statusInfo, null, opt_userId, opt_userIndex, opt_queue); + canvasService.saveFromChanges(ctx, docId, updateTask.statusInfo, null, opt_userId, opt_userIndex, opt_queue, opt_initShardKey); break; } yield utils.sleep(c_oAscLockTimeOutDelay); @@ -2505,7 +2505,7 @@ exports.install = function(server, callbackFunction) { upsertRes = yield canvasService.commandOpenStartPromise(ctx, docId, utils.getBaseUrlByConnection(ctx, conn), true, data.documentCallbackUrl, format); let isInserted = upsertRes.affectedRows == 1; curIndexUser = isInserted ? 1 : upsertRes.insertId; - if (isInserted && undefined !== data.timezoneOffset) { + if ((isInserted || (wopiParams && 2 === curIndexUser)) && (undefined !== data.timezoneOffset || ctx.shardKey)) { //todo insert in commandOpenStartPromise. insert here for database compatibility if (false === canvasService.hasAdditionalCol) { let selectRes = yield taskResult.select(ctx, docId); @@ -2515,8 +2515,13 @@ exports.install = function(server, callbackFunction) { let task = new taskResult.TaskResultData(); task.tenant = ctx.tenant; task.key = docId; - //todo duplicate created_at because CURRENT_TIMESTAMP uses server timezone - task.additional = sqlBase.DocumentAdditional.prototype.setOpenedAt(Date.now(), data.timezoneOffset); + if (undefined !== data.timezoneOffset) { + //todo duplicate created_at because CURRENT_TIMESTAMP uses server timezone + task.additional = sqlBase.DocumentAdditional.prototype.setOpenedAt(Date.now(), data.timezoneOffset); + } + if (ctx.shardKey) { + task.additional += sqlBase.DocumentAdditional.prototype.setShardKey(ctx.shardKey); + } yield taskResult.update(ctx, task); } else { ctx.logger.warn('auth unknown column "additional"'); diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index dabcb298..1612a1fd 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -618,12 +618,15 @@ function* commandSendMailMerge(ctx, cmd, outputData) { outputData.setData(cmd.getSaveKey()); } } -let commandSfctByCmd = co.wrap(function*(ctx, cmd, opt_priority, opt_expiration, opt_queue) { +let commandSfctByCmd = co.wrap(function*(ctx, cmd, opt_priority, opt_expiration, opt_queue, opt_initShardKey) { var selectRes = yield taskResult.select(ctx, cmd.getDocId()); var row = selectRes.length > 0 ? selectRes[0] : null; if (!row) { return; } + if (opt_initShardKey) { + ctx.setShardKey(sqlBase.DocumentAdditional.prototype.getShardKey(row.additional)); + } yield* addRandomKeyTaskCmd(ctx, cmd); addPasswordToCmd(ctx, cmd, row.password); addOriginFormat(ctx, cmd, row); @@ -1520,8 +1523,13 @@ function getPrintFileUrl(ctx, docId, baseUrl, filename) { } //while save printed file Chrome's extension seems to rely on the resource name set in the URI https://stackoverflow.com/a/53593453 //replace '/' with %2f before encodeURIComponent becase nginx determine %2f as '/' and get wrong system path - var userFriendlyName = encodeURIComponent(filename.replace(/\//g, "%2f")); - return `${baseUrl}/printfile/${encodeURIComponent(docId)}/${userFriendlyName}?token=${encodeURIComponent(token)}&filename=${userFriendlyName}`; + let userFriendlyName = encodeURIComponent(filename.replace(/\//g, "%2f")); + let res = `${baseUrl}/printfile/${encodeURIComponent(docId)}/${userFriendlyName}?token=${encodeURIComponent(token)}`; + if (ctx.shardKey) { + res += `&${constants.SHARED_KEY_NAME}=${encodeURIComponent(ctx.shardKey)}`; + } + res += `&filename=${userFriendlyName}`; + return res; }); } exports.getPrintFileUrl = getPrintFileUrl; @@ -1668,7 +1676,7 @@ exports.downloadFile = function(req, res) { } }); }; -exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId, opt_userIndex, opt_queue) { +exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId, opt_userIndex, opt_queue, opt_initShardKey) { return co(function* () { try { var startDate = null; @@ -1682,7 +1690,10 @@ exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId if (row && row.status == commonDefines.FileStatus.SaveVersion && row.status_info == statusInfo) { if (null == optFormat) { optFormat = changeFormatByOrigin(ctx, row, constants.AVS_OFFICESTUDIO_FILE_OTHER_OOXML); - } + } + if (opt_initShardKey) { + ctx.setShardKey(sqlBase.DocumentAdditional.prototype.getShardKey(row.additional)); + } var cmd = new commonDefines.InputCommand(); cmd.setCommand('sfc'); cmd.setDocId(docId); diff --git a/DocService/sources/connectorUtilities.js b/DocService/sources/connectorUtilities.js index 92c0c529..eb658604 100644 --- a/DocService/sources/connectorUtilities.js +++ b/DocService/sources/connectorUtilities.js @@ -152,4 +152,21 @@ DocumentAdditional.prototype.getOpenedAt = function(str) { }); return res; }; -exports.DocumentAdditional = DocumentAdditional; \ No newline at end of file +DocumentAdditional.prototype.setShardKey = function(shardKey) { + let additional = new DocumentAdditional(); + additional.data.push({shardKey}); + return additional.toSQLInsert(); +}; +DocumentAdditional.prototype.getShardKey = function(str) { + let res; + let val = new DocumentAdditional(); + val.fromString(str); + val.data.forEach((elem) => { + if (elem.shardKey) { + res = elem.shardKey; + } + }); + return res; +}; + +exports.DocumentAdditional = DocumentAdditional; diff --git a/DocService/sources/converterservice.js b/DocService/sources/converterservice.js index 4b338afb..e2b05469 100644 --- a/DocService/sources/converterservice.js +++ b/DocService/sources/converterservice.js @@ -185,7 +185,7 @@ function* convertByCmd(ctx, cmd, async, opt_fileTo, opt_taskExist, opt_priority, } async function convertFromChanges(ctx, docId, baseUrl, forceSave, externalChangeInfo, opt_userdata, opt_formdata, opt_userConnectionId, - opt_userConnectionDocId, opt_responseKey, opt_priority, opt_expiration, opt_queue, opt_redisKey) { + opt_userConnectionDocId, opt_responseKey, opt_priority, opt_expiration, opt_queue, opt_redisKey, opt_initShardKey) { var cmd = new commonDefines.InputCommand(); cmd.setCommand('sfcm'); cmd.setDocId(docId); @@ -215,7 +215,7 @@ async function convertFromChanges(ctx, docId, baseUrl, forceSave, externalChange cmd.setRedisKey(opt_redisKey); } - await canvasService.commandSfctByCmd(ctx, cmd, opt_priority, opt_expiration, opt_queue); + await canvasService.commandSfctByCmd(ctx, cmd, opt_priority, opt_expiration, opt_queue, opt_initShardKey); var fileTo = constants.OUTPUT_NAME; let outputExt = formatChecker.getStringFromFormat(cmd.getOutputFormat()); if (outputExt) { diff --git a/DocService/sources/gc.js b/DocService/sources/gc.js index 6690dd68..d2ae650d 100644 --- a/DocService/sources/gc.js +++ b/DocService/sources/gc.js @@ -47,6 +47,7 @@ var commondefines = require('./../../Common/sources/commondefines'); var queueService = require('./../../Common/sources/taskqueueRabbitMQ'); var operationContext = require('./../../Common/sources/operationContext'); var pubsubService = require('./pubsubRabbitMQ'); +const sqlBase = require("./baseConnector"); var cfgExpFilesCron = config.get('services.CoAuthoring.expire.filesCron'); var cfgExpDocumentsCron = config.get('services.CoAuthoring.expire.documentsCron'); @@ -76,7 +77,8 @@ var checkFileExpire = function(expireSeconds) { for (var i = 0; i < expired.length; ++i) { let tenant = expired[i].tenant; let docId = expired[i].id; - ctx.init(tenant, docId, ctx.userId); + let shardKey = sqlBase.DocumentAdditional.prototype.getShardKey(expired[i].additional); + ctx.init(tenant, docId, ctx.userId, shardKey); yield ctx.initTenantCache(); //todo tenant //check that no one is in the document @@ -122,7 +124,8 @@ var checkDocumentExpire = function() { yield ctx.initTenantCache(); var hasChanges = yield docsCoServer.hasChanges(ctx, docId); if (hasChanges) { - yield docsCoServer.createSaveTimer(ctx, docId, null, null, queue, true); + //todo opt_initShardKey from getDocumentPresenceExpired data or from db + yield docsCoServer.createSaveTimer(ctx, docId, null, null, queue, true, true); startSaveCount++; } else { yield docsCoServer.cleanDocumentOnExitNoChangesPromise(ctx, docId); @@ -170,9 +173,10 @@ let forceSaveTimeout = function() { if (docId) { ctx.init(tenant, docId, ctx.userId); yield ctx.initTenantCache(); + //todo opt_initShardKey from ForceSave data or from db actions.push(docsCoServer.startForceSave(ctx, docId, commondefines.c_oAscForceSaveTypes.Timeout, undefined, undefined, undefined, undefined, - undefined, undefined, undefined, undefined, queue, pubsub)); + undefined, undefined, undefined, undefined, queue, pubsub, undefined, true)); } } yield Promise.all(actions);