From cd92ade12b8e5eec9248483c7c67d92e487774c2 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Tue, 6 Sep 2022 19:40:11 +0300 Subject: [PATCH 01/45] [wopi] Add /lool/convert-to handler --- DocService/sources/converterservice.js | 80 ++++++++++++++++++++++++++ DocService/sources/server.js | 5 ++ DocService/sources/wopiClient.js | 2 +- 3 files changed, 86 insertions(+), 1 deletion(-) diff --git a/DocService/sources/converterservice.js b/DocService/sources/converterservice.js index 52d6b510..1d8642be 100644 --- a/DocService/sources/converterservice.js +++ b/DocService/sources/converterservice.js @@ -36,6 +36,7 @@ const path = require('path'); var config = require('config'); var co = require('co'); const locale = require('windows-locale'); +const mime = require('mime'); var taskResult = require('./taskresult'); var utils = require('./../../Common/sources/utils'); var constants = require('./../../Common/sources/constants'); @@ -439,8 +440,87 @@ function builderRequest(req, res) { } }); } +function convertTo(req, res) { + return co(function*() { + let ctx = new operationContext.Context(); + try { + ctx.initFromRequest(req); + ctx.logger.info('convert-to start'); + let format = req.body['format']; + if (req.params.format) { + format = req.params.format; + } + let outputFormat = formatChecker.getFormatFromString(format); + if (constants.AVS_OFFICESTUDIO_FILE_UNKNOWN === outputFormat) { + ctx.logger.warn('convert-to unexpected format = %s', format); + res.sendStatus(400); + return; + } + //todo https://github.com/CollaboraOnline/online/blob/4d6ca688f98d217866b9ea49113a356134dfba3a/wsd/COOLWSD.cpp + //req.body['options'], req.body['FullSheetPreview'], req.body['PDFVer'] + let docId, fileTo, status, originalname; + if (req.file && req.file.originalname && req.file.buffer) { + originalname = req.file.originalname; + let filetype = path.extname(req.file.originalname).substring(1); + if (filetype && !constants.EXTENTION_REGEX.test(filetype)) { + ctx.logger.warn('convertRequest unexpected filetype = %s', filetype); + res.sendStatus(400); + return; + } + + let task = yield* taskResult.addRandomKeyTask(ctx, undefined, 'conv_', 8); + docId = task.key; + ctx.setDocId(docId); + + //todo stream + let buffer = req.file.buffer; + yield storageBase.putObject(ctx, docId + '/origin.' + filetype, buffer, buffer.length); + + let cmd = new commonDefines.InputCommand(); + cmd.setCommand('conv'); + cmd.setDocId(docId); + cmd.setSaveKey(docId); + cmd.setFormat(filetype); + cmd.setOutputFormat(outputFormat); + + fileTo = constants.OUTPUT_NAME; + let outputExt = formatChecker.getStringFromFormat(outputFormat); + if (outputExt) { + fileTo += '.' + outputExt; + } + + let queueData = new commonDefines.TaskQueueData(); + queueData.setCtx(ctx); + queueData.setCmd(cmd); + queueData.setToFile(fileTo); + queueData.setFromOrigin(true); + yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); + + let async = false; + status = yield* convertByCmd(ctx, cmd, async, fileTo); + } + if (status && status.end && constants.NO_ERROR === status.err) { + let filename = path.basename(originalname, path.extname(originalname)) + path.extname(fileTo); + let streamObj = yield storage.createReadStream(ctx, `${docId}/${fileTo}`); + res.setHeader('Content-Disposition', utils.getContentDisposition(filename, null, constants.CONTENT_DISPOSITION_INLINE)); + res.setHeader('Content-Length', streamObj.contentLength); + res.setHeader('Content-Type', mime.getType(filename)); + yield utils.pipeStreams(streamObj.readStream, res, true); + } else { + ctx.logger.error('convert-to error status:%j', status); + res.sendStatus(400); + } + } catch (err) { + ctx.logger.error('convert-to error:%s', err.stack); + res.sendStatus(400); + } finally { + ctx.logger.info('convert-to end'); + } + }); +} exports.convertFromChanges = convertFromChanges; exports.convertJson = convertRequestJson; exports.convertXml = convertRequestXml; +exports.convertTo = convertTo; exports.builder = builderRequest; diff --git a/DocService/sources/server.js b/DocService/sources/server.js index aae379ba..3ae9880a 100644 --- a/DocService/sources/server.js +++ b/DocService/sources/server.js @@ -66,6 +66,7 @@ const cfgTokenEnableBrowser = configCommon.get('services.CoAuthoring.token.enabl const cfgTokenEnableRequestInbox = configCommon.get('services.CoAuthoring.token.enable.request.inbox'); const cfgTokenEnableRequestOutbox = configCommon.get('services.CoAuthoring.token.enable.request.outbox'); const cfgLicenseFile = configCommon.get('license.license_file'); +const cfgDownloadMaxBytes = config.get('FileConverter.converter.maxDownloadBytes'); const app = express(); app.disable('x-powered-by'); @@ -243,8 +244,12 @@ docsCoServer.install(server, () => { app.delete('/internal/cluster/inactive', utils.checkClientIp, docsCoServer.shutdown); if (cfgWopiEnable) { + //todo dest + let fileForms = multer({limits: {fieldSize: cfgDownloadMaxBytes}}); app.get('/hosting/discovery', utils.checkClientIp, wopiClient.discovery); app.get('/hosting/capabilities', utils.checkClientIp, wopiClient.collaboraCapabilities); + app.post('/lool/convert-to/:format?', utils.checkClientIp, urleEcodedParser, fileForms.single('data'), converterService.convertTo); + app.post('/cool/convert-to/:format?', utils.checkClientIp, urleEcodedParser, fileForms.single('data'), converterService.convertTo); app.post('/hosting/wopi/:documentType/:mode', urleEcodedParser, forms.none(), utils.lowercaseQueryString, wopiClient.getEditorHtml); } diff --git a/DocService/sources/wopiClient.js b/DocService/sources/wopiClient.js index bcad1b7b..eb50f079 100644 --- a/DocService/sources/wopiClient.js +++ b/DocService/sources/wopiClient.js @@ -193,7 +193,7 @@ function discovery(req, res) { function collaboraCapabilities(req, res) { return co(function*() { let output = { - "convert-to": {"available": false}, "hasMobileSupport": true, "hasProxyPrefix": false, "hasTemplateSaveAs": false, + "convert-to": {"available": true, "endpoint":"/lool/convert-to"}, "hasMobileSupport": true, "hasProxyPrefix": false, "hasTemplateSaveAs": false, "hasTemplateSource": true, "productVersion": commonDefines.buildVersion }; let ctx = new operationContext.Context(); From 42fe73459a41c07ffef1317f82fcdd720e2088a4 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Wed, 7 Sep 2022 09:32:18 +0300 Subject: [PATCH 02/45] [bug] Fix startup error --- DocService/sources/server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DocService/sources/server.js b/DocService/sources/server.js index 3ae9880a..3f0f4342 100644 --- a/DocService/sources/server.js +++ b/DocService/sources/server.js @@ -66,7 +66,7 @@ const cfgTokenEnableBrowser = configCommon.get('services.CoAuthoring.token.enabl const cfgTokenEnableRequestInbox = configCommon.get('services.CoAuthoring.token.enable.request.inbox'); const cfgTokenEnableRequestOutbox = configCommon.get('services.CoAuthoring.token.enable.request.outbox'); const cfgLicenseFile = configCommon.get('license.license_file'); -const cfgDownloadMaxBytes = config.get('FileConverter.converter.maxDownloadBytes'); +const cfgDownloadMaxBytes = configCommon.get('FileConverter.converter.maxDownloadBytes'); const app = express(); app.disable('x-powered-by'); From 620cfc040d8438443e7944ad07c9a48687f29ccd Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Wed, 7 Sep 2022 14:00:54 +0300 Subject: [PATCH 03/45] [wopi] Add "convert" as wopi discovery action --- Common/config/default.json | 3 +- DocService/sources/converterservice.js | 113 +++++++++++++++++++++- DocService/sources/server.js | 2 + DocService/sources/wopiClient.js | 126 +++++++++++++++++++++++-- 4 files changed, 230 insertions(+), 14 deletions(-) diff --git a/Common/config/default.json b/Common/config/default.json index 3be64ae5..0b588e01 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -81,7 +81,8 @@ "favIconUrlCell" : "/web-apps/apps/spreadsheeteditor/main/resources/img/favicon.ico", "favIconUrlSlide" : "/web-apps/apps/presentationeditor/main/resources/img/favicon.ico", "fileInfoBlockList" : ["FileUrl"], - "wordView": ["pdf", "djvu", "xps", "oxps", "doc", "dotx", "dotm", "dot", "fodt", "ott", "rtf", "mht", "html", "htm", "xml", "epub", "fb2"], + "pdfView": ["pdf", "djvu", "xps", "oxps"], + "wordView": ["doc", "dotx", "dotm", "dot", "fodt", "ott", "rtf", "mht", "html", "htm", "xml", "epub", "fb2"], "wordEdit": ["docx", "docm", "docxf", "oform", "odt", "txt"], "cellView": ["xls", "xlsb", "xltx", "xltm", "xlt", "fods", "ots"], "cellEdit": ["xlsx", "xlsm", "ods", "csv"], diff --git a/DocService/sources/converterservice.js b/DocService/sources/converterservice.js index 1d8642be..08f07f7a 100644 --- a/DocService/sources/converterservice.js +++ b/DocService/sources/converterservice.js @@ -43,6 +43,7 @@ var constants = require('./../../Common/sources/constants'); var commonDefines = require('./../../Common/sources/commondefines'); var docsCoServer = require('./DocsCoServer'); var canvasService = require('./canvasservice'); +var wopiClient = require('./wopiClient'); var storage = require('./../../Common/sources/storage-base'); var formatChecker = require('./../../Common/sources/formatchecker'); var statsDClient = require('./../../Common/sources/statsdclient'); @@ -50,20 +51,20 @@ var storageBase = require('./../../Common/sources/storage-base'); var operationContext = require('./../../Common/sources/operationContext'); const sqlBase = require('./baseConnector'); +const cfgTokenEnableBrowser = config.get('services.CoAuthoring.token.enable.browser'); + var CONVERT_ASYNC_DELAY = 1000; var clientStatsD = statsDClient.getClient(); -function* getConvertStatus(ctx, cmd, selectRes, opt_checkPassword) { +function* getConvertStatus(ctx, docId, encryptedUserPassword, selectRes, opt_checkPassword) { var status = new commonDefines.ConvertStatus(constants.NO_ERROR); if (selectRes.length > 0) { - var docId = cmd.getDocId(); var row = selectRes[0]; let password = opt_checkPassword && sqlBase.DocumentPassword.prototype.getCurPassword(ctx, row.password); switch (row.status) { case taskResult.FileStatus.Ok: if (password) { - let encryptedUserPassword = cmd.getPassword(); let isCorrectPassword; if (encryptedUserPassword) { let decryptedPassword = yield utils.decryptPassword(password); @@ -147,7 +148,7 @@ function* convertByCmd(ctx, cmd, async, opt_fileTo, opt_taskExist, opt_priority, var status; if (!bCreate) { selectRes = yield taskResult.select(ctx, docId); - status = yield* getConvertStatus(ctx, cmd, selectRes, opt_checkPassword); + status = yield* getConvertStatus(ctx, cmd.getDocId() ,cmd.getPassword(), selectRes, opt_checkPassword); } else { var queueData = new commonDefines.TaskQueueData(); queueData.setCtx(ctx); @@ -169,7 +170,7 @@ function* convertByCmd(ctx, cmd, async, opt_fileTo, opt_taskExist, opt_priority, } yield utils.sleep(CONVERT_ASYNC_DELAY); selectRes = yield taskResult.select(ctx, docId); - status = yield* getConvertStatus(ctx, cmd, selectRes, opt_checkPassword); + status = yield* getConvertStatus(ctx, cmd.getDocId() ,cmd.getPassword(), selectRes, opt_checkPassword); waitTime += CONVERT_ASYNC_DELAY; if (waitTime > utils.CONVERTION_TIMEOUT) { status.err = constants.CONVERT_TIMEOUT; @@ -519,8 +520,110 @@ function convertTo(req, res) { } }); } +function convertAndEdit(ctx, wopiParams, filetypeFrom, filetypeTo) { + return co(function*() { + try { + ctx.logger.info('convert-and-edit start'); + + let task = yield* taskResult.addRandomKeyTask(ctx, undefined, 'conv_', 8); + let docId = task.key; + let outputFormat = formatChecker.getFormatFromString(filetypeTo); + if (constants.AVS_OFFICESTUDIO_FILE_UNKNOWN === outputFormat) { + ctx.logger.debug('convert-and-edit unknown outputFormat %s', filetypeTo); + return; + } + + let cmd = new commonDefines.InputCommand(); + cmd.setCommand('conv'); + cmd.setDocId(docId); + cmd.setUrl('dummy-url'); + cmd.setWopiParams(wopiParams); + cmd.setFormat(filetypeFrom); + cmd.setOutputFormat(outputFormat); + + let fileTo = constants.OUTPUT_NAME; + let outputExt = formatChecker.getStringFromFormat(outputFormat); + if (outputExt) { + fileTo += '.' + outputExt; + } + + let queueData = new commonDefines.TaskQueueData(); + queueData.setCtx(ctx); + queueData.setCmd(cmd); + queueData.setToFile(fileTo); + yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); + + let async = true; + yield* convertByCmd(ctx, cmd, async, fileTo); + return docId; + } catch (err) { + ctx.logger.error('convert-and-edit error:%s', err.stack); + } finally { + ctx.logger.info('convert-and-edit end'); + } + }); +} +function getConverterHtmlHandler(req, res) { + return co(function*() { + let isJson = true; + let ctx = new operationContext.Context(); + try { + ctx.initFromRequest(req); + ctx.logger.info('convert-and-edit-handler start'); + + let wopiSrc = req.query['wopisrc']; + let access_token = req.query['access_token']; + let targetext = req.query['targetext']; + let docId = req.query['docid']; + ctx.setDocId(docId); + if (!(wopiSrc && access_token && access_token && targetext && docId) || + constants.AVS_OFFICESTUDIO_FILE_UNKNOWN === formatChecker.getFormatFromString(targetext)) { + ctx.logger.debug('convert-and-edit-handler invalid params: wopiSrc=%s; access_token=%s; targetext=%s; docId=%s', wopiSrc, access_token, targetext, docId); + utils.fillResponse(req, res, new commonDefines.ConvertStatus(constants.CONVERT_PARAMS), isJson); + return; + } + let token = req.query['token']; + if (cfgTokenEnableBrowser) { + let checkJwtRes = yield docsCoServer.checkJwt(ctx, token, commonDefines.c_oAscSecretType.Browser); + if (checkJwtRes.decoded) { + docId = checkJwtRes.decoded.docId; + } else { + ctx.logger.debug('convert-and-edit-handler invalid token %j', token); + utils.fillResponse(req, res, new commonDefines.ConvertStatus(constants.VKEY), isJson); + return; + } + } + ctx.setDocId(docId); + + let selectRes = yield taskResult.select(ctx, docId); + let status = yield* getConvertStatus(ctx, docId, undefined, selectRes); + if (status.end && constants.NO_ERROR === status.err) { + let fileTo = `${docId}/${constants.OUTPUT_NAME}.${targetext}`; + + let metadata = yield storage.headObject(ctx, fileTo); + let streamObj = yield storage.createReadStream(ctx, fileTo); + let postRes = yield wopiClient.putRelativeFile(ctx, wopiSrc, access_token, null, streamObj.readStream, metadata.ContentLength, `.${targetext}`, true); + if (postRes) { + let fileInfo = JSON.parse(postRes.body); + status.setUrl(fileInfo.HostEditUrl); + status.setExtName('.' + targetext); + } else { + status.err = constants.UNKNOWN; + } + } + utils.fillResponse(req, res, status, isJson); + } catch (err) { + ctx.logger.error('convert-and-edit-handler error:%s', err.stack); + utils.fillResponse(req, res, new commonDefines.ConvertStatus(constants.UNKNOWN), isJson); + } finally { + ctx.logger.info('convert-and-edit-handler end'); + } + }); +} exports.convertFromChanges = convertFromChanges; exports.convertJson = convertRequestJson; exports.convertXml = convertRequestXml; exports.convertTo = convertTo; +exports.convertAndEdit = convertAndEdit; +exports.getConverterHtmlHandler = getConverterHtmlHandler; exports.builder = builderRequest; diff --git a/DocService/sources/server.js b/DocService/sources/server.js index 3f0f4342..92fba29a 100644 --- a/DocService/sources/server.js +++ b/DocService/sources/server.js @@ -251,6 +251,8 @@ docsCoServer.install(server, () => { app.post('/lool/convert-to/:format?', utils.checkClientIp, urleEcodedParser, fileForms.single('data'), converterService.convertTo); app.post('/cool/convert-to/:format?', utils.checkClientIp, urleEcodedParser, fileForms.single('data'), converterService.convertTo); app.post('/hosting/wopi/:documentType/:mode', urleEcodedParser, forms.none(), utils.lowercaseQueryString, wopiClient.getEditorHtml); + app.post('/hosting/wopi/convert-and-edit/:ext/:targetext', urleEcodedParser, forms.none(), utils.lowercaseQueryString, wopiClient.getConverterHtml); + app.get('/hosting/wopi/convert-and-edit-handler', utils.lowercaseQueryString, converterService.getConverterHtmlHandler); } app.post('/dummyCallback', utils.checkClientIp, rawFileParser, function(req, res){ diff --git a/DocService/sources/wopiClient.js b/DocService/sources/wopiClient.js index eb50f079..437c40d6 100644 --- a/DocService/sources/wopiClient.js +++ b/DocService/sources/wopiClient.js @@ -49,6 +49,7 @@ const tenantManager = require('./../../Common/sources/tenantManager'); const sqlBase = require('./baseConnector'); const taskResult = require('./taskresult'); const canvasService = require('./canvasservice'); +const converterService = require('./converterservice'); const cfgTokenOutboxAlgorithm = config.get('services.CoAuthoring.token.outbox.algorithm'); const cfgTokenOutboxExpires = config.get('services.CoAuthoring.token.outbox.expires'); @@ -57,6 +58,7 @@ const cfgCallbackRequestTimeout = config.get('services.CoAuthoring.server.callba const cfgDownloadTimeout = config.get('FileConverter.converter.downloadTimeout'); const cfgWopiFileInfoBlockList = config.get('wopi.fileInfoBlockList'); const cfgWopiWopiZone = config.get('wopi.wopiZone'); +const cfgWopiPdfView = config.get('wopi.pdfView'); const cfgWopiWordView = config.get('wopi.wordView'); const cfgWopiWordEdit = config.get('wopi.wordEdit'); const cfgWopiCellView = config.get('wopi.cellView'); @@ -105,8 +107,12 @@ function discovery(req, res) { let baseUrl = cfgWopiHost || utils.getBaseUrlByRequest(req); let names = ['Word','Excel','PowerPoint']; let favIconUrls = [cfgWopiFavIconUrlWord, cfgWopiFavIconUrlCell, cfgWopiFavIconUrlSlide]; - let exts = [{view: cfgWopiWordView, edit: cfgWopiWordEdit}, {view: cfgWopiCellView, edit: cfgWopiCellEdit}, - {view: cfgWopiSlideView, edit: cfgWopiSlideEdit}]; + let exts = [ + {targetext: 'docx', view: cfgWopiPdfView.concat(cfgWopiWordView), edit: cfgWopiWordEdit}, + {targetext: 'xlsx', view: cfgWopiCellView, edit: cfgWopiCellEdit}, + {targetext: 'pptx', view: cfgWopiSlideView, edit: cfgWopiSlideEdit} + ]; + let templateStart = `${baseUrl}/hosting/wopi`; let templateEnd = `&<rs=DC_LLCC&><dchat=DISABLE_CHAT&><embed=EMBEDDED&>`; templateEnd += `<fs=FULLSCREEN&><hid=HOST_SESSION_ID&><rec=RECORDING&>`; @@ -129,6 +135,10 @@ function discovery(req, res) { for (let j = 0; j < ext.view.length; ++j) { output += ``; output += ``; + if (-1 === cfgWopiPdfView.indexOf(ext.view[j])) { + let urlConvert = `${templateStart}/convert-and-edit/${ext.view[j]}/${ext.targetext}?${templateEnd}`; + output += ``; + } } for (let j = 0; j < ext.edit.length; ++j) { output += ``; @@ -155,6 +165,10 @@ function discovery(req, res) { output += ``; output += ``; output += ``; + if (-1 === cfgWopiPdfView.indexOf(ext.view[j])) { + let urlConvert = `${templateStart}/convert-and-edit/${ext.view[j]}/${ext.targetext}?${templateEnd}`; + output += ``; + } output += ``; }); } @@ -328,9 +342,8 @@ function getEditorHtml(req, res) { let access_token = req.body['access_token'] || ""; let access_token_ttl = parseInt(req.body['access_token_ttl']) || 0; - let uri = `${encodeURI(wopiSrc)}?access_token=${encodeURIComponent(access_token)}`; - let fileInfo = params.fileInfo = yield checkFileInfo(ctx, uri, access_token, sc); + let fileInfo = params.fileInfo = yield checkFileInfo(ctx, wopiSrc, access_token, sc); if (!fileInfo) { params.fileInfo = {}; return; @@ -394,7 +407,7 @@ function getEditorHtml(req, res) { if (cfgTokenEnableBrowser) { let options = {algorithm: cfgTokenOutboxAlgorithm, expiresIn: cfgTokenOutboxExpires}; - let secret = yield tenantManager.getTenantSecret(ctx, commonDefines.c_oAscSecretType.Inbox); + let secret = yield tenantManager.getTenantSecret(ctx, commonDefines.c_oAscSecretType.Browser); params.token = jwt.sign(params, secret, options); } } catch (err) { @@ -412,6 +425,64 @@ function getEditorHtml(req, res) { } }); } +function getConverterHtml(req, res) { + return co(function*() { + let params = {statusHandler: undefined}; + let ctx = new operationContext.Context(); + try { + ctx.initFromRequest(req); + let wopiSrc = req.query['wopisrc']; + let fileId = wopiSrc.substring(wopiSrc.lastIndexOf('/') + 1); + ctx.setDocId(fileId); + ctx.logger.info('convert-and-edit start'); + + let access_token = req.body['access_token'] || ""; + let access_token_ttl = parseInt(req.body['access_token_ttl']) || 0; + let ext = req.params.ext; + let targetext = req.params.targetext; + + if (!(wopiSrc && access_token && access_token_ttl && ext && targetext)) { + ctx.logger.debug('convert-and-edit invalid params: wopiSrc=%s; access_token=%s; access_token_ttl=%s; ext=%s; targetext=%s', wopiSrc, access_token, access_token_ttl, ext, targetext); + return; + } + + let fileInfo = yield checkFileInfo(ctx, wopiSrc, access_token); + if (!fileInfo) { + ctx.logger.info('convert-and-edit checkFileInfo error'); + return; + } + + let wopiParams = getWopiParams(null, fileInfo, wopiSrc, access_token, access_token_ttl); + + let docId = yield converterService.convertAndEdit(ctx, wopiParams, ext, targetext); + if (docId) { + let baseUrl = cfgWopiHost || utils.getBaseUrlByRequest(req); + params.statusHandler = `${baseUrl}/hosting/wopi/convert-and-edit-handler`; + params.statusHandler += `?wopiSrc=${encodeURI(wopiSrc)}&access_token=${encodeURI(access_token)}`; + params.statusHandler += `&targetext=${encodeURI(targetext)}&docId=${encodeURI(docId)}`; + if (cfgTokenEnableBrowser) { + let tokenData = {docId: docId}; + let options = {algorithm: cfgTokenOutboxAlgorithm, expiresIn: cfgTokenOutboxExpires}; + let secret = yield tenantManager.getTenantSecret(ctx, commonDefines.c_oAscSecretType.Browser); + let token = jwt.sign(tokenData, secret, options); + + params.statusHandler += `&token=${encodeURI(token)}`; + } + } + } catch (err) { + ctx.logger.error('convert-and-edit error:%s', err.stack); + } finally { + ctx.logger.debug('convert-and-edit render params=%j', params); + try { + res.render("convert-and-edit-wopi", params); + } catch (err) { + ctx.logger.error('convert-and-edit error:%s', err.stack); + res.sendStatus(400); + } + ctx.logger.info('convert-and-edit end'); + } + }); +} function putFile(ctx, wopiParams, data, dataStream, dataSize, userLastChangeId, isModifiedByUser, isAutosave, isExitSave) { return co(function* () { let postRes = null; @@ -457,6 +528,34 @@ function putFile(ctx, wopiParams, data, dataStream, dataSize, userLastChangeId, return postRes; }); } +function putRelativeFile(ctx, wopiSrc, access_token, data, dataStream, dataSize, suggestedTarget, isFileConversion) { + return co(function* () { + let postRes = null; + try { + ctx.logger.info('wopi putRelativeFile start'); + + let uri = `${wopiSrc}?access_token=${access_token}`; + let filterStatus = yield checkIpFilter(ctx, uri); + if (0 !== filterStatus) { + return postRes; + } + + let headers = {'X-WOPI-Override': 'PUT_RELATIVE', 'X-WOPI-SuggestedTarget': utf7.encode(suggestedTarget), + 'X-WOPI-FileConversion': isFileConversion}; + fillStandardHeaders(headers, uri, access_token); + + ctx.logger.debug('wopi putRelativeFile request uri=%s headers=%j', uri, headers); + postRes = yield utils.postRequestPromise(uri, data, dataStream, dataSize, cfgCallbackRequestTimeout, undefined, headers); + ctx.logger.debug('wopi putRelativeFile response headers=%j', postRes.response.headers); + ctx.logger.debug('wopi putRelativeFile response body:%s', postRes.body); + } catch (err) { + ctx.logger.error('wopi error putRelativeFile:%s', err.stack); + } finally { + ctx.logger.info('wopi putRelativeFile end'); + } + return postRes; + }); +} function renameFile(ctx, wopiParams, name) { return co(function* () { let res = undefined; @@ -501,18 +600,19 @@ function renameFile(ctx, wopiParams, name) { return res; }); } -function checkFileInfo(ctx, uri, access_token, sc) { +function checkFileInfo(ctx, wopiSrc, access_token, opt_sc) { return co(function* () { let fileInfo = undefined; try { ctx.logger.info('wopi checkFileInfo start'); + let uri = `${encodeURI(wopiSrc)}?access_token=${encodeURIComponent(access_token)}`; let filterStatus = yield checkIpFilter(ctx, uri); if (0 !== filterStatus) { return fileInfo; } let headers = {}; - if (sc) { - headers['X-WOPI-SessionContext'] = sc; + if (opt_sc) { + headers['X-WOPI-SessionContext'] = opt_sc; } fillStandardHeaders(headers, uri, access_token); ctx.logger.debug('wopi checkFileInfo request uri=%s headers=%j', uri, headers); @@ -650,12 +750,22 @@ function checkIpFilter(ctx, uri){ return filterStatus; }); } +function getWopiParams(lockId, fileInfo, wopiSrc, access_token, access_token_ttl) { + let commonInfo = {lockId: lockId, fileInfo: fileInfo}; + let userAuth = { + wopiSrc: wopiSrc, access_token: access_token, access_token_ttl: access_token_ttl, + hostSessionId: null, userSessionId: null, mode: null + }; + return {commonInfo: commonInfo, userAuth: userAuth, LastModifiedTime: null}; +}; exports.discovery = discovery; exports.collaboraCapabilities = collaboraCapabilities; exports.parseWopiCallback = parseWopiCallback; exports.getEditorHtml = getEditorHtml; +exports.getConverterHtml = getConverterHtml; exports.putFile = putFile; +exports.putRelativeFile = putRelativeFile; exports.renameFile = renameFile; exports.lock = lock; exports.unlock = unlock; From 6f1ebe82e89d74bf27ce60a7b6e4095403a68538 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 19 Sep 2022 18:03:38 +0300 Subject: [PATCH 04/45] [bug] Allow custom key in docbuilder service --- DocService/sources/converterservice.js | 29 ++++++++++++++------------ 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/DocService/sources/converterservice.js b/DocService/sources/converterservice.js index 08f07f7a..a7689eeb 100644 --- a/DocService/sources/converterservice.js +++ b/DocService/sources/converterservice.js @@ -397,30 +397,33 @@ function builderRequest(req, res) { let error = authRes.code; let urls; let end = false; - if (error === constants.NO_ERROR && - (params.key || params.url || (req.body && Buffer.isBuffer(req.body) && req.body.length > 0))) { + let needCreateId = !docId; + let isInBody = req.body && Buffer.isBuffer(req.body) && req.body.length > 0; + if (error === constants.NO_ERROR && (params.key || params.url || isInBody)) { + if (needCreateId) { + let task = yield* taskResult.addRandomKeyTask(ctx, undefined, 'bld_', 8); + docId = task.key; + ctx.setDocId(docId); + } let cmd = new commonDefines.InputCommand(); cmd.setCommand('builder'); cmd.setIsBuilder(true); cmd.setWithAuthorization(true); cmd.setDocId(docId); - if (!docId) { - let task = yield* taskResult.addRandomKeyTask(ctx, undefined, 'bld_', 8); - docId = task.key; - cmd.setDocId(docId); - if (params.url) { - cmd.setUrl(params.url); - cmd.setFormat('docbuilder'); - } else { - yield storageBase.putObject(ctx, docId + '/script.docbuilder', req.body, req.body.length); - } + if (params.url) { + cmd.setUrl(params.url); + cmd.setFormat('docbuilder'); + } else if (isInBody) { + yield storageBase.putObject(ctx, docId + '/script.docbuilder', req.body, req.body.length); + } + if (needCreateId) { let queueData = new commonDefines.TaskQueueData(); queueData.setCtx(ctx); queueData.setCmd(cmd); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW); } let async = (typeof params.async === 'string') ? 'true' === params.async : params.async; - let status = yield* convertByCmd(ctx, cmd, async, utils.getBaseUrlByRequest(req), undefined, true); + let status = yield* convertByCmd(ctx, cmd, async, undefined, undefined, constants.QUEUE_PRIORITY_LOW); end = status.end; error = status.err; if (end) { From 1d383ee6dccb8bb516a7e31e068ab653aec2787b Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Sun, 25 Sep 2022 15:52:04 +0300 Subject: [PATCH 05/45] [bug] Fix bug with using built-in Root CAs instead of Windows Store. --- Common/sources/utils.js | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/Common/sources/utils.js b/Common/sources/utils.js index 940d36eb..850aec9e 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -59,7 +59,13 @@ const logger = require('./logger'); const forwarded = require('forwarded'); const { RequestFilteringHttpAgent, RequestFilteringHttpsAgent } = require("request-filtering-agent"); const openpgp = require('openpgp'); -require('win-ca'); +const https = require('https'); +const ca = require('win-ca/api'); + +if(!ca.disabled) { + ca({inject: true}); +} + const contentDisposition = require('content-disposition'); var configIpFilter = config.get('services.CoAuthoring.ipfilter'); @@ -80,7 +86,7 @@ const cfgTokenOutboxUrlExclusionRegex = config.get('services.CoAuthoring.token.o const cfgPasswordEncrypt = config.get('openpgpjs.encrypt'); const cfgPasswordDecrypt = config.get('openpgpjs.decrypt'); const cfgPasswordConfig = config.get('openpgpjs.config'); -const cfgRequesFilteringAgent = config.get('services.CoAuthoring.request-filtering-agent'); +const cfgRequesFilteringAgent = Object.assign({}, https.globalAgent.options, config.get('services.CoAuthoring.request-filtering-agent')); const cfgStorageExternalHost = config.get('storage.externalHost'); Object.assign(openpgp.config, cfgPasswordConfig); @@ -298,6 +304,9 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A var options = {uri: urlParsed, encoding: null, timeout: connectionAndInactivity, followRedirect: false}; if (opt_filterPrivate) { options.agent = getRequestFilterAgent(uri, cfgRequesFilteringAgent); + } else { + //baseRequest creates new agent(win-ca injects in globalAgent) + options.agentOptions = https.globalAgent.options; } if (opt_Authorization) { options.headers = {}; @@ -392,6 +401,8 @@ function postRequestPromise(uri, postData, postDataStream, postDataSize, optTime } let connectionAndInactivity = optTimeout && optTimeout.connectionAndInactivity && ms(optTimeout.connectionAndInactivity); var options = {uri: urlParsed, encoding: 'utf8', headers: headers, timeout: connectionAndInactivity}; + //baseRequest creates new agent(win-ca injects in globalAgent) + options.agentOptions = https.globalAgent.options; if (postData) { options.body = postData; } From 0bd6d5a2573b887cb8e9c3f3dbdc1a607d3d1d81 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Sun, 16 Oct 2022 23:07:19 +0300 Subject: [PATCH 06/45] [feature] Allow outputType convert param --- DocService/sources/converterservice.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DocService/sources/converterservice.js b/DocService/sources/converterservice.js index a7689eeb..ba49079e 100644 --- a/DocService/sources/converterservice.js +++ b/DocService/sources/converterservice.js @@ -241,7 +241,7 @@ function convertRequest(req, res, isJson) { utils.fillResponse(req, res, new commonDefines.ConvertStatus(authRes.code), isJson); return; } - let outputtype = params.outputtype || ''; + let outputtype = params.outputtype || params.outputType || ''; let docId = 'conv_' + params.key + '_' + outputtype; ctx.setDocId(docId); From f2f4a1891693c93cb072424709b8b98a3d0489a5 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 17 Oct 2022 23:31:05 +0300 Subject: [PATCH 07/45] [bug] For bug 59383 --- DocService/sources/DocsCoServer.js | 49 ++++++++++++++++-------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 025b5ae2..f27a4511 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -3635,28 +3635,33 @@ exports.licenseInfo = function(req, res) { var precisionIndex = 0; for (let i = redisRes.length - 1; i >= 0; i--) { let elem = redisRes[i]; - //skip duplicates in cluster - if (prevTime - elem.time >= expDocumentsStep95) { - for (let j = precisionIndex; j < PRECISION.length; ++j) { - if (now - elem.time < PRECISION[j].val) { - let precision = precisionSum[PRECISION[j].name]; - precision.edit.min = Math.min(precision.edit.min, elem.edit); - precision.edit.max = Math.max(precision.edit.max, elem.edit); - precision.edit.sum += elem.edit; - precision.edit.count++; - precision.view.min = Math.min(precision.view.min, elem.view); - precision.view.max = Math.max(precision.view.max, elem.view); - precision.view.sum += elem.view; - precision.view.count++; - if (elem.liveview) { - precision.liveview.min = Math.min(precision.liveview.min, elem.liveview); - precision.liveview.max = Math.max(precision.liveview.max, elem.liveview); - precision.liveview.sum += elem.liveview; - precision.liveview.count++; - } - } else { - precisionIndex = j + 1; - } + let edit = elem.edit || 0; + let view = elem.view || 0; + let liveview = elem.liveview || 0; + //for cluster + while (i > 0 && elem.time - redisRes[i - 1].time < expDocumentsStep95) { + edit += elem.edit || 0; + view += elem.view || 0; + liveview += elem.liveview || 0; + i--; + } + for (let j = precisionIndex; j < PRECISION.length; ++j) { + if (now - elem.time < PRECISION[j].val) { + let precision = precisionSum[PRECISION[j].name]; + precision.edit.min = Math.min(precision.edit.min, edit); + precision.edit.max = Math.max(precision.edit.max, edit); + precision.edit.sum += edit + precision.edit.count++; + precision.view.min = Math.min(precision.view.min, view); + precision.view.max = Math.max(precision.view.max, view); + precision.view.sum += view; + precision.view.count++; + precision.liveview.min = Math.min(precision.liveview.min, liveview); + precision.liveview.max = Math.max(precision.liveview.max, liveview); + precision.liveview.sum += liveview; + precision.liveview.count++; + } else { + precisionIndex = j + 1; } } prevTime = elem.time; From 48d1b32f0623f38ef0a4425cf12fab93dce918a7 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Fri, 21 Oct 2022 13:37:04 +0300 Subject: [PATCH 08/45] [bug] Remove idle timeout for connection with isCloseCoAuthoring --- DocService/sources/DocsCoServer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index f27a4511..f42e1ad3 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -3399,7 +3399,7 @@ exports.install = function(server, callbackFunction) { continue; } } - if (cfgExpSessionIdle > 0 && !conn.user?.view) { + if (cfgExpSessionIdle > 0 && !(conn.user?.view || conn.isCloseCoAuthoring)) { if (maxMs - conn.sessionTimeLastAction > cfgExpSessionIdle && !conn.sessionIsSendWarning) { conn.sessionIsSendWarning = true; sendDataSession(ctx, conn, { From 01a1ac74217f2152fa4e551ae8fd88e79a70539e Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 24 Oct 2022 11:10:22 +0300 Subject: [PATCH 09/45] [feature] Allow both filetype and fileType convert params --- DocService/sources/converterservice.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/DocService/sources/converterservice.js b/DocService/sources/converterservice.js index ba49079e..9207b421 100644 --- a/DocService/sources/converterservice.js +++ b/DocService/sources/converterservice.js @@ -241,6 +241,7 @@ function convertRequest(req, res, isJson) { utils.fillResponse(req, res, new commonDefines.ConvertStatus(authRes.code), isJson); return; } + let filetype = params.filetype || params.fileType || ''; let outputtype = params.outputtype || params.outputType || ''; let docId = 'conv_' + params.key + '_' + outputtype; ctx.setDocId(docId); @@ -250,8 +251,8 @@ function convertRequest(req, res, isJson) { utils.fillResponse(req, res, new commonDefines.ConvertStatus(constants.CONVERT_PARAMS), isJson); return; } - if (params.filetype && !constants.EXTENTION_REGEX.test(params.filetype)) { - ctx.logger.warn('convertRequest unexpected filetype = %s', params.filetype); + if (filetype && !constants.EXTENTION_REGEX.test(filetype)) { + ctx.logger.warn('convertRequest unexpected filetype = %s', filetype); utils.fillResponse(req, res, new commonDefines.ConvertStatus(constants.CONVERT_PARAMS), isJson); return; } @@ -265,7 +266,7 @@ function convertRequest(req, res, isJson) { cmd.setCommand('conv'); cmd.setUrl(params.url); cmd.setEmbeddedFonts(false);//params.embeddedfonts']; - cmd.setFormat(params.filetype); + cmd.setFormat(filetype); cmd.setDocId(docId); cmd.setOutputFormat(outputFormat); From 002613c53a7f64a5af89765a19bb6f3405986505 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Tue, 25 Oct 2022 10:59:07 +0300 Subject: [PATCH 10/45] [feature] Stop outdated save from changes --- Common/sources/commondefines.js | 12 ++++ Common/sources/utils.js | 4 ++ DocService/sources/DocsCoServer.js | 22 ++++---- DocService/sources/canvasservice.js | 76 +++++++++++++------------- DocService/sources/converterservice.js | 18 +++--- DocService/sources/taskresult.js | 18 +----- FileConverter/sources/converter.js | 36 +++++++++--- 7 files changed, 104 insertions(+), 82 deletions(-) diff --git a/Common/sources/commondefines.js b/Common/sources/commondefines.js index 99b09290..0be4e0c1 100644 --- a/Common/sources/commondefines.js +++ b/Common/sources/commondefines.js @@ -1110,6 +1110,17 @@ const c_oAscUnlockRes = { Unlocked: 1, Empty: 2 }; +const FileStatus = { + None: 0, + Ok: 1, + WaitQueue: 2, + NeedParams: 3, + Err: 5, + ErrToReload: 6, + SaveVersion: 7, + UpdateVersion: 8, + NeedPassword: 9 +}; const buildVersion = '4.1.2'; const buildNumber = 37; @@ -1136,5 +1147,6 @@ exports.c_oAscUrlTypes = c_oAscUrlTypes; exports.c_oAscSecretType = c_oAscSecretType; exports.c_oAscQueueType = c_oAscQueueType; exports.c_oAscUnlockRes = c_oAscUnlockRes; +exports.FileStatus = FileStatus; exports.buildVersion = buildVersion; exports.buildNumber = buildNumber; diff --git a/Common/sources/utils.js b/Common/sources/utils.js index 4f0fb135..4cc853b9 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -1039,3 +1039,7 @@ exports.getFunctionArguments = function(func) { join(''). split(/\s*,\s*/); }; +exports.isUselesSfc = function(row, cmd) { + return !(row && commonDefines.FileStatus.SaveVersion === row.status && cmd.getStatusInfoIn() === row.status_info); +}; + diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index f42e1ad3..84919392 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -1127,9 +1127,9 @@ function createSaveTimer(ctx, docId, opt_userId, opt_userIndex, opt_queue, opt_n var updateMask = new taskResult.TaskResultData(); updateMask.tenant = ctx.tenant; updateMask.key = docId; - updateMask.status = taskResult.FileStatus.Ok; + updateMask.status = commonDefines.FileStatus.Ok; var updateTask = new taskResult.TaskResultData(); - updateTask.status = taskResult.FileStatus.SaveVersion; + updateTask.status = commonDefines.FileStatus.SaveVersion; updateTask.statusInfo = utils.getMillisecondsOfHour(new Date()); var updateIfRes = yield taskResult.updateIf(ctx, updateTask, updateMask); if (updateIfRes.affectedRows > 0) { @@ -1540,7 +1540,7 @@ exports.install = function(server, callbackFunction) { let selectRes = yield taskResult.select(ctx, docId); if (selectRes.length > 0) { var row = selectRes[0]; - if (taskResult.FileStatus.UpdateVersion === row.status) { + if (commonDefines.FileStatus.UpdateVersion === row.status) { needSendStatus = false; } } @@ -2332,16 +2332,16 @@ exports.install = function(server, callbackFunction) { } if (!conn.user.view) { var status = result && result.length > 0 ? result[0]['status'] : null; - if (taskResult.FileStatus.Ok === status) { + if (commonDefines.FileStatus.Ok === status) { // Все хорошо, статус обновлять не нужно - } else if (taskResult.FileStatus.SaveVersion === status || - (!bIsRestore && taskResult.FileStatus.UpdateVersion === status && + } else if (commonDefines.FileStatus.SaveVersion === status || + (!bIsRestore && commonDefines.FileStatus.UpdateVersion === status && Date.now() - result[0]['status_info'] * 60000 > cfgExpUpdateVersionStatus)) { - let newStatus = taskResult.FileStatus.Ok; - if (taskResult.FileStatus.UpdateVersion === status) { + let newStatus = commonDefines.FileStatus.Ok; + if (commonDefines.FileStatus.UpdateVersion === status) { ctx.logger.warn("UpdateVersion expired"); //FileStatus.None to open file again from new url - newStatus = taskResult.FileStatus.None; + newStatus = commonDefines.FileStatus.None; } // Обновим статус файла (идет сборка, нужно ее остановить) var updateMask = new taskResult.TaskResultData(); @@ -2358,11 +2358,11 @@ exports.install = function(server, callbackFunction) { yield* sendFileErrorAuth(ctx, conn, data.sessionId, 'Update Version error', constants.UPDATE_VERSION_CODE); return; } - } else if (bIsRestore && taskResult.FileStatus.UpdateVersion === status) { + } else if (bIsRestore && commonDefines.FileStatus.UpdateVersion === status) { // error version yield* sendFileErrorAuth(ctx, conn, data.sessionId, 'Update Version error', constants.UPDATE_VERSION_CODE); return; - } else if (taskResult.FileStatus.None === status && conn.encrypted) { + } else if (commonDefines.FileStatus.None === status && conn.encrypted) { //ok } else if (bIsRestore) { // Other error diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index b848032e..c36839e9 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -182,18 +182,18 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd openedAt = getOpenedAt(row); } switch (status) { - case taskResult.FileStatus.SaveVersion: - case taskResult.FileStatus.UpdateVersion: - case taskResult.FileStatus.Ok: - if(taskResult.FileStatus.Ok == status) { + case commonDefines.FileStatus.SaveVersion: + case commonDefines.FileStatus.UpdateVersion: + case commonDefines.FileStatus.Ok: + if(commonDefines.FileStatus.Ok == status) { outputData.setStatus('ok'); - } else if (taskResult.FileStatus.SaveVersion == status || - (!opt_bIsRestore && taskResult.FileStatus.UpdateVersion === status && + } else if (commonDefines.FileStatus.SaveVersion == status || + (!opt_bIsRestore && commonDefines.FileStatus.UpdateVersion === status && Date.now() - statusInfo * 60000 > cfgExpUpdateVersionStatus)) { if (optConn && (optConn.user.view || optConn.isCloseCoAuthoring)) { outputData.setStatus(constants.FILE_STATUS_UPDATE_VERSION); } else { - if (taskResult.FileStatus.UpdateVersion === status) { + if (commonDefines.FileStatus.UpdateVersion === status) { ctx.logger.warn("UpdateVersion expired"); } var updateMask = new taskResult.TaskResultData(); @@ -202,7 +202,7 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd updateMask.status = status; updateMask.statusInfo = statusInfo; var updateTask = new taskResult.TaskResultData(); - updateTask.status = taskResult.FileStatus.Ok; + updateTask.status = commonDefines.FileStatus.Ok; updateTask.statusInfo = constants.NO_ERROR; var updateIfRes = yield taskResult.updateIf(ctx, updateTask, updateMask); if (updateIfRes.affectedRows > 0) { @@ -264,7 +264,7 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd } } break; - case taskResult.FileStatus.NeedParams: + case commonDefines.FileStatus.NeedParams: outputData.setStatus('needparams'); var settingsPath = key + '/' + 'origin.' + cmd.getFormat(); if (optConn) { @@ -276,15 +276,15 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd optAdditionalOutput.needUrlType = commonDefines.c_oAscUrlTypes.Temporary; } break; - case taskResult.FileStatus.NeedPassword: + case commonDefines.FileStatus.NeedPassword: outputData.setStatus('needpassword'); outputData.setData(statusInfo); break; - case taskResult.FileStatus.Err: - case taskResult.FileStatus.ErrToReload: + case commonDefines.FileStatus.Err: + case commonDefines.FileStatus.ErrToReload: outputData.setStatus('err'); outputData.setData(statusInfo); - if (taskResult.FileStatus.ErrToReload == status) { + if (commonDefines.FileStatus.ErrToReload == status) { let userAuthStr = sqlBase.UserCallback.prototype.getCallbackByUserIndex(ctx, row.callback); let wopiParams = wopiClient.parseWopiCallback(ctx, userAuthStr); if (!wopiParams) { @@ -293,10 +293,10 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd } } break; - case taskResult.FileStatus.None: + case commonDefines.FileStatus.None: outputData.setStatus('none'); break; - case taskResult.FileStatus.WaitQueue: + case commonDefines.FileStatus.WaitQueue: //task in the queue. response will be after convertion break; default: @@ -378,7 +378,7 @@ function* getUpdateResponse(ctx, cmd) { updateTask.key = cmd.getSaveKey() ? cmd.getSaveKey() : cmd.getDocId(); var statusInfo = cmd.getStatusInfo(); if (constants.NO_ERROR == statusInfo) { - updateTask.status = taskResult.FileStatus.Ok; + updateTask.status = commonDefines.FileStatus.Ok; let password = cmd.getPassword(); if (password) { if (false === hasPasswordCol) { @@ -390,21 +390,21 @@ function* getUpdateResponse(ctx, cmd) { } } } else if (constants.CONVERT_DOWNLOAD == statusInfo) { - updateTask.status = taskResult.FileStatus.ErrToReload; + updateTask.status = commonDefines.FileStatus.ErrToReload; } else if (constants.CONVERT_NEED_PARAMS == statusInfo) { - updateTask.status = taskResult.FileStatus.NeedParams; + updateTask.status = commonDefines.FileStatus.NeedParams; } else if (constants.CONVERT_DRM == statusInfo || constants.CONVERT_PASSWORD == statusInfo) { if (cfgOpenProtectedFile) { - updateTask.status = taskResult.FileStatus.NeedPassword; + updateTask.status = commonDefines.FileStatus.NeedPassword; } else { - updateTask.status = taskResult.FileStatus.Err; + updateTask.status = commonDefines.FileStatus.Err; } } else if (constants.CONVERT_DRM_UNSUPPORTED == statusInfo) { - updateTask.status = taskResult.FileStatus.Err; + updateTask.status = commonDefines.FileStatus.Err; } else if (constants.CONVERT_DEAD_LETTER == statusInfo) { - updateTask.status = taskResult.FileStatus.ErrToReload; + updateTask.status = commonDefines.FileStatus.ErrToReload; } else { - updateTask.status = taskResult.FileStatus.Err; + updateTask.status = commonDefines.FileStatus.Err; } updateTask.statusInfo = statusInfo; return updateTask; @@ -443,7 +443,7 @@ function commandOpenStartPromise(ctx, docId, baseUrl, opt_updateUserIndex, opt_d task.tenant = ctx.tenant; task.key = docId; //None instead WaitQueue to prevent: conversion task is lost when entering and leaving the editor quickly(that leads to an endless opening) - task.status = taskResult.FileStatus.None; + task.status = commonDefines.FileStatus.None; task.statusInfo = constants.NO_ERROR; task.baseurl = baseUrl; if (opt_documentCallbackUrl) { @@ -478,10 +478,10 @@ function* commandOpen(ctx, conn, cmd, outputData, opt_upsertRes, opt_bIsRestore) let updateMask = new taskResult.TaskResultData(); updateMask.tenant = ctx.tenant; updateMask.key = cmd.getDocId(); - updateMask.status = taskResult.FileStatus.None; + updateMask.status = commonDefines.FileStatus.None; let task = new taskResult.TaskResultData(); - task.status = taskResult.FileStatus.WaitQueue; + task.status = commonDefines.FileStatus.WaitQueue; task.statusInfo = constants.NO_ERROR; let updateIfRes = yield taskResult.updateIf(ctx, task, updateMask); @@ -537,10 +537,10 @@ function* commandReopen(ctx, conn, cmd, outputData) { let updateMask = new taskResult.TaskResultData(); updateMask.tenant = ctx.tenant; updateMask.key = cmd.getDocId(); - updateMask.status = isPassword ? taskResult.FileStatus.NeedPassword : taskResult.FileStatus.NeedParams; + updateMask.status = isPassword ? commonDefines.FileStatus.NeedPassword : commonDefines.FileStatus.NeedParams; var task = new taskResult.TaskResultData(); - task.status = taskResult.FileStatus.WaitQueue; + task.status = commonDefines.FileStatus.WaitQueue; task.statusInfo = constants.NO_ERROR; var upsertRes = yield taskResult.updateIf(ctx, task, updateMask); @@ -821,7 +821,7 @@ function* commandSetPassword(ctx, conn, cmd, outputData) { if (selectRes.length > 0) { let row = selectRes[0]; hasPasswordCol = undefined !== row.password; - if (taskResult.FileStatus.Ok === row.status && sqlBase.DocumentPassword.prototype.getCurPassword(ctx, row.password)) { + if (commonDefines.FileStatus.Ok === row.status && sqlBase.DocumentPassword.prototype.getCurPassword(ctx, row.password)) { hasDocumentPassword = true; } } @@ -830,7 +830,7 @@ function* commandSetPassword(ctx, conn, cmd, outputData) { let updateMask = new taskResult.TaskResultData(); updateMask.tenant = ctx.tenant; updateMask.key = cmd.getDocId(); - updateMask.status = taskResult.FileStatus.Ok; + updateMask.status = commonDefines.FileStatus.Ok; let newChangesLastDate = new Date(); newChangesLastDate.setMilliseconds(0);//remove milliseconds avoid issues with MySQL datetime rounding @@ -924,10 +924,10 @@ function* commandSfcCallback(ctx, cmd, isSfcm, isEncrypted) { statusErr = docsCoServer.c_oAscServerStatus.Corrupted; } let recoverTask = new taskResult.TaskResultData(); - recoverTask.status = taskResult.FileStatus.Ok; + recoverTask.status = commonDefines.FileStatus.Ok; recoverTask.statusInfo = constants.NO_ERROR; let updateIfTask = new taskResult.TaskResultData(); - updateIfTask.status = taskResult.FileStatus.UpdateVersion; + updateIfTask.status = commonDefines.FileStatus.UpdateVersion; updateIfTask.statusInfo = Math.floor(Date.now() / 60000);//minutes let updateIfRes; @@ -938,12 +938,12 @@ function* commandSfcCallback(ctx, cmd, isSfcm, isEncrypted) { if (isEncrypted) { recoverTask.status = updateMask.status = row.status; recoverTask.statusInfo = updateMask.statusInfo = row.status_info; - } else if ((taskResult.FileStatus.SaveVersion === row.status && cmd.getStatusInfoIn() === row.status_info) || - taskResult.FileStatus.UpdateVersion === row.status) { - if (taskResult.FileStatus.UpdateVersion === row.status) { + } else if ((commonDefines.FileStatus.SaveVersion === row.status && cmd.getStatusInfoIn() === row.status_info) || + commonDefines.FileStatus.UpdateVersion === row.status) { + if (commonDefines.FileStatus.UpdateVersion === row.status) { updateIfRes = {affectedRows: 1}; } - recoverTask.status = taskResult.FileStatus.SaveVersion; + recoverTask.status = commonDefines.FileStatus.SaveVersion; recoverTask.statusInfo = cmd.getStatusInfoIn(); updateMask.status = row.status; updateMask.statusInfo = row.status_info; @@ -1017,7 +1017,7 @@ function* commandSfcCallback(ctx, cmd, isSfcm, isEncrypted) { let selectRes = yield taskResult.select(ctx, docId); let row = selectRes.length > 0 ? selectRes[0] : null; //send only if FileStatus.Ok to prevent forcesave after final save - if (row && row.status == taskResult.FileStatus.Ok) { + if (row && row.status == commonDefines.FileStatus.Ok) { if (forceSave) { let forceSaveDate = forceSave.getTime() ? new Date(forceSave.getTime()): new Date(); outputSfc.setForceSaveType(forceSaveType); @@ -1589,7 +1589,7 @@ exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId //делаем select, потому что за время timeout информация могла измениться var selectRes = yield taskResult.select(ctx, docId); var row = selectRes.length > 0 ? selectRes[0] : null; - if (row && row.status == taskResult.FileStatus.SaveVersion && row.status_info == statusInfo) { + if (row && row.status == commonDefines.FileStatus.SaveVersion && row.status_info == statusInfo) { if (null == optFormat) { optFormat = changeFormatByOrigin(ctx, row, constants.AVS_OFFICESTUDIO_FILE_OTHER_OOXML); } diff --git a/DocService/sources/converterservice.js b/DocService/sources/converterservice.js index 9207b421..7bdedeba 100644 --- a/DocService/sources/converterservice.js +++ b/DocService/sources/converterservice.js @@ -63,7 +63,7 @@ function* getConvertStatus(ctx, docId, encryptedUserPassword, selectRes, opt_che var row = selectRes[0]; let password = opt_checkPassword && sqlBase.DocumentPassword.prototype.getCurPassword(ctx, row.password); switch (row.status) { - case taskResult.FileStatus.Ok: + case commonDefines.FileStatus.Ok: if (password) { let isCorrectPassword; if (encryptedUserPassword) { @@ -82,17 +82,17 @@ function* getConvertStatus(ctx, docId, encryptedUserPassword, selectRes, opt_che status.end = true; } break; - case taskResult.FileStatus.Err: - case taskResult.FileStatus.ErrToReload: - case taskResult.FileStatus.NeedPassword: + case commonDefines.FileStatus.Err: + case commonDefines.FileStatus.ErrToReload: + case commonDefines.FileStatus.NeedPassword: status.err = row.status_info; - if (taskResult.FileStatus.ErrToReload == row.status || taskResult.FileStatus.NeedPassword == row.status) { + if (commonDefines.FileStatus.ErrToReload == row.status || commonDefines.FileStatus.NeedPassword == row.status) { yield canvasService.cleanupCache(ctx); } break; - case taskResult.FileStatus.NeedParams: - case taskResult.FileStatus.SaveVersion: - case taskResult.FileStatus.UpdateVersion: + case commonDefines.FileStatus.NeedParams: + case commonDefines.FileStatus.SaveVersion: + case commonDefines.FileStatus.UpdateVersion: status.err = constants.UNKNOWN; break; } @@ -136,7 +136,7 @@ function* convertByCmd(ctx, cmd, async, opt_fileTo, opt_taskExist, opt_priority, let task = new taskResult.TaskResultData(); task.tenant = ctx.tenant; task.key = docId; - task.status = taskResult.FileStatus.WaitQueue; + task.status = commonDefines.FileStatus.WaitQueue; task.statusInfo = constants.NO_ERROR; let upsertRes = yield taskResult.upsert(ctx, task); diff --git a/DocService/sources/taskresult.js b/DocService/sources/taskresult.js index a1d657ea..699ecd45 100644 --- a/DocService/sources/taskresult.js +++ b/DocService/sources/taskresult.js @@ -36,6 +36,7 @@ const crypto = require('crypto'); var sqlBase = require('./baseConnector'); var utils = require('./../../Common/sources/utils'); var constants = require('./../../Common/sources/constants'); +var commonDefines = require('./../../Common/sources/commondefines'); var tenantManager = require('./../../Common/sources/tenantManager'); let addSqlParam = sqlBase.baseConnector.addSqlParameter; @@ -43,18 +44,6 @@ let concatParams = sqlBase.baseConnector.concatParams; var RANDOM_KEY_MAX = 10000; -var FileStatus = { - None: 0, - Ok: 1, - WaitQueue: 2, - NeedParams: 3, - Err: 5, - ErrToReload: 6, - SaveVersion: 7, - UpdateVersion: 8, - NeedPassword: 9 -}; - function TaskResultData() { this.tenant = null; this.key = null; @@ -79,7 +68,7 @@ TaskResultData.prototype.completeDefaults = function() { this.key = ''; } if (!this.status) { - this.status = FileStatus.None; + this.status = commonDefines.FileStatus.None; } if (!this.statusInfo) { this.statusInfo = constants.NO_ERROR; @@ -267,7 +256,7 @@ function* addRandomKeyTask(ctx, key, opt_prefix, opt_size) { var task = new TaskResultData(); task.tenant = ctx.tenant; task.key = key; - task.status = FileStatus.WaitQueue; + task.status = commonDefines.FileStatus.WaitQueue; //nTryCount чтобы не зависнуть если реально будут проблемы с DB var nTryCount = RANDOM_KEY_MAX; var addRes = null; @@ -340,7 +329,6 @@ function getExpired(ctx, maxCount, expireSeconds) { }); } -exports.FileStatus = FileStatus; exports.TaskResultData = TaskResultData; exports.upsert = upsert; exports.select = select; diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index 0cf54ff7..95d231b3 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -50,6 +50,7 @@ var logger = require('./../../Common/sources/logger'); var constants = require('./../../Common/sources/constants'); var baseConnector = require('./../../DocService/sources/baseConnector'); const wopiClient = require('./../../DocService/sources/wopiClient'); +const taskResult = require('./../../DocService/sources/taskresult'); var statsDClient = require('./../../Common/sources/statsdclient'); var queueService = require('./../../Common/sources/taskqueueRabbitMQ'); const formatChecker = require('./../../Common/sources/formatchecker'); @@ -281,6 +282,17 @@ function getTempDir() { fs.mkdirSync(resultDir); return {temp: newTemp, source: sourceDir, result: resultDir}; } +function* isUselessConvertion(ctx, task, cmd) { + if (task.getFromChanges()) { + let selectRes = yield taskResult.select(ctx, cmd.getDocId()); + let row = selectRes.length > 0 ? selectRes[0] : null; + if (utils.isUselesSfc(row, cmd)) { + ctx.logger.warn('isUselessConvertion return true. row=%j', row); + return constants.CONVERT_PARAMS; + } + } + return constants.NO_ERROR; +} function* replaceEmptyFile(ctx, fileFrom, ext, _lcid) { if (!fs.existsSync(fileFrom) || 0 === fs.lstatSync(fileFrom).size) { let locale = 'en-US'; @@ -404,7 +416,7 @@ function* processDownloadFromStorage(ctx, dataConvert, cmd, task, tempDirs, auth yield* concatFiles(tempDirs.source); } if (task.getFromChanges()) { - res = yield* processChanges(ctx, tempDirs, cmd, authorProps); + res = yield* processChanges(ctx, tempDirs, task, cmd, authorProps); } //todo rework if (!fs.existsSync(dataConvert.fileFrom)) { @@ -447,7 +459,7 @@ function* concatFiles(source) { } } -function* processChanges(ctx, tempDirs, cmd, authorProps) { +function* processChanges(ctx, tempDirs, task, cmd, authorProps) { let res = constants.NO_ERROR; let changesDir = path.join(tempDirs.source, constants.CHANGES_NAME); fs.mkdirSync(changesDir); @@ -483,6 +495,15 @@ function* processChanges(ctx, tempDirs, cmd, authorProps) { let changes = []; if (curIndexStart < curIndexEnd) { changes = yield baseConnector.getChangesPromise(ctx, cmd.getDocId(), curIndexStart, curIndexEnd, forceSaveTime); + if (changes.length > 0 && changes[0].change_data.startsWith('ENCRYPTED;')) { + ctx.logger.warn('processChanges encrypted changes'); + //todo sql request instead? + res = constants.EDITOR_CHANGES; + } + res = yield* isUselessConvertion(ctx, task, cmd); + if (constants.NO_ERROR !== res) { + break; + } } if (0 === changes.length && extChanges) { changes = extChanges; @@ -490,12 +511,6 @@ function* processChanges(ctx, tempDirs, cmd, authorProps) { extChanges = undefined; for (let i = 0; i < changes.length; ++i) { let change = changes[i]; - if (change.change_data.startsWith('ENCRYPTED;')) { - ctx.logger.warn('processChanges encrypted changes'); - //todo sql request instead? - res = constants.EDITOR_CHANGES; - break; - } if (null === changesAuthor || changesAuthor !== change.user_id_original) { if (null !== changesAuthor) { yield* streamEnd(streamObj, ']'); @@ -744,7 +759,10 @@ function* ExecuteTask(ctx, task) { dataConvert.fileTo = fileTo ? path.join(tempDirs.result, fileTo) : ''; let isBuilder = cmd.getIsBuilder(); let authorProps = {lastModifiedBy: null, modified: null}; - if (cmd.getUrl()) { + error = yield* isUselessConvertion(ctx, task, cmd); + if (constants.NO_ERROR !== error) { + ; + } else if (cmd.getUrl()) { let format = cmd.getFormat(); dataConvert.fileFrom = path.join(tempDirs.source, dataConvert.key + '.' + format); if (utils.checkPathTraversal(ctx, dataConvert.key, tempDirs.source, dataConvert.fileFrom)) { From 09fe41af7572644e7019ab925adbb8019f53e694 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 12 Sep 2022 12:04:36 +0300 Subject: [PATCH 11/45] [feature] Add socket.io module --- Common/config/default.json | 9 ++ Common/sources/tenantManager.js | 2 +- Common/sources/utils.js | 1 + DocService/npm-shrinkwrap.json | 138 +++++++++++++++++++++++++++++ DocService/package.json | 1 + DocService/sources/DocsCoServer.js | 67 +++++++++----- 6 files changed, 196 insertions(+), 22 deletions(-) diff --git a/Common/config/default.json b/Common/config/default.json index d3600d29..5a4abca0 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -268,6 +268,15 @@ "disable_cors": true, "websocket": true }, + "socketio": { + "connection": { + "path": "/doc/socket.io/", + "serveClient": false, + "pingTimeout": 20000, + "pingInterval": 25000, + "maxHttpBufferSize": 1e8 + } + }, "callbackBackoffOptions": { "retries": 0, "timeout":{ diff --git a/Common/sources/tenantManager.js b/Common/sources/tenantManager.js index 66dbb967..2317cbc8 100644 --- a/Common/sources/tenantManager.js +++ b/Common/sources/tenantManager.js @@ -77,7 +77,7 @@ function getTenant(ctx, domain) { return tenant; } function getTenantByConnection(ctx, conn) { - return isMultitenantMode() ? getTenant(ctx, utils.getDomainByConnection(ctx, conn)) : getDefautTenant(); + return isMultitenantMode() ? getTenant(ctx, utils.getDomainByConnection(ctx, conn.request)) : getDefautTenant(); } function getTenantByRequest(ctx, req) { return isMultitenantMode() ? getTenant(ctx, utils.getDomainByRequest(ctx, req)) : getDefautTenant(); diff --git a/Common/sources/utils.js b/Common/sources/utils.js index 4cc853b9..8c34b9fd 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -684,6 +684,7 @@ function getBaseUrl(protocol, hostHeader, forwardedProtoHeader, forwardedHostHea return url; } function getBaseUrlByConnection(conn) { + conn = conn.request; return getBaseUrl('', conn.headers['host'], conn.headers['x-forwarded-proto'], conn.headers['x-forwarded-host'], conn.headers['x-forwarded-prefix']); } function getBaseUrlByRequest(req) { diff --git a/DocService/npm-shrinkwrap.json b/DocService/npm-shrinkwrap.json index 08430ea8..4487f96f 100644 --- a/DocService/npm-shrinkwrap.json +++ b/DocService/npm-shrinkwrap.json @@ -4,6 +4,26 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@socket.io/component-emitter": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@socket.io/component-emitter/-/component-emitter-3.1.0.tgz", + "integrity": "sha512-+9jVqKhRSpsc591z5vX+X5Yyw+he/HCB4iQ/RYxw35CEPaY1gnsNE43nf9n9AaYjAQrTiI/mOwKUKdUs9vf7Xg==" + }, + "@types/cookie": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.4.1.tgz", + "integrity": "sha512-XW/Aa8APYr6jSVVA1y/DEIZX0/GMKLEVekNG727R8cs56ahETkRAy/3DR7+fJyh7oUgGwNQaRfXCun0+KbWY7Q==" + }, + "@types/cors": { + "version": "2.8.12", + "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz", + "integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==" + }, + "@types/node": { + "version": "18.7.16", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.16.tgz", + "integrity": "sha512-EQHhixfu+mkqHMZl1R2Ovuvn47PUw18azMJOTwSZr9/fhzHNGXAJ0ma0dayRVchprpCj0Kc1K1xKoWaATWF1qg==" + }, "accepts": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", @@ -59,6 +79,11 @@ "resolved": "https://registry.npmjs.org/base64-stream/-/base64-stream-1.0.0.tgz", "integrity": "sha512-BQQZftaO48FcE1Kof9CmXMFaAdqkcNorgc8CxesZv9nMbbTF1EFyQe89UOuh//QMmdtfUDXyO8rgUalemL5ODA==" }, + "base64id": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/base64id/-/base64id-2.0.0.tgz", + "integrity": "sha512-lGe34o6EHj9y3Kts9R4ZYs/Gr+6N7MCaMlIFA3F1R2O5/m7K06AxfSeO5530PEERE6/WyEg3lsuyw4GHlPZHog==" + }, "body-parser": { "version": "1.18.3", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz", @@ -187,6 +212,15 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, + "cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "requires": { + "object-assign": "^4", + "vary": "^1" + } + }, "cron": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/cron/-/cron-1.5.0.tgz", @@ -290,6 +324,48 @@ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" }, + "engine.io": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.2.0.tgz", + "integrity": "sha512-4KzwW3F3bk+KlzSOY57fj/Jx6LyRQ1nbcyIadehl+AnXjKT7gDO0ORdRi/84ixvMKTym6ZKuxvbzN62HDDU1Lg==", + "requires": { + "@types/cookie": "^0.4.1", + "@types/cors": "^2.8.12", + "@types/node": ">=10.0.0", + "accepts": "~1.3.4", + "base64id": "2.0.0", + "cookie": "~0.4.1", + "cors": "~2.8.5", + "debug": "~4.3.1", + "engine.io-parser": "~5.0.3", + "ws": "~8.2.3" + }, + "dependencies": { + "cookie": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz", + "integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==" + }, + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, + "engine.io-parser": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.0.4.tgz", + "integrity": "sha512-+nVFp+5z1E3HcToEnO7ZIj3g+3k9389DvWtvJZz0T6/eOCPIyyxehFcedoYrZQrp0LgQbD9pPXhpMBKMd5QURg==" + }, "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", @@ -1115,6 +1191,63 @@ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==" }, + "socket.io": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.5.2.tgz", + "integrity": "sha512-6fCnk4ARMPZN448+SQcnn1u8OHUC72puJcNtSgg2xS34Cu7br1gQ09YKkO1PFfDn/wyUE9ZgMAwosJed003+NQ==", + "requires": { + "accepts": "~1.3.4", + "base64id": "~2.0.0", + "debug": "~4.3.2", + "engine.io": "~6.2.0", + "socket.io-adapter": "~2.4.0", + "socket.io-parser": "~4.2.0" + }, + "dependencies": { + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, + "socket.io-adapter": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-2.4.0.tgz", + "integrity": "sha512-W4N+o69rkMEGVuk2D/cvca3uYsvGlMwsySWV447y99gUPghxq42BxqLNMndb+a1mm/5/7NeXVQS7RLa2XyXvYg==" + }, + "socket.io-parser": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.1.tgz", + "integrity": "sha512-V4GrkLy+HeF1F/en3SpUaM+7XxYXpuMUWLGde1kSSh5nQMN4hLrbPIkD+otwh6q9R6NOQBN4AMaOZ2zVjui82g==", + "requires": { + "@socket.io/component-emitter": "~3.1.0", + "debug": "~4.3.1" + }, + "dependencies": { + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "sockjs": { "version": "0.3.21", "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.21.tgz", @@ -1256,6 +1389,11 @@ "resolved": "https://registry.npmjs.org/windows-locale/-/windows-locale-1.0.1.tgz", "integrity": "sha512-X8B22Cg9njwV4h3C5j28xmZ2eWaO69j63WhReeglB69LOT3LoqSO4Vb6TTVSfFikh4KQ9qBOJb6+WvR4tVLTfQ==" }, + "ws": { + "version": "8.2.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.2.3.tgz", + "integrity": "sha512-wBuoj1BDpC6ZQ1B7DWQBYVLphPWkm8i9Y0/3YdHjHKHiohOJ1ws+3OccDWtH+PoC9DZD5WOTrJvNbWvjS6JWaA==" + }, "xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", diff --git a/DocService/package.json b/DocService/package.json index f3694edd..c2fb20a8 100644 --- a/DocService/package.json +++ b/DocService/package.json @@ -33,6 +33,7 @@ "pg": "^8.5.1", "redis": "^2.8.0", "retry": "^0.12.0", + "socket.io": "^4.5.2", "sockjs": "^0.3.21", "underscore": "^1.13.1", "utf7": "^1.0.2", diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 84919392..9e1175c4 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -73,6 +73,7 @@ 'use strict'; const sockjs = require('sockjs'); +const { Server } = require("socket.io"); const _ = require('underscore'); const url = require('url'); const os = require('os'); @@ -150,6 +151,7 @@ const cfgErrorFiles = configCommon.get('FileConverter.converter.errorfiles'); const cfgOpenProtectedFile = config.get('server.openProtectedFile'); const cfgRefreshLockInterval = ms(configCommon.get('wopi.refreshLockInterval')); const cfgTokenRequiredParams = config.get('server.tokenRequiredParams'); +const cfgSocketIoConnection = configCommon.get('services.CoAuthoring.socketio.connection'); const EditorTypes = { document : 0, @@ -1308,9 +1310,31 @@ function encryptPasswordParams(ctx, data) { } exports.encryptPasswordParams = encryptPasswordParams; exports.install = function(server, callbackFunction) { - var sockjs_echo = sockjs.createServer(cfgSockjs); + const io = new Server(server, cfgSocketIoConnection); - sockjs_echo.on('connection', function(conn) { + io.use((socket, next) => { + co(function*(){ + let ctx = new operationContext.Context(); + let checkJwtRes; + try { + ctx.initFromConnection(socket); + ctx.logger.info('io.use start'); + let handshake = socket.handshake; + if (cfgTokenEnableBrowser) { + checkJwtRes = yield checkJwt(ctx, handshake?.auth?.token, commonDefines.c_oAscSecretType.Browser); + } + } catch (err) { + ctx.logger.info('io.use error: %s', err.stack); + } finally { + ctx.logger.info('io.use end'); + next(checkJwtRes.decoded ? undefined : new Error("not authorized")); + } + + + }); + }); + + io.on('connection', function(conn) { if (!conn) { operationContext.global.logger.error("null == conn"); return; @@ -1325,7 +1349,7 @@ exports.install = function(server, callbackFunction) { conn.sessionIsSendWarning = false; conn.sessionTimeConnect = conn.sessionTimeLastAction = new Date().getTime(); - conn.on('data', function(message) { + conn.on('message', function(data) { return co(function* () { var docId = 'null'; let ctx = new operationContext.Context(); @@ -1336,7 +1360,6 @@ exports.install = function(server, callbackFunction) { startDate = new Date(); } - var data = JSON.parse(message); docId = conn.docId; ctx.logger.info('data.type = %s', data.type); if(getIsShutdown()) @@ -1392,7 +1415,7 @@ exports.install = function(server, callbackFunction) { yield* checkEndAuthLock(ctx, data.unlock, data.isSave, docId, conn.user.id, data.releaseLocks, data.deleteIndex, conn); break; case 'close': - yield* closeDocument(ctx, conn, false); + yield* closeDocument(ctx, conn); break; case 'versionHistory' : { let cmd = new commonDefines.InputCommand(data.cmd); @@ -1433,7 +1456,7 @@ exports.install = function(server, callbackFunction) { delete conn.authChangesAck; break; default: - ctx.logger.debug("unknown command %s", message); + ctx.logger.debug("unknown command %d", data); break; } if(clientStatsD) { @@ -1446,17 +1469,12 @@ exports.install = function(server, callbackFunction) { } }); }); - conn.on('error', function() { - let ctx = new operationContext.Context(); - ctx.initFromConnection(conn); - ctx.logger.error("On error"); - }); - conn.on('close', function() { + conn.on("disconnect", function(reason) { return co(function* () { let ctx = new operationContext.Context(); try { ctx.initFromConnection(conn); - yield* closeDocument(ctx, conn, true); + yield* closeDocument(ctx, conn, reason); } catch (err) { ctx.logger.error('Error conn close: %s', err.stack); } @@ -1465,12 +1483,19 @@ exports.install = function(server, callbackFunction) { _checkLicense(ctx, conn); }); + io.engine.on("connection_error", (err) => { + console.log(err.req); // the request object + console.log(err.code); // the error code, for example 1 + console.log(err.message); // the error message, for example "Session ID unknown" + console.log(err.context); // some additional error context + }); /** * + * @param ctx * @param conn - * @param isCloseConnection - закрываем ли мы окончательно соединение + * @param reason - the reason of the disconnection (either client or server-side) */ - function* closeDocument(ctx, conn, isCloseConnection) { + function* closeDocument(ctx, conn, reason) { var userLocks, reconnected = false, bHasEditors, bHasChanges; var docId = conn.docId; if (null == docId) { @@ -1480,9 +1505,9 @@ exports.install = function(server, callbackFunction) { let participantsTimestamp; var tmpUser = conn.user; var isView = tmpUser.view; - ctx.logger.info("Connection closed or timed out: isCloseConnection = %s", isCloseConnection); + ctx.logger.info("Connection closed or timed out: reason = %s", reason); var isCloseCoAuthoringTmp = conn.isCloseCoAuthoring; - if (isCloseConnection) { + if (reason) { //Notify that participant has gone connections = _.reject(connections, function(el) { return el.id === conn.id;//Delete this connection @@ -3145,10 +3170,10 @@ exports.install = function(server, callbackFunction) { return licenseType; } - sockjs_echo.installHandlers(server, {prefix: '/doc/['+constants.DOC_ID_PATTERN+']*/c', log: function(severity, message) { - //TODO: handle severity - operationContext.global.logger.info(message); - }}); + // sockjs_echo.installHandlers(server, {prefix: '/doc/['+constants.DOC_ID_PATTERN+']*/c', log: function(severity, message) { + // //TODO: handle severity + // operationContext.global.logger.info(message); + // }}); //publish subscribe message brocker function pubsubOnMessage(msg) { From 2f124fe07fc16503af0a803aad2922e8eae06ec9 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Sun, 18 Sep 2022 18:03:53 +0300 Subject: [PATCH 12/45] [socket.io] Move from sockjs built-in properties to socketio --- Common/config/default.json | 2 +- Common/sources/constants.js | 2 +- Common/sources/operationContext.js | 3 ++- Common/sources/tenantManager.js | 2 +- Common/sources/utils.js | 5 ++-- DocService/sources/DocsCoServer.js | 42 +++++++++++++++--------------- 6 files changed, 29 insertions(+), 27 deletions(-) diff --git a/Common/config/default.json b/Common/config/default.json index 5a4abca0..21883ad3 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -270,7 +270,7 @@ }, "socketio": { "connection": { - "path": "/doc/socket.io/", + "path": "/doc/", "serveClient": false, "pingTimeout": 20000, "pingInterval": 25000, diff --git a/Common/sources/constants.js b/Common/sources/constants.js index 65d4c6f3..17c5f889 100644 --- a/Common/sources/constants.js +++ b/Common/sources/constants.js @@ -295,7 +295,7 @@ exports.RESTORE = 'no cache'; exports.CONTENT_DISPOSITION_INLINE = 'inline'; exports.CONTENT_DISPOSITION_ATTACHMENT = 'attachment'; -exports.CONN_CLOSED = 3; +exports.CONN_CLOSED = "closed"; exports.FILE_STATUS_OK = 'ok'; exports.FILE_STATUS_UPDATE_VERSION = 'updateversion'; diff --git a/Common/sources/operationContext.js b/Common/sources/operationContext.js index 73a6ccdf..859c7f24 100644 --- a/Common/sources/operationContext.js +++ b/Common/sources/operationContext.js @@ -53,7 +53,8 @@ Context.prototype.initFromConnection = function(conn) { let tenant = tenantManager.getTenantByConnection(this, conn); let docId = conn.docid; if (!docId) { - const docIdParsed = constants.DOC_ID_SOCKET_PATTERN.exec(conn.url); + let handshake = conn.handshake; + const docIdParsed = constants.DOC_ID_SOCKET_PATTERN.exec(handshake.url); if (docIdParsed && 1 < docIdParsed.length) { docId = docIdParsed[1]; } diff --git a/Common/sources/tenantManager.js b/Common/sources/tenantManager.js index 2317cbc8..66dbb967 100644 --- a/Common/sources/tenantManager.js +++ b/Common/sources/tenantManager.js @@ -77,7 +77,7 @@ function getTenant(ctx, domain) { return tenant; } function getTenantByConnection(ctx, conn) { - return isMultitenantMode() ? getTenant(ctx, utils.getDomainByConnection(ctx, conn.request)) : getDefautTenant(); + return isMultitenantMode() ? getTenant(ctx, utils.getDomainByConnection(ctx, conn)) : getDefautTenant(); } function getTenantByRequest(ctx, req) { return isMultitenantMode() ? getTenant(ctx, utils.getDomainByRequest(ctx, req)) : getDefautTenant(); diff --git a/Common/sources/utils.js b/Common/sources/utils.js index 8c34b9fd..d692f99e 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -693,8 +693,9 @@ function getBaseUrlByRequest(req) { exports.getBaseUrlByConnection = getBaseUrlByConnection; exports.getBaseUrlByRequest = getBaseUrlByRequest; function getDomainByConnection(ctx, conn) { - let host = conn.headers['host']; - let forwardedHost = conn.headers['x-forwarded-host']; + let incomingMessage = conn.request; + let host = incomingMessage.headers['host']; + let forwardedHost = incomingMessage.headers['x-forwarded-host']; ctx.logger.debug("getDomainByConnection headers['host']=%s headers['x-forwarded-host']=%s", host, forwardedHost); return getDomain(host, forwardedHost); } diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 9e1175c4..0d5d3e58 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -508,7 +508,7 @@ function fillJwtByConnection(ctx, conn) { } function sendData(ctx, conn, data) { - conn.write(JSON.stringify(data)); + conn.emit('message', JSON.stringify(data)); const type = data ? data.type : null; ctx.logger.debug('sendData: type = %s', type); } @@ -1031,12 +1031,13 @@ function* publishCloseUsersConnection(ctx, docId, users, isOriginalId, code, des } } function closeUsersConnection(docId, usersMap, isOriginalId, code, description) { - let elConnection; + //close + let conn; for (let i = connections.length - 1; i >= 0; --i) { - elConnection = connections[i]; - if (elConnection.docId === docId) { - if (isOriginalId ? usersMap[elConnection.user.idOriginal] : usersMap[elConnection.user.id]) { - elConnection.close(code, description); + conn = connections[i]; + if (conn.docId === docId) { + if (isOriginalId ? usersMap[conn.user.idOriginal] : usersMap[conn.user.id]) { + conn.close(code, description); } } } @@ -1329,8 +1330,6 @@ exports.install = function(server, callbackFunction) { ctx.logger.info('io.use end'); next(checkJwtRes.decoded ? undefined : new Error("not authorized")); } - - }); }); @@ -1456,7 +1455,7 @@ exports.install = function(server, callbackFunction) { delete conn.authChangesAck; break; default: - ctx.logger.debug("unknown command %d", data); + ctx.logger.debug("unknown command %j", data); break; } if(clientStatsD) { @@ -1789,7 +1788,7 @@ exports.install = function(server, callbackFunction) { return el.sessionId === sessionId;//Delete this connection }); //closing could happen during async action - if (constants.CONN_CLOSED !== conn.readyState) { + if (constants.CONN_CLOSED !== conn.conn.readyState) { // Кладем в массив, т.к. нам нужно отправлять данные для открытия/сохранения документа connections.push(conn); yield addPresence(ctx, conn, true); @@ -2261,7 +2260,7 @@ exports.install = function(server, callbackFunction) { } } } - if (constants.CONN_CLOSED === conn.readyState) { + if (constants.CONN_CLOSED === conn.conn.readyState) { //closing could happen during async action return; } @@ -2323,7 +2322,7 @@ exports.install = function(server, callbackFunction) { return el.sessionId === data.sessionId;//Delete this connection }); //closing could happen during async action - if (constants.CONN_CLOSED !== conn.readyState) { + if (constants.CONN_CLOSED !== conn.conn.readyState) { // Кладем в массив, т.к. нам нужно отправлять данные для открытия/сохранения документа connections.push(conn); yield addPresence(ctx, conn, true); @@ -2454,7 +2453,7 @@ exports.install = function(server, callbackFunction) { const docId = conn.docId; const tmpUser = conn.user; let hasForgotten; - if (constants.CONN_CLOSED === conn.readyState) { + if (constants.CONN_CLOSED === conn.conn.readyState) { //closing could happen during async action return false; } @@ -2472,7 +2471,7 @@ exports.install = function(server, callbackFunction) { } } } - if (constants.CONN_CLOSED === conn.readyState) { + if (constants.CONN_CLOSED === conn.conn.readyState) { //closing could happen during async action return false; } @@ -2489,7 +2488,7 @@ exports.install = function(server, callbackFunction) { } } - if (constants.CONN_CLOSED === conn.readyState) { + if (constants.CONN_CLOSED === conn.conn.readyState) { //closing could happen during async action return false; } @@ -2498,7 +2497,7 @@ exports.install = function(server, callbackFunction) { if (!bIsRestore && 2 === countNoView && !tmpUser.view) { // Ставим lock на документ const lockRes = yield editorData.lockAuth(ctx, docId, firstParticipantNoView.id, 2 * cfgExpLockDoc); - if (constants.CONN_CLOSED === conn.readyState) { + if (constants.CONN_CLOSED === conn.conn.readyState) { //closing could happen during async action return false; } @@ -2512,7 +2511,7 @@ exports.install = function(server, callbackFunction) { yield* setLockDocumentTimer(ctx, docId, lockDocument.id); } } - if (constants.CONN_CLOSED === conn.readyState) { + if (constants.CONN_CLOSED === conn.conn.readyState) { //closing could happen during async action return false; } @@ -2527,13 +2526,13 @@ exports.install = function(server, callbackFunction) { if (!bIsRestore && needSendChanges(conn)) { yield* sendAuthChanges(ctx, conn.docId, [conn]); } - if (constants.CONN_CLOSED === conn.readyState) { + if (constants.CONN_CLOSED === conn.conn.readyState) { //closing could happen during async action return false; } yield* sendAuthInfo(ctx, conn, bIsRestore, participantsMap, hasForgotten, opt_openedAt); } - if (constants.CONN_CLOSED === conn.readyState) { + if (constants.CONN_CLOSED === conn.conn.readyState) { //closing could happen during async action return false; } @@ -3055,7 +3054,8 @@ exports.install = function(server, callbackFunction) { let rights = constants.RIGHTS.Edit; if (config.get('server.edit_singleton')) { // ToDo docId from url ? - const docIdParsed = constants.DOC_ID_SOCKET_PATTERN.exec(conn.url); + let handshake = conn.handshake; + const docIdParsed = constants.DOC_ID_SOCKET_PATTERN.exec(handshake.url); if (docIdParsed && 1 < docIdParsed.length) { const participantsMap = yield getParticipantMap(ctx, docIdParsed[1]); for (let i = 0; i < participantsMap.length; ++i) { @@ -3437,7 +3437,7 @@ exports.install = function(server, callbackFunction) { continue; } } - if (constants.CONN_CLOSED === conn.readyState) { + if (constants.CONN_CLOSED === conn.conn.readyState) { ctx.logger.error('expireDoc connection closed'); } yield addPresence(ctx, conn, false); From e320e9463a8839ff2de54ba4ad6a50ef1e88c1fd Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Sun, 18 Sep 2022 18:45:20 +0300 Subject: [PATCH 13/45] [socket.io] Remove SockJs module --- 3rd-Party.txt | 6 ++--- DocService/npm-shrinkwrap.json | 43 ------------------------------ DocService/package.json | 1 - DocService/sources/DocsCoServer.js | 7 ----- license/SockJs.license | 19 ------------- license/SocketIO.license | 22 +++++++++++++++ 6 files changed, 25 insertions(+), 73 deletions(-) delete mode 100644 license/SockJs.license create mode 100644 license/SocketIO.license diff --git a/3rd-Party.txt b/3rd-Party.txt index cf52de1b..63b096fe 100644 --- a/3rd-Party.txt +++ b/3rd-Party.txt @@ -28,11 +28,11 @@ License: MIT License License File: license/Megapixel.license -5. SockJS - WebSocket emulation - Javascript client +5. SocketIO - WebSocket emulation - Javascript client -URL: http://sockjs.org +URL: https://socket.io License: MIT License -License File: license/SockJS.license +License File: license/SocketIO.license 6. Underscore - Underscore is a utility-belt library for JavaScript that provides a lot of the functional programming support that you would expect in Prototype.js (or Ruby), but without extending any of the built-in JavaScript objects. It's the tie to go along with jQuery's tux, and Backbone.js's suspenders. diff --git a/DocService/npm-shrinkwrap.json b/DocService/npm-shrinkwrap.json index 4487f96f..baeac6b8 100644 --- a/DocService/npm-shrinkwrap.json +++ b/DocService/npm-shrinkwrap.json @@ -525,14 +525,6 @@ "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, - "faye-websocket": { - "version": "0.11.4", - "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.11.4.tgz", - "integrity": "sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g==", - "requires": { - "websocket-driver": ">=0.5.1" - } - }, "fd-slicer": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", @@ -584,11 +576,6 @@ "statuses": ">= 1.4.0 < 2" } }, - "http-parser-js": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.5.3.tgz", - "integrity": "sha512-t7hjvef/5HEK7RWTdUzVUhl8zkEu+LlaE0IYzdMuvbSDipxBRpOn4Uhw8ZyECEa808iVT8XCjzo6xmYt4CiLZg==" - }, "iconv-lite": { "version": "0.4.23", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz", @@ -1248,16 +1235,6 @@ } } }, - "sockjs": { - "version": "0.3.21", - "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.21.tgz", - "integrity": "sha512-DhbPFGpxjc6Z3I+uX07Id5ZO2XwYsWOrYjaSeieES78cq+JaJvVe5q/m1uvjIQhXinhIeCFRH6JgXe+mvVMyXw==", - "requires": { - "faye-websocket": "^0.11.3", - "uuid": "^3.4.0", - "websocket-driver": "^0.7.4" - } - }, "split2": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz", @@ -1359,31 +1336,11 @@ "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, - "uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" - }, "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" }, - "websocket-driver": { - "version": "0.7.4", - "resolved": "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.4.tgz", - "integrity": "sha512-b17KeDIQVjvb0ssuSDF2cYXSg2iztliJ4B9WdsuB6J952qCPKmnVq4DyW5motImXHDC1cBT/1UezrJVsKw5zjg==", - "requires": { - "http-parser-js": ">=0.5.1", - "safe-buffer": ">=5.1.0", - "websocket-extensions": ">=0.1.1" - } - }, - "websocket-extensions": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.4.tgz", - "integrity": "sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==" - }, "windows-locale": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/windows-locale/-/windows-locale-1.0.1.tgz", diff --git a/DocService/package.json b/DocService/package.json index c2fb20a8..8e37f454 100644 --- a/DocService/package.json +++ b/DocService/package.json @@ -34,7 +34,6 @@ "redis": "^2.8.0", "retry": "^0.12.0", "socket.io": "^4.5.2", - "sockjs": "^0.3.21", "underscore": "^1.13.1", "utf7": "^1.0.2", "windows-locale": "^1.0.1" diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 0d5d3e58..fdb8cbba 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -72,7 +72,6 @@ 'use strict'; -const sockjs = require('sockjs'); const { Server } = require("socket.io"); const _ = require('underscore'); const url = require('url'); @@ -131,7 +130,6 @@ const cfgExpSessionIdle = ms(config.get('expire.sessionidle')); const cfgExpSessionAbsolute = ms(config.get('expire.sessionabsolute')); const cfgExpSessionCloseCommand = ms(config.get('expire.sessionclosecommand')); const cfgExpUpdateVersionStatus = ms(config.get('expire.updateVersionStatus')); -const cfgSockjs = config.get('sockjs'); const cfgTokenEnableBrowser = config.get('token.enable.browser'); const cfgTokenEnableRequestInbox = config.get('token.enable.request.inbox'); const cfgTokenSessionAlgorithm = config.get('token.session.algorithm'); @@ -3170,11 +3168,6 @@ exports.install = function(server, callbackFunction) { return licenseType; } - // sockjs_echo.installHandlers(server, {prefix: '/doc/['+constants.DOC_ID_PATTERN+']*/c', log: function(severity, message) { - // //TODO: handle severity - // operationContext.global.logger.info(message); - // }}); - //publish subscribe message brocker function pubsubOnMessage(msg) { return co(function* () { diff --git a/license/SockJs.license b/license/SockJs.license deleted file mode 100644 index a7c85de1..00000000 --- a/license/SockJs.license +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (c) 2011-2012 VMware, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. \ No newline at end of file diff --git a/license/SocketIO.license b/license/SocketIO.license new file mode 100644 index 00000000..659a042c --- /dev/null +++ b/license/SocketIO.license @@ -0,0 +1,22 @@ +(The MIT License) + +Copyright (c) 2014-2018 Automattic + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +'Software'), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file From 99bda12ad17cc209a6b0f50045270394e9995942 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Fri, 21 Oct 2022 17:20:23 +0300 Subject: [PATCH 14/45] [feature] For binary changes --- DocService/sources/DocsCoServer.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index fdb8cbba..68e40c0b 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -1652,7 +1652,7 @@ exports.install = function(server, callbackFunction) { element = arrayElements[j]; // Добавляем GMT, т.к. в базу данных мы пишем UTC, но сохраняется туда строка без UTC и при зачитывании будет неправильное время - objChangesDocument.push({docid: docId, change: element['change_data'], + objChangesDocument.push({docid: docId, change: element['change_data'].data, time: element['change_date'].getTime(), user: element['user_id'], useridoriginal: element['user_id_original']}); } @@ -2813,7 +2813,7 @@ exports.install = function(server, callbackFunction) { // Стартовый индекс изменения при добавлении const startIndex = puckerIndex; - const newChanges = JSON.parse(data.changes); + const newChanges = data.changes; let newChangesLastDate = new Date(); newChangesLastDate.setMilliseconds(0);//remove milliseconds avoid issues with MySQL datetime rounding let newChangesLastTime = newChangesLastDate.getTime(); @@ -2824,7 +2824,7 @@ exports.install = function(server, callbackFunction) { for (let i = 0; i < newChanges.length; ++i) { oElement = newChanges[i]; - arrNewDocumentChanges.push({docid: docId, change: JSON.stringify(oElement), time: newChangesLastDate, + arrNewDocumentChanges.push({docid: docId, change: oElement, time: newChangesLastDate, user: userId, useridoriginal: conn.user.idOriginal}); } From abe92206dacd5bcf0280ce073706ba909ee046f9 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 24 Oct 2022 18:19:01 +0300 Subject: [PATCH 15/45] [feature] Add upgrade scripts; Add fixes for postgres --- Common/config/default.json | 2 +- DocService/sources/DocsCoServer.js | 4 ++-- DocService/sources/postgreSqlBaseConnector.js | 2 +- schema/mysql/upgrade/upgradev730.sql | 18 ++++++++++++++++++ schema/postgresql/upgrade/upgradev730.sql | 9 +++++++++ 5 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 schema/mysql/upgrade/upgradev730.sql create mode 100644 schema/postgresql/upgrade/upgradev730.sql diff --git a/Common/config/default.json b/Common/config/default.json index 21883ad3..3312d088 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -180,7 +180,7 @@ "options": {} }, "pubsub": { - "maxChanges": 1000 + "maxChanges": 0 }, "expire": { "saveLock": 60, diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 68e40c0b..f3a5d925 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -506,7 +506,7 @@ function fillJwtByConnection(ctx, conn) { } function sendData(ctx, conn, data) { - conn.emit('message', JSON.stringify(data)); + conn.emit('message', data); const type = data ? data.type : null; ctx.logger.debug('sendData: type = %s', type); } @@ -1652,7 +1652,7 @@ exports.install = function(server, callbackFunction) { element = arrayElements[j]; // Добавляем GMT, т.к. в базу данных мы пишем UTC, но сохраняется туда строка без UTC и при зачитывании будет неправильное время - objChangesDocument.push({docid: docId, change: element['change_data'].data, + objChangesDocument.push({docid: docId, change: element['change_data'], time: element['change_date'].getTime(), user: element['user_id'], useridoriginal: element['user_id_original']}); } diff --git a/DocService/sources/postgreSqlBaseConnector.js b/DocService/sources/postgreSqlBaseConnector.js index 3c491f71..54798cbb 100644 --- a/DocService/sources/postgreSqlBaseConnector.js +++ b/DocService/sources/postgreSqlBaseConnector.js @@ -182,7 +182,7 @@ exports.insertChanges = function(ctx, tableChanges, startIndex, objChanges, docI let time = []; //Postgres 9.4 multi-argument unnest let sqlCommand = `INSERT INTO ${tableChanges} (tenant, id, change_id, user_id, user_id_original, user_name, change_data, change_date) `; - sqlCommand += "SELECT * FROM UNNEST ($1::text[], $2::text[], $3::int[], $4::text[], $5::text[], $6::text[], $7::text[], $8::timestamp[]);"; + sqlCommand += "SELECT * FROM UNNEST ($1::text[], $2::text[], $3::int[], $4::text[], $5::text[], $6::text[], $7::bytea[], $8::timestamp[]);"; let values = [tenant, id, changeId, userId, userIdOriginal, username, change, time]; let curLength = sqlCommand.length; for (; i < objChanges.length; ++i) { diff --git a/schema/mysql/upgrade/upgradev730.sql b/schema/mysql/upgrade/upgradev730.sql new file mode 100644 index 00000000..2c144f03 --- /dev/null +++ b/schema/mysql/upgrade/upgradev730.sql @@ -0,0 +1,18 @@ +DELIMITER DLM00 + +DROP PROCEDURE IF EXISTS upgrade730 DLM00 + +CREATE PROCEDURE upgrade730() +BEGIN + + IF (SELECT DATA_TYPE FROM information_schema.`COLUMNS` WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'doc_changes' AND COLUMN_NAME = 'change_data') <> 'longblob' THEN + SET SQL_SAFE_UPDATES=0; + ALTER TABLE `doc_changes` CHANGE COLUMN `change_data` `change_data` LONGBLOB NOT NULL ; + SET SQL_SAFE_UPDATES=1; + END IF; + +END DLM00 + +CALL upgrade730() DLM00 + +DELIMITER ; diff --git a/schema/postgresql/upgrade/upgradev730.sql b/schema/postgresql/upgrade/upgradev730.sql new file mode 100644 index 00000000..a13b7ea8 --- /dev/null +++ b/schema/postgresql/upgrade/upgradev730.sql @@ -0,0 +1,9 @@ +DO $$ + BEGIN + BEGIN + ALTER TABLE doc_changes ALTER COLUMN change_data TYPE bytea USING change_data::bytea; + EXCEPTION + WHEN duplicate_column THEN RAISE NOTICE 'cant modify doc_changes.change_data colummn'; + END; + END; +$$ From 2749eb11722b36faa0d97613cf0faf5eb804d702 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Tue, 25 Oct 2022 18:32:19 +0300 Subject: [PATCH 16/45] [feature] Change history changes format --- DocService/sources/DocsCoServer.js | 10 ++-------- FileConverter/sources/converter.js | 19 ++++++++----------- 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index f3a5d925..87ef0e83 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -2546,14 +2546,8 @@ exports.install = function(server, callbackFunction) { do { changes = yield sqlBase.getChangesPromise(ctx, docId, index, index + cfgMaxRequestChanges); if (changes.length > 0) { - let changesJSON = indexChunk > 1 ? ',[' : '['; - changesJSON += changes[0].change_data; - for (let i = 1; i < changes.length; ++i) { - changesJSON += ','; - changesJSON += changes[i].change_data; - } - changesJSON += ']\r\n'; - let buffer = Buffer.from(changesJSON, 'utf8'); + let buffers = changes.map(elem => elem.change_data); + let buffer = Buffer.concat(buffers); yield storage.putObject(ctx, changesPrefix + (indexChunk++).toString().padStart(3, '0'), buffer, buffer.length, cfgErrorFiles); } index += cfgMaxRequestChanges; diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index 95d231b3..32f7431b 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -482,7 +482,7 @@ function* processChanges(ctx, tempDirs, task, cmd, authorProps) { let extChanges; if (extChangeInfo) { extChanges = [{ - id: cmd.getDocId(), change_id: 0, change_data: "", user_id: extChangeInfo.user_id, + id: cmd.getDocId(), change_id: 0, change_data: Buffer.alloc(0), user_id: extChangeInfo.user_id, user_id_original: extChangeInfo.user_id_original, user_name: extChangeInfo.user_name, change_date: new Date(extChangeInfo.change_date) }]; @@ -495,7 +495,7 @@ function* processChanges(ctx, tempDirs, task, cmd, authorProps) { let changes = []; if (curIndexStart < curIndexEnd) { changes = yield baseConnector.getChangesPromise(ctx, cmd.getDocId(), curIndexStart, curIndexEnd, forceSaveTime); - if (changes.length > 0 && changes[0].change_data.startsWith('ENCRYPTED;')) { + if (changes.length > 0 && changes[0].change_data.subarray(0, 'ENCRYPTED;'.length).includes('ENCRYPTED;')) { ctx.logger.warn('processChanges encrypted changes'); //todo sql request instead? res = constants.EDITOR_CHANGES; @@ -518,9 +518,6 @@ function* processChanges(ctx, tempDirs, task, cmd, authorProps) { } let strDate = baseConnector.getDateTime(change.change_date); changesHistory.changes.push({'created': strDate, 'user': {'id': change.user_id_original, 'name': change.user_name}}); - yield* streamWrite(streamObj, '['); - } else { - yield* streamWrite(streamObj, ','); } changesAuthor = change.user_id_original; changesAuthorUnique = change.user_id; @@ -538,7 +535,7 @@ function* processChanges(ctx, tempDirs, task, cmd, authorProps) { break; } } - yield* streamEnd(streamObj, ']'); + yield* streamEnd(streamObj); if (streamObj.isNoChangesInFile) { fs.unlinkSync(streamObj.filePath); } @@ -558,7 +555,7 @@ function* processChanges(ctx, tempDirs, task, cmd, authorProps) { } function* streamCreate(ctx, changesDir, indexFile, opt_options) { - let fileName = constants.CHANGES_NAME + indexFile + '.json'; + let fileName = constants.CHANGES_NAME + indexFile + '.bin'; let filePath = path.join(changesDir, fileName); let writeStream = yield utils.promiseCreateWriteStream(filePath, opt_options); writeStream.on('error', function(err) { @@ -568,14 +565,14 @@ function* streamCreate(ctx, changesDir, indexFile, opt_options) { return {writeStream: writeStream, filePath: filePath, isNoChangesInFile: true}; } -function* streamWrite(streamObj, text) { - if (!streamObj.writeStream.write(text, 'utf8')) { +function* streamWrite(streamObj, buf) { + if (!streamObj.writeStream.write(buf)) { yield utils.promiseWaitDrain(streamObj.writeStream); } } -function* streamEnd(streamObj, text) { - streamObj.writeStream.end(text, 'utf8'); +function* streamEnd(streamObj) { + streamObj.writeStream.end(); yield utils.promiseWaitClose(streamObj.writeStream); } function* processUploadToStorage(ctx, dir, storagePath) { From a050a5df4399f5f8c2778cfdcd5c10d6701b868c Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Wed, 26 Oct 2022 13:29:47 +0300 Subject: [PATCH 17/45] [bug] Use DB table names from config --- DocService/sources/DocsCoServer.js | 3 ++- DocService/sources/baseConnector.js | 18 ++++++++--------- DocService/sources/mySqlBaseConnector.js | 8 ++++++-- DocService/sources/postgreSqlBaseConnector.js | 9 +++++---- DocService/sources/taskresult.js | 20 +++++++++++-------- 5 files changed, 34 insertions(+), 24 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 87ef0e83..08d222cd 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -150,6 +150,7 @@ const cfgOpenProtectedFile = config.get('server.openProtectedFile'); const cfgRefreshLockInterval = ms(configCommon.get('wopi.refreshLockInterval')); const cfgTokenRequiredParams = config.get('server.tokenRequiredParams'); const cfgSocketIoConnection = configCommon.get('services.CoAuthoring.socketio.connection'); +const cfgTableResult = configCommon.get('services.CoAuthoring.sql.tableResult'); const EditorTypes = { document : 0, @@ -3519,7 +3520,7 @@ exports.install = function(server, callbackFunction) { } gc.startGC(); - let tableName = 'task_result'; + let tableName = cfgTableResult; const tableRequiredColumn = 'tenant'; //check data base compatibility sqlBase.getTableColumns(operationContext.global, tableName).then(function(res) { diff --git a/DocService/sources/baseConnector.js b/DocService/sources/baseConnector.js index 32b117ef..eba4e6ee 100644 --- a/DocService/sources/baseConnector.js +++ b/DocService/sources/baseConnector.js @@ -42,8 +42,8 @@ var config = require('config').get('services.CoAuthoring.sql'); var baseConnector = (sqlDataBaseType.mySql === config.get('type') || sqlDataBaseType.mariaDB === config.get('type')) ? require('./mySqlBaseConnector') : require('./postgreSqlBaseConnector'); let constants = require('./../../Common/sources/constants'); -const tableChanges = config.get('tableChanges'), - tableResult = config.get('tableResult'); +const cfgTableResult = config.get('tableResult'); +const cfgTableChanges = config.get('tableChanges'); var g_oCriticalSection = {}; let isSupportFastInsert = !!baseConnector.insertChanges; @@ -64,7 +64,7 @@ exports.insertChangesPromiseCompatibility = function (ctx, objChanges, docId, in }; exports.insertChangesPromiseFast = function (ctx, objChanges, docId, index, user) { return new Promise(function(resolve, reject) { - baseConnector.insertChanges(ctx, tableChanges, 0, objChanges, docId, index, user, function(error, result, isSupported) { + baseConnector.insertChanges(ctx, cfgTableChanges, 0, objChanges, docId, index, user, function(error, result, isSupported) { isSupportFastInsert = isSupported; if (error) { if (!isSupportFastInsert) { @@ -93,7 +93,7 @@ function _getDateTime2(oDate) { exports.getDateTime = _getDateTime2; function _insertChangesCallback (ctx, startIndex, objChanges, docId, index, user, callback) { - var sqlCommand = `INSERT INTO ${tableChanges} VALUES`; + var sqlCommand = `INSERT INTO ${cfgTableChanges} VALUES`; var i = startIndex, l = objChanges.length, lengthUtf8Current = sqlCommand.length, lengthUtf8Row = 0, values = []; if (i === l) return; @@ -136,9 +136,9 @@ exports.deleteChangesCallback = function(ctx, docId, deleteIndex, callback) { let p2 = addSqlParam(docId, values); if (null !== deleteIndex) { let sqlParam2 = addSqlParam(deleteIndex, values); - sqlCommand = `DELETE FROM ${tableChanges} WHERE tenant=${p1} AND id=${p2} AND change_id >= ${sqlParam2};`; + sqlCommand = `DELETE FROM ${cfgTableChanges} WHERE tenant=${p1} AND id=${p2} AND change_id >= ${sqlParam2};`; } else { - sqlCommand = `DELETE FROM ${tableChanges} WHERE tenant=${p1} AND id=${p2};`; + sqlCommand = `DELETE FROM ${cfgTableChanges} WHERE tenant=${p1} AND id=${p2};`; } baseConnector.sqlQuery(ctx, sqlCommand, callback, undefined, undefined, values); }; @@ -163,7 +163,7 @@ exports.getChangesIndex = function(ctx, docId, callback) { let values = []; let p1 = addSqlParam(ctx.tenant, values); let p2 = addSqlParam(docId, values); - var sqlCommand = `SELECT MAX(change_id) as change_id FROM ${tableChanges} WHERE tenant=${p1} AND id=${p2};`; + var sqlCommand = `SELECT MAX(change_id) as change_id FROM ${cfgTableChanges} WHERE tenant=${p1} AND id=${p2};`; baseConnector.sqlQuery(ctx, sqlCommand, callback, undefined, undefined, values); }; exports.getChangesIndexPromise = function(ctx, docId) { @@ -200,7 +200,7 @@ exports.getChangesPromise = function (ctx, docId, optStartIndex, optEndIndex, op sqlWhere += ` AND change_date<=${sqlParam}`; } sqlWhere += ' ORDER BY change_id ASC'; - var sqlCommand = `SELECT * FROM ${tableChanges} WHERE ${sqlWhere};`; + var sqlCommand = `SELECT * FROM ${cfgTableChanges} WHERE ${sqlWhere};`; baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { if (error) { @@ -252,7 +252,7 @@ exports.healthCheck = function (ctx) { exports.getEmptyCallbacks = function(ctx) { return new Promise(function(resolve, reject) { - const sqlCommand = "SELECT DISTINCT t1.tenant, t1.id FROM doc_changes t1 LEFT JOIN task_result t2 ON t2.tenant = t1.tenant AND t2.id = t1.id WHERE t2.callback = '';"; + const sqlCommand = `SELECT DISTINCT t1.tenant, t1.id FROM ${cfgTableChanges} t1 LEFT JOIN ${cfgTableResult} t2 ON t2.tenant = t1.tenant AND t2.id = t1.id WHERE t2.callback = '';`; baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { if (error) { reject(error); diff --git a/DocService/sources/mySqlBaseConnector.js b/DocService/sources/mySqlBaseConnector.js index e5420235..7b01b762 100644 --- a/DocService/sources/mySqlBaseConnector.js +++ b/DocService/sources/mySqlBaseConnector.js @@ -34,7 +34,11 @@ var mysql = require('mysql2'); var sqlBase = require('./baseConnector'); -var configSql = require('config').get('services.CoAuthoring.sql'); +const config = require('config'); + +const configSql = config.get('services.CoAuthoring.sql'); +const cfgTableResult = config.get('services.CoAuthoring.sql.tableResult'); + var pool = mysql.createPool({ host : configSql.get('dbHost'), port : configSql.get('dbPort'), @@ -102,7 +106,7 @@ exports.upsert = function(ctx, task, opt_updateUserIndex) { let p7 = addSqlParam(cbInsert, values); let p8 = addSqlParam(task.baseurl, values); let p9 = addSqlParam(dateNow, values); - var sqlCommand = 'INSERT INTO task_result (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl)'+ + var sqlCommand = `INSERT INTO ${cfgTableResult} (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl)`+ ` VALUES (${p0}, ${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8}) ON DUPLICATE KEY UPDATE` + ` last_open_date = ${p9}`; if (task.callback) { diff --git a/DocService/sources/postgreSqlBaseConnector.js b/DocService/sources/postgreSqlBaseConnector.js index 54798cbb..96c27a76 100644 --- a/DocService/sources/postgreSqlBaseConnector.js +++ b/DocService/sources/postgreSqlBaseConnector.js @@ -38,6 +38,7 @@ var types = require('pg').types; var sqlBase = require('./baseConnector'); const config = require('config'); var configSql = config.get('services.CoAuthoring.sql'); +const cfgTableResult = config.get('services.CoAuthoring.sql.tableResult'); var pgPoolExtraOptions = configSql.get('pgPoolExtraOptions'); let connectionConfig = { host: configSql.get('dbHost'), @@ -122,19 +123,19 @@ function getUpsertString(task, values) { if (isSupportOnConflict) { let p9 = addSqlParam(dateNow, values); //http://stackoverflow.com/questions/34762732/how-to-find-out-if-an-upsert-was-an-update-with-postgresql-9-5-upsert - let sqlCommand = "INSERT INTO task_result (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl)"; + let sqlCommand = `INSERT INTO ${cfgTableResult} (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl)`; sqlCommand += ` VALUES (${p0}, ${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8})`; sqlCommand += ` ON CONFLICT (tenant, id) DO UPDATE SET last_open_date = ${p9}`; if (task.callback) { let p10 = addSqlParam(JSON.stringify(task.callback), values); - sqlCommand += `, callback = task_result.callback || '${sqlBase.UserCallback.prototype.delimiter}{"userIndex":' `; - sqlCommand += ` || (task_result.user_index + 1)::text || ',"callback":' || ${p10}::text || '}'`; + sqlCommand += `, callback = ${cfgTableResult}.callback || '${sqlBase.UserCallback.prototype.delimiter}{"userIndex":' `; + sqlCommand += ` || (${cfgTableResult}.user_index + 1)::text || ',"callback":' || ${p10}::text || '}'`; } if (task.baseurl) { let p11 = addSqlParam(task.baseurl, values); sqlCommand += `, baseurl = ${p11}`; } - sqlCommand += ", user_index = task_result.user_index + 1 RETURNING user_index as userindex;"; + sqlCommand += ", user_index = ${cfgTableResult}.user_index + 1 RETURNING user_index as userindex;"; return sqlCommand; } else { return `SELECT * FROM merge_db(${p0}, ${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8});`; diff --git a/DocService/sources/taskresult.js b/DocService/sources/taskresult.js index 699ecd45..ec860dd3 100644 --- a/DocService/sources/taskresult.js +++ b/DocService/sources/taskresult.js @@ -38,6 +38,10 @@ var utils = require('./../../Common/sources/utils'); var constants = require('./../../Common/sources/constants'); var commonDefines = require('./../../Common/sources/commondefines'); var tenantManager = require('./../../Common/sources/tenantManager'); +var config = require('config'); + +const cfgTableResult = config.get('services.CoAuthoring.sql.tableResult'); +const cfgTableChanges = config.get('services.CoAuthoring.sql.tableChanges'); let addSqlParam = sqlBase.baseConnector.addSqlParameter; let concatParams = sqlBase.baseConnector.concatParams; @@ -102,7 +106,7 @@ function select(ctx, docId) { let values = []; let p1 = addSqlParam(ctx.tenant, values); let p2 = addSqlParam(docId, values); - let sqlCommand = `SELECT * FROM task_result WHERE tenant=${p1} AND id=${p2};`; + let sqlCommand = `SELECT * FROM ${cfgTableResult} WHERE tenant=${p1} AND id=${p2};`; sqlBase.baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { if (error) { reject(error); @@ -171,7 +175,7 @@ function update(ctx, task, setPassword) { let sqlSet = updateElems.join(', '); let p1 = addSqlParam(task.tenant, values); let p2 = addSqlParam(task.key, values); - let sqlCommand = `UPDATE task_result SET ${sqlSet} WHERE tenant=${p1} AND id=${p2};`; + let sqlCommand = `UPDATE ${cfgTableResult} SET ${sqlSet} WHERE tenant=${p1} AND id=${p2};`; sqlBase.baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { if (error) { reject(error); @@ -191,7 +195,7 @@ function updateIf(ctx, task, mask) { commandArgMask.push('id=' + addSqlParam(mask.key, values)); let sqlSet = commandArg.join(', '); let sqlWhere = commandArgMask.join(' AND '); - let sqlCommand = `UPDATE task_result SET ${sqlSet} WHERE ${sqlWhere};`; + let sqlCommand = `UPDATE ${cfgTableResult} SET ${sqlSet} WHERE ${sqlWhere};`; sqlBase.baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { if (error) { reject(error); @@ -241,7 +245,7 @@ function addRandomKey(ctx, task, opt_prefix, opt_size) { let p6 = addSqlParam(task.changeId, values); let p7 = addSqlParam(task.callback, values); let p8 = addSqlParam(task.baseurl, values); - let sqlCommand = 'INSERT INTO task_result (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl)' + + let sqlCommand = 'INSERT INTO ${cfgTableResult} (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl)' + ` VALUES (${p0}, ${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8});`; sqlBase.baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { if (error) { @@ -283,7 +287,7 @@ function remove(ctx, docId) { let values = []; let p1 = addSqlParam(ctx.tenant, values); let p2 = addSqlParam(docId, values); - const sqlCommand = `DELETE FROM task_result WHERE tenant=${p1} AND id=${p2};`; + const sqlCommand = `DELETE FROM ${cfgTableResult} WHERE tenant=${p1} AND id=${p2};`; sqlBase.baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { if (error) { reject(error); @@ -300,7 +304,7 @@ function removeIf(ctx, mask) { commandArgMask.push('tenant=' + addSqlParam(mask.tenant, values)); commandArgMask.push('id=' + addSqlParam(mask.key, values)); let sqlWhere = commandArgMask.join(' AND '); - const sqlCommand = `DELETE FROM task_result WHERE ${sqlWhere};`; + const sqlCommand = `DELETE FROM ${cfgTableResult} WHERE ${sqlWhere};`; sqlBase.baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { if (error) { reject(error); @@ -317,8 +321,8 @@ function getExpired(ctx, maxCount, expireSeconds) { utils.addSeconds(expireDate, -expireSeconds); let sqlParam1 = addSqlParam(expireDate, values); let sqlParam2 = addSqlParam(maxCount, values); - let sqlCommand = `SELECT * FROM task_result WHERE last_open_date <= ${sqlParam1}` + - ` AND NOT EXISTS(SELECT tenant, id FROM doc_changes WHERE doc_changes.tenant = task_result.tenant AND doc_changes.id = task_result.id LIMIT 1) LIMIT ${sqlParam2};`; + let sqlCommand = `SELECT * FROM ${cfgTableResult} WHERE last_open_date <= ${sqlParam1}` + + ` AND NOT EXISTS(SELECT tenant, id FROM ${cfgTableChanges} WHERE ${cfgTableChanges}.tenant = ${cfgTableResult}.tenant AND ${cfgTableChanges}.id = ${cfgTableResult}.id LIMIT 1) LIMIT ${sqlParam2};`; sqlBase.baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { if (error) { reject(error); From fcb9a57a7d40716a7b9e39c56adda7f730044668 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Thu, 27 Oct 2022 14:20:13 +0300 Subject: [PATCH 18/45] [feature] Add binaryChanges config param --- Common/config/default.json | 1 + Common/sources/utils.js | 4 +- DocService/sources/DocsCoServer.js | 30 +++- DocService/sources/postgreSqlBaseConnector.js | 7 +- DocService/sources/taskresult.js | 2 +- FileConverter/sources/converter.js | 146 ++++++++++++++++-- schema/mysql/createdb.sql | 24 +++ schema/postgresql/createdb.sql | 16 ++ 8 files changed, 207 insertions(+), 23 deletions(-) diff --git a/Common/config/default.json b/Common/config/default.json index 3312d088..7b552977 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -260,6 +260,7 @@ "attempts": 50, "delay": "2s" }, + "binaryChanges": false, "websocketMaxPayloadSize": "1.5MB", "maxChangesSize": "0mb" }, diff --git a/Common/sources/utils.js b/Common/sources/utils.js index d692f99e..79a9ea17 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -1044,4 +1044,6 @@ exports.getFunctionArguments = function(func) { exports.isUselesSfc = function(row, cmd) { return !(row && commonDefines.FileStatus.SaveVersion === row.status && cmd.getStatusInfoIn() === row.status_info); }; - +exports.getChangesFileHeader = function() { + return `CHANGES|${commonDefines.buildVersion};`; +}; diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 08d222cd..a535ba40 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -1482,10 +1482,10 @@ exports.install = function(server, callbackFunction) { _checkLicense(ctx, conn); }); io.engine.on("connection_error", (err) => { - console.log(err.req); // the request object - console.log(err.code); // the error code, for example 1 - console.log(err.message); // the error message, for example "Session ID unknown" - console.log(err.context); // some additional error context + operationContext.global.logger.warn(err.req); // the request object + operationContext.global.logger.warn(err.code); // the error code, for example 1 + operationContext.global.logger.warn(err.message); // the error message, for example "Session ID unknown" + operationContext.global.logger.warn(err.context); // some additional error context }); /** * @@ -2547,8 +2547,21 @@ exports.install = function(server, callbackFunction) { do { changes = yield sqlBase.getChangesPromise(ctx, docId, index, index + cfgMaxRequestChanges); if (changes.length > 0) { - let buffers = changes.map(elem => elem.change_data); - let buffer = Buffer.concat(buffers); + let buffer; + if (cfgEditor['binaryChanges']) { + let buffers = changes.map(elem => elem.change_data); + buffers.unshift(Buffer.from(utils.getChangesFileHeader(), 'utf-8')) + buffer = Buffer.concat(buffers); + } else { + let changesJSON = indexChunk > 1 ? ',[' : '['; + changesJSON += changes[0].change_data; + for (let i = 1; i < changes.length; ++i) { + changesJSON += ','; + changesJSON += changes[i].change_data; + } + changesJSON += ']\r\n'; + buffer = Buffer.from(changesJSON, 'utf8'); + } yield storage.putObject(ctx, changesPrefix + (indexChunk++).toString().padStart(3, '0'), buffer, buffer.length, cfgErrorFiles); } index += cfgMaxRequestChanges; @@ -2808,7 +2821,7 @@ exports.install = function(server, callbackFunction) { // Стартовый индекс изменения при добавлении const startIndex = puckerIndex; - const newChanges = data.changes; + const newChanges = cfgEditor['binaryChanges'] ? data.changes : JSON.parse(data.changes); let newChangesLastDate = new Date(); newChangesLastDate.setMilliseconds(0);//remove milliseconds avoid issues with MySQL datetime rounding let newChangesLastTime = newChangesLastDate.getTime(); @@ -2819,7 +2832,8 @@ exports.install = function(server, callbackFunction) { for (let i = 0; i < newChanges.length; ++i) { oElement = newChanges[i]; - arrNewDocumentChanges.push({docid: docId, change: oElement, time: newChangesLastDate, + let change = cfgEditor['binaryChanges'] ? oElement : JSON.stringify(oElement); + arrNewDocumentChanges.push({docid: docId, change: change, time: newChangesLastDate, user: userId, useridoriginal: conn.user.idOriginal}); } diff --git a/DocService/sources/postgreSqlBaseConnector.js b/DocService/sources/postgreSqlBaseConnector.js index 96c27a76..cd6b9d71 100644 --- a/DocService/sources/postgreSqlBaseConnector.js +++ b/DocService/sources/postgreSqlBaseConnector.js @@ -40,6 +40,8 @@ const config = require('config'); var configSql = config.get('services.CoAuthoring.sql'); const cfgTableResult = config.get('services.CoAuthoring.sql.tableResult'); var pgPoolExtraOptions = configSql.get('pgPoolExtraOptions'); +const cfgEditor = config.get('services.CoAuthoring.editor'); + let connectionConfig = { host: configSql.get('dbHost'), port: configSql.get('dbPort'), @@ -135,7 +137,7 @@ function getUpsertString(task, values) { let p11 = addSqlParam(task.baseurl, values); sqlCommand += `, baseurl = ${p11}`; } - sqlCommand += ", user_index = ${cfgTableResult}.user_index + 1 RETURNING user_index as userindex;"; + sqlCommand += `, user_index = ${cfgTableResult}.user_index + 1 RETURNING user_index as userindex;`; return sqlCommand; } else { return `SELECT * FROM merge_db(${p0}, ${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8});`; @@ -183,7 +185,8 @@ exports.insertChanges = function(ctx, tableChanges, startIndex, objChanges, docI let time = []; //Postgres 9.4 multi-argument unnest let sqlCommand = `INSERT INTO ${tableChanges} (tenant, id, change_id, user_id, user_id_original, user_name, change_data, change_date) `; - sqlCommand += "SELECT * FROM UNNEST ($1::text[], $2::text[], $3::int[], $4::text[], $5::text[], $6::text[], $7::bytea[], $8::timestamp[]);"; + let changesType = cfgEditor['binaryChanges'] ? 'bytea' : 'text'; + sqlCommand += `SELECT * FROM UNNEST ($1::text[], $2::text[], $3::int[], $4::text[], $5::text[], $6::text[], $7::${changesType}[], $8::timestamp[]);`; let values = [tenant, id, changeId, userId, userIdOriginal, username, change, time]; let curLength = sqlCommand.length; for (; i < objChanges.length; ++i) { diff --git a/DocService/sources/taskresult.js b/DocService/sources/taskresult.js index ec860dd3..a0b2ab78 100644 --- a/DocService/sources/taskresult.js +++ b/DocService/sources/taskresult.js @@ -245,7 +245,7 @@ function addRandomKey(ctx, task, opt_prefix, opt_size) { let p6 = addSqlParam(task.changeId, values); let p7 = addSqlParam(task.callback, values); let p8 = addSqlParam(task.baseurl, values); - let sqlCommand = 'INSERT INTO ${cfgTableResult} (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl)' + + let sqlCommand = `INSERT INTO ${cfgTableResult} (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl)` + ` VALUES (${p0}, ${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8});`; sqlBase.baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { if (error) { diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index 32f7431b..e75e32f8 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -35,7 +35,6 @@ var os = require('os'); var path = require('path'); var fs = require('fs'); var url = require('url'); -var childProcess = require('child_process'); var co = require('co'); var config = require('config'); var spawnAsync = require('@expo/spawn-async'); @@ -78,6 +77,7 @@ const cfgMaxRequestChanges = config.get('services.CoAuthoring.server.maxRequestC const cfgForgottenFiles = config.get('services.CoAuthoring.server.forgottenfiles'); const cfgForgottenFilesName = config.get('services.CoAuthoring.server.forgottenfilesname'); const cfgNewFileTemplate = config.get('services.CoAuthoring.server.newFileTemplate'); +const cfgEditor = config.get('services.CoAuthoring.editor'); //windows limit 512(2048) https://msdn.microsoft.com/en-us/library/6e3b887c.aspx //Ubuntu 14.04 limit 4096 http://underyx.me/2015/05/18/raising-the-maximum-number-of-file-descriptors.html @@ -416,7 +416,11 @@ function* processDownloadFromStorage(ctx, dataConvert, cmd, task, tempDirs, auth yield* concatFiles(tempDirs.source); } if (task.getFromChanges()) { - res = yield* processChanges(ctx, tempDirs, task, cmd, authorProps); + if(cfgEditor['binaryChanges']) { + res = yield* processChangesBin(ctx, tempDirs, task, cmd, authorProps); + } else { + res = yield* processChangesBase64(ctx, tempDirs, task, cmd, authorProps); + } } //todo rework if (!fs.existsSync(dataConvert.fileFrom)) { @@ -458,8 +462,7 @@ function* concatFiles(source) { } } } - -function* processChanges(ctx, tempDirs, task, cmd, authorProps) { +function* processChangesBin(ctx, tempDirs, task, cmd, authorProps) { let res = constants.NO_ERROR; let changesDir = path.join(tempDirs.source, constants.CHANGES_NAME); fs.mkdirSync(changesDir); @@ -488,7 +491,8 @@ function* processChanges(ctx, tempDirs, task, cmd, authorProps) { }]; } - let streamObj = yield* streamCreate(ctx, changesDir, indexFile++, {highWaterMark: cfgStreamWriterBufferSize}); + let streamObj = yield* streamCreateBin(ctx, changesDir, indexFile++, {highWaterMark: cfgStreamWriterBufferSize}); + yield* streamWriteBin(streamObj, Buffer.from(utils.getChangesFileHeader(), 'utf-8')); let curIndexStart = 0; let curIndexEnd = Math.min(curIndexStart + cfgMaxRequestChanges, forceSaveIndex); while (curIndexStart < curIndexEnd || extChanges) { @@ -509,6 +513,123 @@ function* processChanges(ctx, tempDirs, task, cmd, authorProps) { changes = extChanges; } extChanges = undefined; + for (let i = 0; i < changes.length; ++i) { + let change = changes[i]; + if (null === changesAuthor || changesAuthor !== change.user_id_original) { + if (null !== changesAuthor) { + yield* streamEndBin(streamObj); + streamObj = yield* streamCreateBin(ctx, changesDir, indexFile++); + yield* streamWriteBin(streamObj, Buffer.from(utils.getChangesFileHeader(), 'utf-8')); + } + let strDate = baseConnector.getDateTime(change.change_date); + changesHistory.changes.push({'created': strDate, 'user': {'id': change.user_id_original, 'name': change.user_name}}); + } + changesAuthor = change.user_id_original; + changesAuthorUnique = change.user_id; + yield* streamWriteBin(streamObj, change.change_data); + streamObj.isNoChangesInFile = false; + } + if (changes.length > 0) { + authorProps.lastModifiedBy = changes[changes.length - 1].user_name; + authorProps.modified = changes[changes.length - 1].change_date.toISOString().slice(0, 19) + 'Z'; + } + if (changes.length === curIndexEnd - curIndexStart) { + curIndexStart += cfgMaxRequestChanges; + curIndexEnd = Math.min(curIndexStart + cfgMaxRequestChanges, forceSaveIndex); + } else { + break; + } + } + yield* streamEndBin(streamObj); + if (streamObj.isNoChangesInFile) { + fs.unlinkSync(streamObj.filePath); + } + if (null !== changesAuthorUnique) { + changesIndex = utils.getIndexFromUserId(changesAuthorUnique, changesAuthor); + } + if (null == changesAuthor && null == changesIndex && forceSave && undefined !== forceSave.getAuthorUserId() && + undefined !== forceSave.getAuthorUserIndex()) { + changesAuthor = forceSave.getAuthorUserId(); + changesIndex = forceSave.getAuthorUserIndex(); + } + cmd.setUserId(changesAuthor); + cmd.setUserIndex(changesIndex); + fs.writeFileSync(path.join(tempDirs.result, 'changesHistory.json'), JSON.stringify(changesHistory), 'utf8'); + ctx.logger.debug('processChanges end'); + return res; +} + +function* streamCreateBin(ctx, changesDir, indexFile, opt_options) { + let fileName = constants.CHANGES_NAME + indexFile + '.bin'; + let filePath = path.join(changesDir, fileName); + let writeStream = yield utils.promiseCreateWriteStream(filePath, opt_options); + writeStream.on('error', function(err) { + //todo integrate error handle in main thread (probable: set flag here and check it in main thread) + ctx.logger.error('WriteStreamError %s', err.stack); + }); + return {writeStream: writeStream, filePath: filePath, isNoChangesInFile: true}; +} + +function* streamWriteBin(streamObj, buf) { + if (!streamObj.writeStream.write(buf)) { + yield utils.promiseWaitDrain(streamObj.writeStream); + } +} + +function* streamEndBin(streamObj) { + streamObj.writeStream.end(); + yield utils.promiseWaitClose(streamObj.writeStream); +} +function* processChangesBase64(ctx, tempDirs, task, cmd, authorProps) { + let res = constants.NO_ERROR; + let changesDir = path.join(tempDirs.source, constants.CHANGES_NAME); + fs.mkdirSync(changesDir); + let indexFile = 0; + let changesAuthor = null; + let changesAuthorUnique = null; + let changesIndex = null; + let changesHistory = { + serverVersion: commonDefines.buildVersion, + changes: [] + }; + let forceSave = cmd.getForceSave(); + let forceSaveTime; + let forceSaveIndex = Number.MAX_VALUE; + if (forceSave && undefined !== forceSave.getTime() && undefined !== forceSave.getIndex()) { + forceSaveTime = forceSave.getTime(); + forceSaveIndex = forceSave.getIndex(); + } + let extChangeInfo = cmd.getExternalChangeInfo(); + let extChanges; + if (extChangeInfo) { + extChanges = [{ + id: cmd.getDocId(), change_id: 0, change_data: "", user_id: extChangeInfo.user_id, + user_id_original: extChangeInfo.user_id_original, user_name: extChangeInfo.user_name, + change_date: new Date(extChangeInfo.change_date) + }]; + } + + let streamObj = yield* streamCreate(ctx, changesDir, indexFile++, {highWaterMark: cfgStreamWriterBufferSize}); + let curIndexStart = 0; + let curIndexEnd = Math.min(curIndexStart + cfgMaxRequestChanges, forceSaveIndex); + while (curIndexStart < curIndexEnd || extChanges) { + let changes = []; + if (curIndexStart < curIndexEnd) { + changes = yield baseConnector.getChangesPromise(ctx, cmd.getDocId(), curIndexStart, curIndexEnd, forceSaveTime); + if (changes.length > 0 && changes[0].change_data.startsWith('ENCRYPTED;')) { + ctx.logger.warn('processChanges encrypted changes'); + //todo sql request instead? + res = constants.EDITOR_CHANGES; + } + res = yield* isUselessConvertion(ctx, task, cmd); + if (constants.NO_ERROR !== res) { + break; + } + } + if (0 === changes.length && extChanges) { + changes = extChanges; + } + extChanges = undefined; for (let i = 0; i < changes.length; ++i) { let change = changes[i]; if (null === changesAuthor || changesAuthor !== change.user_id_original) { @@ -518,6 +639,9 @@ function* processChanges(ctx, tempDirs, task, cmd, authorProps) { } let strDate = baseConnector.getDateTime(change.change_date); changesHistory.changes.push({'created': strDate, 'user': {'id': change.user_id_original, 'name': change.user_name}}); + yield* streamWrite(streamObj, '['); + } else { + yield* streamWrite(streamObj, ','); } changesAuthor = change.user_id_original; changesAuthorUnique = change.user_id; @@ -535,7 +659,7 @@ function* processChanges(ctx, tempDirs, task, cmd, authorProps) { break; } } - yield* streamEnd(streamObj); + yield* streamEnd(streamObj, ']'); if (streamObj.isNoChangesInFile) { fs.unlinkSync(streamObj.filePath); } @@ -555,7 +679,7 @@ function* processChanges(ctx, tempDirs, task, cmd, authorProps) { } function* streamCreate(ctx, changesDir, indexFile, opt_options) { - let fileName = constants.CHANGES_NAME + indexFile + '.bin'; + let fileName = constants.CHANGES_NAME + indexFile + '.json'; let filePath = path.join(changesDir, fileName); let writeStream = yield utils.promiseCreateWriteStream(filePath, opt_options); writeStream.on('error', function(err) { @@ -565,14 +689,14 @@ function* streamCreate(ctx, changesDir, indexFile, opt_options) { return {writeStream: writeStream, filePath: filePath, isNoChangesInFile: true}; } -function* streamWrite(streamObj, buf) { - if (!streamObj.writeStream.write(buf)) { +function* streamWrite(streamObj, text) { + if (!streamObj.writeStream.write(text, 'utf8')) { yield utils.promiseWaitDrain(streamObj.writeStream); } } -function* streamEnd(streamObj) { - streamObj.writeStream.end(); +function* streamEnd(streamObj, text) { + streamObj.writeStream.end(text, 'utf8'); yield utils.promiseWaitClose(streamObj.writeStream); } function* processUploadToStorage(ctx, dir, storagePath) { diff --git a/schema/mysql/createdb.sql b/schema/mysql/createdb.sql index e98a8d8f..c9f6b296 100644 --- a/schema/mysql/createdb.sql +++ b/schema/mysql/createdb.sql @@ -44,6 +44,30 @@ CREATE TABLE IF NOT EXISTS `doc_changes` ( /*!40000 ALTER TABLE `doc_changes` DISABLE KEYS */; /*!40000 ALTER TABLE `doc_changes` ENABLE KEYS */; + +-- +-- Definition of table `doc_changes` +-- + +CREATE TABLE IF NOT EXISTS `doc_changes2` ( + `tenant` varchar(255) NOT NULL, + `id` varchar(255) NOT NULL, + `change_id` int(10) unsigned NOT NULL, + `user_id` varchar(255) NOT NULL, + `user_id_original` varchar(255) NOT NULL, + `user_name` varchar(255) NOT NULL, + `change_data` longblob NOT NULL, + `change_date` datetime NOT NULL, + PRIMARY KEY (`tenant`, `id`,`change_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- +-- Dumping data for table `doc_changes` +-- + +/*!40000 ALTER TABLE `doc_changes2` DISABLE KEYS */; +/*!40000 ALTER TABLE `doc_changes2` ENABLE KEYS */; + -- -- Definition of table `task_result` -- diff --git a/schema/postgresql/createdb.sql b/schema/postgresql/createdb.sql index 0d054523..12b5764c 100644 --- a/schema/postgresql/createdb.sql +++ b/schema/postgresql/createdb.sql @@ -20,6 +20,22 @@ PRIMARY KEY ("tenant", "id", "change_id") ) WITH (OIDS=FALSE); +-- ---------------------------- +-- Table structure for doc_changes2 +-- ---------------------------- +CREATE TABLE IF NOT EXISTS "public"."doc_changes2" ( +"tenant" varchar(255) COLLATE "default" NOT NULL, +"id" varchar(255) COLLATE "default" NOT NULL, +"change_id" int4 NOT NULL, +"user_id" varchar(255) COLLATE "default" NOT NULL, +"user_id_original" varchar(255) COLLATE "default" NOT NULL, +"user_name" varchar(255) COLLATE "default" NOT NULL, +"change_data" bytea COLLATE "default" NOT NULL, +"change_date" timestamp without time zone NOT NULL, +PRIMARY KEY ("tenant", "id", "change_id") +) +WITH (OIDS=FALSE); + -- ---------------------------- -- Table structure for task_result -- ---------------------------- From 6fedd4612666ef9a90d0acab4b26eabcc8454641 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Thu, 27 Oct 2022 15:23:29 +0300 Subject: [PATCH 19/45] [feature] Change header of file containing changes --- Common/sources/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common/sources/utils.js b/Common/sources/utils.js index 79a9ea17..66ff3e94 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -1045,5 +1045,5 @@ exports.isUselesSfc = function(row, cmd) { return !(row && commonDefines.FileStatus.SaveVersion === row.status && cmd.getStatusInfoIn() === row.status_info); }; exports.getChangesFileHeader = function() { - return `CHANGES|${commonDefines.buildVersion};`; + return `CHANGES\t${commonDefines.buildVersion}\n`; }; From a6615dea841cf2eecd3130cbf512b894dbb8fa41 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Thu, 27 Oct 2022 16:17:10 +0300 Subject: [PATCH 20/45] [bug] Fix connection without jwt --- DocService/sources/DocsCoServer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index a535ba40..9e5a8a24 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -1327,7 +1327,7 @@ exports.install = function(server, callbackFunction) { ctx.logger.info('io.use error: %s', err.stack); } finally { ctx.logger.info('io.use end'); - next(checkJwtRes.decoded ? undefined : new Error("not authorized")); + next((checkJwtRes && !checkJwtRes.decoded) ? new Error("not authorized") : undefined); } }); }); From 1cf189943b59351c70c628304e5949bcceb98efa Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 31 Oct 2022 16:00:41 +0300 Subject: [PATCH 21/45] [feature] Add bottleneck to throttle database requests --- Common/config/default.json | 4 ++ DocService/npm-shrinkwrap.json | 5 ++ DocService/package.json | 1 + DocService/sources/baseConnector.js | 88 ++++++++++++++++++----------- 4 files changed, 64 insertions(+), 34 deletions(-) diff --git a/Common/config/default.json b/Common/config/default.json index 7b552977..0067bfe2 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -73,6 +73,10 @@ "passwords": ["verysecretstring"] } }, + "bottleneck": { + "getChanges": { + } + }, "wopi": { "enable": false, "host" : "", diff --git a/DocService/npm-shrinkwrap.json b/DocService/npm-shrinkwrap.json index baeac6b8..2bab1536 100644 --- a/DocService/npm-shrinkwrap.json +++ b/DocService/npm-shrinkwrap.json @@ -108,6 +108,11 @@ } } }, + "bottleneck": { + "version": "2.19.5", + "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", + "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==" + }, "buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", diff --git a/DocService/package.json b/DocService/package.json index 8e37f454..c1d3cff9 100644 --- a/DocService/package.json +++ b/DocService/package.json @@ -13,6 +13,7 @@ "apicache": "^1.6.2", "base64-stream": "^1.0.0", "body-parser": "^1.18.3", + "bottleneck": "^2.19.5", "bytes": "^3.0.0", "co": "^4.6.0", "config": "^2.0.1", diff --git a/DocService/sources/baseConnector.js b/DocService/sources/baseConnector.js index eba4e6ee..b5fbad97 100644 --- a/DocService/sources/baseConnector.js +++ b/DocService/sources/baseConnector.js @@ -38,17 +38,27 @@ var sqlDataBaseType = { postgreSql : 'postgres' }; -var config = require('config').get('services.CoAuthoring.sql'); -var baseConnector = (sqlDataBaseType.mySql === config.get('type') || sqlDataBaseType.mariaDB === config.get('type')) ? require('./mySqlBaseConnector') : require('./postgreSqlBaseConnector'); +var bottleneck = require("bottleneck"); +var config = require('config'); +var configSql = config.get('services.CoAuthoring.sql'); +var baseConnector = (sqlDataBaseType.mySql === configSql.get('type') || sqlDataBaseType.mariaDB === configSql.get('type')) ? require('./mySqlBaseConnector') : require('./postgreSqlBaseConnector'); let constants = require('./../../Common/sources/constants'); -const cfgTableResult = config.get('tableResult'); -const cfgTableChanges = config.get('tableChanges'); +const cfgTableResult = configSql.get('tableResult'); +const cfgTableChanges = configSql.get('tableChanges'); var g_oCriticalSection = {}; let isSupportFastInsert = !!baseConnector.insertChanges; let addSqlParam = baseConnector.addSqlParameter; -var maxPacketSize = config.get('max_allowed_packet'); // Размер по умолчанию для запроса в базу данных 1Mb - 1 (т.к. он не пишет 1048575, а пишет 1048574) +var maxPacketSize = configSql.get('max_allowed_packet'); // Размер по умолчанию для запроса в базу данных 1Mb - 1 (т.к. он не пишет 1048575, а пишет 1048574) +const cfgBottleneckGetChanges = config.get('bottleneck.getChanges'); + +let reservoirMaximum = cfgBottleneckGetChanges.reservoirIncreaseMaximum || cfgBottleneckGetChanges.reservoirRefreshAmount; +let group = new bottleneck.Group(cfgBottleneckGetChanges); + +function getChangesSize(changes) { + return changes.reduce((accumulator, currentValue) => accumulator + currentValue.change_data.length, 0); +} exports.baseConnector = baseConnector; exports.insertChangesPromiseCompatibility = function (ctx, objChanges, docId, index, user) { @@ -178,37 +188,47 @@ exports.getChangesIndexPromise = function(ctx, docId) { }); }; exports.getChangesPromise = function (ctx, docId, optStartIndex, optEndIndex, opt_time) { - return new Promise(function(resolve, reject) { - let values = []; - let sqlParam = addSqlParam(ctx.tenant, values); - let sqlWhere = `tenant=${sqlParam}`; - sqlParam = addSqlParam(docId, values); - sqlWhere += ` AND id=${sqlParam}`; - if (null != optStartIndex) { - sqlParam = addSqlParam(optStartIndex, values); - sqlWhere += ` AND change_id>=${sqlParam}`; - } - if (null != optEndIndex) { - sqlParam = addSqlParam(optEndIndex, values); - sqlWhere += ` AND change_id<${sqlParam}`; - } - if (null != opt_time) { - if (!(opt_time instanceof Date)) { - opt_time = new Date(opt_time); + let limiter = group.key(`${ctx.tenant}\t${docId}\tchanges`); + return limiter.schedule(() => { + return new Promise(function(resolve, reject) { + let values = []; + let sqlParam = addSqlParam(ctx.tenant, values); + let sqlWhere = `tenant=${sqlParam}`; + sqlParam = addSqlParam(docId, values); + sqlWhere += ` AND id=${sqlParam}`; + if (null != optStartIndex) { + sqlParam = addSqlParam(optStartIndex, values); + sqlWhere += ` AND change_id>=${sqlParam}`; } - sqlParam = addSqlParam(opt_time, values); - sqlWhere += ` AND change_date<=${sqlParam}`; - } - sqlWhere += ' ORDER BY change_id ASC'; - var sqlCommand = `SELECT * FROM ${cfgTableChanges} WHERE ${sqlWhere};`; + if (null != optEndIndex) { + sqlParam = addSqlParam(optEndIndex, values); + sqlWhere += ` AND change_id<${sqlParam}`; + } + if (null != opt_time) { + if (!(opt_time instanceof Date)) { + opt_time = new Date(opt_time); + } + sqlParam = addSqlParam(opt_time, values); + sqlWhere += ` AND change_date<=${sqlParam}`; + } + sqlWhere += ' ORDER BY change_id ASC'; + var sqlCommand = `SELECT * FROM ${cfgTableChanges} WHERE ${sqlWhere};`; - baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { - if (error) { - reject(error); - } else { - resolve(result); - } - }, undefined, undefined, values); + baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { + if (error) { + reject(error); + } else { + if (reservoirMaximum > 0) { + let size = Math.min(getChangesSize(result), reservoirMaximum); + let cur = limiter.incrementReservoir(-size).then((cur) => { + resolve(result); + }); + } else { + resolve(result); + } + } + }, undefined, undefined, values); + }); }); }; From 9b387b500fe2415478836eac29f228368dc6485d Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Wed, 9 Nov 2022 13:36:25 +0300 Subject: [PATCH 22/45] [bug] Fix file collection with setting FileConverter.converter.errorfiles --- FileConverter/sources/converter.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index e75e32f8..d1394c06 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -699,20 +699,20 @@ function* streamEnd(streamObj, text) { streamObj.writeStream.end(text, 'utf8'); yield utils.promiseWaitClose(streamObj.writeStream); } -function* processUploadToStorage(ctx, dir, storagePath) { +function* processUploadToStorage(ctx, dir, storagePath, opt_specialDirDst) { var list = yield utils.listObjects(dir); if (list.length < MAX_OPEN_FILES) { - yield* processUploadToStorageChunk(ctx, list, dir, storagePath); + yield* processUploadToStorageChunk(ctx, list, dir, storagePath, opt_specialDirDst); } else { for (var i = 0, j = list.length; i < j; i += MAX_OPEN_FILES) { - yield* processUploadToStorageChunk(ctx, list.slice(i, i + MAX_OPEN_FILES), dir, storagePath); + yield* processUploadToStorageChunk(ctx, list.slice(i, i + MAX_OPEN_FILES), dir, storagePath, opt_specialDirDst); } } } -function* processUploadToStorageChunk(ctx, list, dir, storagePath) { +function* processUploadToStorageChunk(ctx, list, dir, storagePath, opt_specialDirDst) { yield Promise.all(list.map(function (curValue) { let localValue = storagePath + '/' + curValue.substring(dir.length + 1); - return storage.uploadObject(ctx, localValue, curValue); + return storage.uploadObject(ctx, localValue, curValue, opt_specialDirDst); })); } function writeProcessOutputToLog(ctx, childRes, isDebug) { From c00a9f83a8f59cd2830ac5ef86b27042dc822d99 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Thu, 10 Nov 2022 19:42:11 +0300 Subject: [PATCH 23/45] [log] Add log message --- DocService/sources/baseConnector.js | 1 + 1 file changed, 1 insertion(+) diff --git a/DocService/sources/baseConnector.js b/DocService/sources/baseConnector.js index b5fbad97..74e92efe 100644 --- a/DocService/sources/baseConnector.js +++ b/DocService/sources/baseConnector.js @@ -221,6 +221,7 @@ exports.getChangesPromise = function (ctx, docId, optStartIndex, optEndIndex, op if (reservoirMaximum > 0) { let size = Math.min(getChangesSize(result), reservoirMaximum); let cur = limiter.incrementReservoir(-size).then((cur) => { + ctx.logger.debug("getChangesPromise bottleneck reservoir cur=%s", cur); resolve(result); }); } else { From 1feebf0bcf27cc1e60dc2627e8a2264ae8e45b42 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Fri, 11 Nov 2022 13:07:08 +0300 Subject: [PATCH 24/45] [bug] Fix bug when assembling "force save" files --- FileConverter/sources/converter.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index d1394c06..ba06c95e 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -283,7 +283,7 @@ function getTempDir() { return {temp: newTemp, source: sourceDir, result: resultDir}; } function* isUselessConvertion(ctx, task, cmd) { - if (task.getFromChanges()) { + if (task.getFromChanges() && 'sfc' === cmd.getCommand()) { let selectRes = yield taskResult.select(ctx, cmd.getDocId()); let row = selectRes.length > 0 ? selectRes[0] : null; if (utils.isUselesSfc(row, cmd)) { From e1ddab132fa794ed95dfa2088cae3cacec1e06d7 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Wed, 16 Nov 2022 14:22:30 +0300 Subject: [PATCH 25/45] [feature] Replace 'changesError' with 'clientLog' command --- DocService/sources/DocsCoServer.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 9e5a8a24..72c5d039 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -1426,9 +1426,12 @@ exports.install = function(server, callbackFunction) { yield canvasService.openDocument(ctx, conn, cmd); break; } - case 'changesError': - ctx.logger.error("changesError: %s", data.stack); - if (cfgErrorFiles && docId) { + case 'clientLog': + let level = data.level?.toLowerCase(); + if("trace" === level || "debug" === level || "info" === level || "warn" === level || "error" === level || "fatal" === level) { + ctx.logger[level]("clientLog: %s", data.msg); + } + if ("error" === level && cfgErrorFiles && docId) { let destDir = 'browser/' + docId; yield storage.copyPath(ctx, docId, destDir, undefined, cfgErrorFiles); yield* saveErrorChanges(ctx, docId, destDir); @@ -2168,6 +2171,7 @@ exports.install = function(server, callbackFunction) { function* auth(ctx, conn, data) { //TODO: Do authorization etc. check md5 or query db + ctx.logger.debug('auth time: %d', data.time); if (data.token && data.user) { ctx.setUserId(data.user.id); let licenseInfo = yield tenantManager.getTenantLicense(ctx); From 17e2797e7549da3148d04a5d9edcc5a82a3195be Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Wed, 16 Nov 2022 17:47:59 +0300 Subject: [PATCH 26/45] [wopi] Fix crash with postgres<9.5 --- DocService/sources/DocsCoServer.js | 10 ++++++---- DocService/sources/canvasservice.js | 4 ++-- DocService/sources/wopiClient.js | 25 ++++++++++++++++--------- FileConverter/sources/converter.js | 11 ++++++----- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 72c5d039..e3995455 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -1100,7 +1100,9 @@ let unlockWopiDoc = co.wrap(function*(ctx, docId, opt_userIndex) { if (getRes && getRes.wopiParams && getRes.wopiParams.userAuth && 'view' !== getRes.wopiParams.userAuth.mode) { yield wopiClient.unlock(ctx, getRes.wopiParams); let unlockInfo = wopiClient.getWopiUnlockMarker(getRes.wopiParams); - yield canvasService.commandOpenStartPromise(ctx, docId, undefined, true, unlockInfo); + if (unlockInfo) { + yield canvasService.commandOpenStartPromise(ctx, docId, undefined, true, unlockInfo); + } } }); function* cleanDocumentOnExit(ctx, docId, deleteChanges, opt_userIndex) { @@ -2345,8 +2347,8 @@ exports.install = function(server, callbackFunction) { cmd.setWopiParams(wopiParams); if (wopiParams) { documentCallback = null; - if (!wopiParams.userAuth) { - yield* sendFileErrorAuth(ctx, conn, data.sessionId, 'Wopi without userAuth'); + if (!wopiParams.userAuth || !wopiParams.commonInfo) { + yield* sendFileErrorAuth(ctx, conn, data.sessionId, `invalid wopi callback (maybe postgres<9.5) ${JSON.stringify(wopiParams)}`); return; } } @@ -3505,7 +3507,7 @@ exports.install = function(server, callbackFunction) { let callback = selectRes[0].callback; let callbackUrl = sqlBase.UserCallback.prototype.getCallbackByUserIndex(ctx, callback); let wopiParams = wopiClient.parseWopiCallback(ctx, callbackUrl, callback); - if (wopiParams) { + if (wopiParams && wopiParams.commonInfo) { yield wopiClient.lock(ctx, 'REFRESH_LOCK', wopiParams.commonInfo.lockId, wopiParams.commonInfo.fileInfo, wopiParams.userAuth); } diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index c36839e9..6567f4ec 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -617,7 +617,7 @@ let commandSfctByCmd = co.wrap(function*(ctx, cmd, opt_priority, opt_expiration, yield* addRandomKeyTaskCmd(ctx, cmd); addPasswordToCmd(ctx, cmd, row.password); let userAuthStr = sqlBase.UserCallback.prototype.getCallbackByUserIndex(ctx, row.callback); - cmd.setWopiParams(wopiClient.parseWopiCallback(ctx, userAuthStr)); + cmd.setWopiParams(wopiClient.parseWopiCallback(ctx, userAuthStr, row.callback)); cmd.setOutputFormat(changeFormatByOrigin(ctx, row, cmd.getOutputFormat())); cmd.setJsonParams(getOpenedAtJSONParams(row)); var queueData = getSaveTask(ctx, cmd); @@ -1602,7 +1602,7 @@ exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId cmd.setUserActionIndex(opt_userIndex); cmd.setJsonParams(getOpenedAtJSONParams(row)); let userAuthStr = sqlBase.UserCallback.prototype.getCallbackByUserIndex(ctx, row.callback); - cmd.setWopiParams(wopiClient.parseWopiCallback(ctx, userAuthStr)); + cmd.setWopiParams(wopiClient.parseWopiCallback(ctx, userAuthStr, row.callback)); addPasswordToCmd(ctx, cmd, row && row.password); yield* addRandomKeyTaskCmd(ctx, cmd); var queueData = getSaveTask(ctx, cmd); diff --git a/DocService/sources/wopiClient.js b/DocService/sources/wopiClient.js index 437c40d6..b87a2693 100644 --- a/DocService/sources/wopiClient.js +++ b/DocService/sources/wopiClient.js @@ -235,6 +235,9 @@ function isWopiModifiedMarker(url) { } } function getWopiUnlockMarker(wopiParams) { + if (!wopiParams.userAuth || !wopiParams.commonInfo) { + return; + } return JSON.stringify(Object.assign({unlockId: wopiParams.commonInfo.lockId}, wopiParams.userAuth)); } function getWopiModifiedMarker(wopiParams, lastModifiedTime) { @@ -265,10 +268,14 @@ function parseWopiCallback(ctx, userAuthStr, opt_url) { let commonInfoStr = sqlBase.UserCallback.prototype.getCallbackByUserIndex(ctx, opt_url, 1); if (isWopiCallback(commonInfoStr)) { commonInfo = JSON.parse(commonInfoStr); - lastModifiedTime = commonInfo.fileInfo.LastModifiedTime; - if (lastModifiedTime) { - let callbacks = sqlBase.UserCallback.prototype.getCallbacks(ctx, opt_url); - lastModifiedTime = getLastModifiedTimeFromCallbacks(callbacks); + if (commonInfo.fileInfo) { + lastModifiedTime = commonInfo.fileInfo.LastModifiedTime; + if (lastModifiedTime) { + let callbacks = sqlBase.UserCallback.prototype.getCallbacks(ctx, opt_url); + lastModifiedTime = getLastModifiedTimeFromCallbacks(callbacks); + } + } else { + commonInfo = null; } } } @@ -488,7 +495,7 @@ function putFile(ctx, wopiParams, data, dataStream, dataSize, userLastChangeId, let postRes = null; try { ctx.logger.info('wopi PutFile start'); - if (!wopiParams.userAuth) { + if (!wopiParams.userAuth || !wopiParams.commonInfo) { return postRes; } let fileInfo = wopiParams.commonInfo.fileInfo; @@ -561,7 +568,7 @@ function renameFile(ctx, wopiParams, name) { let res = undefined; try { ctx.logger.info('wopi RenameFile start'); - if (!wopiParams.userAuth) { + if (!wopiParams.userAuth || !wopiParams.commonInfo) { return res; } let fileInfo = wopiParams.commonInfo.fileInfo; @@ -665,11 +672,11 @@ function unlock(ctx, wopiParams) { return co(function* () { try { ctx.logger.info('wopi Unlock start'); + if (!wopiParams.userAuth || !wopiParams.commonInfo) { + return; + } let fileInfo = wopiParams.commonInfo.fileInfo; if (fileInfo && fileInfo.SupportsLocks) { - if (!wopiParams.userAuth) { - return; - } let wopiSrc = wopiParams.userAuth.wopiSrc; let lockId = wopiParams.commonInfo.lockId; let access_token = wopiParams.userAuth.access_token; diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index ba06c95e..cd18afde 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -896,16 +896,17 @@ function* ExecuteTask(ctx, task) { if (wopiParams) { withAuthorization = false; filterPrivate = false; - let fileInfo = wopiParams.commonInfo.fileInfo; + let fileInfo = wopiParams.commonInfo?.fileInfo; let userAuth = wopiParams.userAuth; - fileSize = fileInfo.Size; - if (fileInfo.FileUrl) { + fileSize = fileInfo?.Size; + if (fileInfo?.FileUrl) { + //Requests to the FileUrl can not be signed using proof keys. The FileUrl is used exactly as provided by the host, so it does not necessarily include the access token, which is required to construct the expected proof. url = fileInfo.FileUrl; - } else if (fileInfo.TemplateSource) { + } else if (fileInfo?.TemplateSource) { url = fileInfo.TemplateSource; } else if (userAuth) { url = `${userAuth.wopiSrc}/contents?access_token=${userAuth.access_token}`; - headers = {'X-WOPI-MaxExpectedSize': cfgDownloadMaxBytes, 'X-WOPI-ItemVersion': fileInfo.Version}; + headers = {'X-WOPI-MaxExpectedSize': cfgDownloadMaxBytes}; wopiClient.fillStandardHeaders(headers, url, userAuth.access_token); } ctx.logger.debug('wopi url=%s; headers=%j', url, headers); From 0e52b68078e663fee7722643749310846c13b46d Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Thu, 17 Nov 2022 18:02:26 +0300 Subject: [PATCH 27/45] [wopi] Fix 'convert-to' options: PDFVer, FullSheetPreview, lang --- DocService/sources/converterservice.js | 27 ++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/DocService/sources/converterservice.js b/DocService/sources/converterservice.js index 7bdedeba..642e2a06 100644 --- a/DocService/sources/converterservice.js +++ b/DocService/sources/converterservice.js @@ -455,14 +455,20 @@ function convertTo(req, res) { if (req.params.format) { format = req.params.format; } + let pdfVer = req.body['PDFVer']; + if (pdfVer && pdfVer.startsWith("PDF/A") && 'pdf' === format) { + format = 'pdfa'; + } + let fullSheetPreview = req.body['FullSheetPreview']; + let lang = req.body['lang']; let outputFormat = formatChecker.getFormatFromString(format); if (constants.AVS_OFFICESTUDIO_FILE_UNKNOWN === outputFormat) { ctx.logger.warn('convert-to unexpected format = %s', format); res.sendStatus(400); return; } - //todo https://github.com/CollaboraOnline/online/blob/4d6ca688f98d217866b9ea49113a356134dfba3a/wsd/COOLWSD.cpp - //req.body['options'], req.body['FullSheetPreview'], req.body['PDFVer'] + //todo https://github.com/CollaboraOnline/online/blob/master/wsd/COOLWSD.cpp + //req.body['options'] let docId, fileTo, status, originalname; if (req.file && req.file.originalname && req.file.buffer) { @@ -488,6 +494,23 @@ function convertTo(req, res) { cmd.setSaveKey(docId); cmd.setFormat(filetype); cmd.setOutputFormat(outputFormat); + if (lang && locale[lang.toLowerCase()]) { + cmd.setLCID(locale[lang.toLowerCase()].id); + } + if (fullSheetPreview) { + cmd.setJsonParams(JSON.stringify({'spreadsheetLayout': { + "ignorePrintArea": true, + "fitToWidth": 1, + "fitToHeight": 1 + }})); + } else { + cmd.setJsonParams(JSON.stringify({'spreadsheetLayout': { + "ignorePrintArea": true, + "fitToWidth": 0, + "fitToHeight": 0, + "scale": 100 + }})); + } fileTo = constants.OUTPUT_NAME; let outputExt = formatChecker.getStringFromFormat(outputFormat); From df59b28b0643894edc83b7b9613856a1c1acbaac Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Thu, 24 Nov 2022 10:16:02 +0300 Subject: [PATCH 28/45] [sql] Fix doc_changes2 creation; For bug 59826 --- schema/postgresql/createdb.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/postgresql/createdb.sql b/schema/postgresql/createdb.sql index 12b5764c..51548100 100644 --- a/schema/postgresql/createdb.sql +++ b/schema/postgresql/createdb.sql @@ -30,7 +30,7 @@ CREATE TABLE IF NOT EXISTS "public"."doc_changes2" ( "user_id" varchar(255) COLLATE "default" NOT NULL, "user_id_original" varchar(255) COLLATE "default" NOT NULL, "user_name" varchar(255) COLLATE "default" NOT NULL, -"change_data" bytea COLLATE "default" NOT NULL, +"change_data" bytea NOT NULL, "change_date" timestamp without time zone NOT NULL, PRIMARY KEY ("tenant", "id", "change_id") ) From 8126867ef85af3a832384108cb7683b0ea6b35e4 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Tue, 6 Dec 2022 14:33:40 +0300 Subject: [PATCH 29/45] [config] Increase pingInterval for development env to fix disconnection during debugging --- Common/config/development-linux.json | 6 ++++++ Common/config/development-mac.json | 6 ++++++ Common/config/development-windows.json | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/Common/config/development-linux.json b/Common/config/development-linux.json index b9f2c44e..c5c2272c 100644 --- a/Common/config/development-linux.json +++ b/Common/config/development-linux.json @@ -41,6 +41,12 @@ }, "sockjs": { "sockjs_url": "/web-apps/vendor/sockjs/sockjs.min.js" + }, + "socketio": { + "connection": { + "pingTimeout": 86400000, + "pingInterval": 86400000 + } } } }, diff --git a/Common/config/development-mac.json b/Common/config/development-mac.json index 20533ca7..701204e7 100644 --- a/Common/config/development-mac.json +++ b/Common/config/development-mac.json @@ -47,6 +47,12 @@ }, "sockjs": { "sockjs_url": "/web-apps/vendor/sockjs/sockjs.min.js" + }, + "socketio": { + "connection": { + "pingTimeout": 86400000, + "pingInterval": 86400000 + } } } }, diff --git a/Common/config/development-windows.json b/Common/config/development-windows.json index e0fa7068..5c5d0183 100644 --- a/Common/config/development-windows.json +++ b/Common/config/development-windows.json @@ -47,6 +47,12 @@ }, "sockjs": { "sockjs_url": "/web-apps/vendor/sockjs/sockjs.min.js" + }, + "socketio": { + "connection": { + "pingTimeout": 86400000, + "pingInterval": 86400000 + } } } }, From eb9d90da3a67a54a7c46d4c1c05fd02d6d541b26 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Tue, 6 Dec 2022 15:58:24 +0300 Subject: [PATCH 30/45] [wopi] Fix discovery.xml injection vulnerability --- DocService/npm-shrinkwrap.json | 71 ++++++++++++++++++++++++++++++++ DocService/package.json | 3 +- DocService/sources/wopiClient.js | 57 +++++++++++++------------ 3 files changed, 103 insertions(+), 28 deletions(-) diff --git a/DocService/npm-shrinkwrap.json b/DocService/npm-shrinkwrap.json index 2bab1536..93752ab3 100644 --- a/DocService/npm-shrinkwrap.json +++ b/DocService/npm-shrinkwrap.json @@ -4,6 +4,38 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@oozcitak/dom": { + "version": "1.15.10", + "resolved": "https://registry.npmjs.org/@oozcitak/dom/-/dom-1.15.10.tgz", + "integrity": "sha512-0JT29/LaxVgRcGKvHmSrUTEvZ8BXvZhGl2LASRUgHqDTC1M5g1pLmVv56IYNyt3bG2CUjDkc67wnyZC14pbQrQ==", + "requires": { + "@oozcitak/infra": "1.0.8", + "@oozcitak/url": "1.0.4", + "@oozcitak/util": "8.3.8" + } + }, + "@oozcitak/infra": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@oozcitak/infra/-/infra-1.0.8.tgz", + "integrity": "sha512-JRAUc9VR6IGHOL7OGF+yrvs0LO8SlqGnPAMqyzOuFZPSZSXI7Xf2O9+awQPSMXgIWGtgUf/dA6Hs6X6ySEaWTg==", + "requires": { + "@oozcitak/util": "8.3.8" + } + }, + "@oozcitak/url": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@oozcitak/url/-/url-1.0.4.tgz", + "integrity": "sha512-kDcD8y+y3FCSOvnBI6HJgl00viO/nGbQoCINmQ0h98OhnGITrWR3bOGfwYCthgcrV8AnTJz8MzslTQbC3SOAmw==", + "requires": { + "@oozcitak/infra": "1.0.8", + "@oozcitak/util": "8.3.8" + } + }, + "@oozcitak/util": { + "version": "8.3.8", + "resolved": "https://registry.npmjs.org/@oozcitak/util/-/util-8.3.8.tgz", + "integrity": "sha512-T8TbSnGsxo6TDBJx/Sgv/BlVJL3tshxZP7Aq5R1mSnM5OcHY2dQaxLMu2+E8u3gN0MLOzdjurqN4ZRVuzQycOQ==" + }, "@socket.io/component-emitter": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@socket.io/component-emitter/-/component-emitter-3.1.0.tgz", @@ -69,6 +101,14 @@ "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz", "integrity": "sha1-HjRA6RXwsSA9I3SOeO3XubW0PlY=" }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "requires": { + "sprintf-js": "~1.0.2" + } + }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -376,6 +416,11 @@ "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" + }, "etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", @@ -609,6 +654,15 @@ "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, + "js-yaml": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.0.tgz", + "integrity": "sha512-/4IbIeHcD9VMHFqDR/gQ7EdZdLimOvW2DdcxFjdyyZ9NsbS+ccrXqVWDtab/lRl5AlUqmpBx8EhPaWR+OtY17A==", + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, "json-schema-traverse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", @@ -1260,6 +1314,11 @@ } } }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" + }, "statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", @@ -1356,6 +1415,18 @@ "resolved": "https://registry.npmjs.org/ws/-/ws-8.2.3.tgz", "integrity": "sha512-wBuoj1BDpC6ZQ1B7DWQBYVLphPWkm8i9Y0/3YdHjHKHiohOJ1ws+3OccDWtH+PoC9DZD5WOTrJvNbWvjS6JWaA==" }, + "xmlbuilder2": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/xmlbuilder2/-/xmlbuilder2-3.0.2.tgz", + "integrity": "sha512-h4MUawGY21CTdhV4xm3DG9dgsqyhDkZvVJBx88beqX8wJs3VgyGQgAn5VreHuae6unTQxh115aMK5InCVmOIKw==", + "requires": { + "@oozcitak/dom": "1.15.10", + "@oozcitak/infra": "1.0.8", + "@oozcitak/util": "8.3.8", + "@types/node": "*", + "js-yaml": "3.14.0" + } + }, "xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", diff --git a/DocService/package.json b/DocService/package.json index c1d3cff9..2cfc489c 100644 --- a/DocService/package.json +++ b/DocService/package.json @@ -37,7 +37,8 @@ "socket.io": "^4.5.2", "underscore": "^1.13.1", "utf7": "^1.0.2", - "windows-locale": "^1.0.1" + "windows-locale": "^1.0.1", + "xmlbuilder2": "^3.0.2" }, "pkg": { "scripts": [ diff --git a/DocService/sources/wopiClient.js b/DocService/sources/wopiClient.js index b87a2693..bfd4a5c3 100644 --- a/DocService/sources/wopiClient.js +++ b/DocService/sources/wopiClient.js @@ -40,6 +40,7 @@ const jwt = require('jsonwebtoken'); const config = require('config'); const utf7 = require('utf7'); const mimeDB = require('mime-db'); +const xmlbuilder2 = require('xmlbuilder2'); const logger = require('./../../Common/sources/logger'); const utils = require('./../../Common/sources/utils'); const constants = require('./../../Common/sources/constants'); @@ -100,6 +101,7 @@ let mimeTypesByExt = (function() { function discovery(req, res) { return co(function*() { let output = ''; + const xml = xmlbuilder2.create({version: '1.0', encoding: 'utf-8'}); let ctx = new operationContext.Context(); try { ctx.initFromRequest(req); @@ -119,7 +121,7 @@ function discovery(req, res) { templateEnd += `<sc=SESSION_CONTEXT&><thm=THEME_ID&><ui=UI_LLCC&>`; templateEnd += `<wopisrc=WOPI_SOURCE&>&`; let documentTypes = [`word`, `cell`, `slide`]; - output += ``; + let xmlZone = xml.ele('wopi-discovery').ele('net-zone', { name: cfgWopiWopiZone }); //start section for MS WOPI connectors for(let i = 0; i < names.length; ++i) { let name = names[i]; @@ -131,25 +133,25 @@ function discovery(req, res) { let urlTemplateView = `${templateStart}/${documentTypes[i]}/view?${templateEnd}`; let urlTemplateEmbedView = `${templateStart}/${documentTypes[i]}/view?embed=1${templateEnd}`; let urlTemplateEdit = `${templateStart}/${documentTypes[i]}/edit?${templateEnd}`; - output +=``; + let xmlApp = xmlZone.ele('app', {name: name, favIconUrl: favIconUrl}); for (let j = 0; j < ext.view.length; ++j) { - output += ``; - output += ``; + xmlApp.ele('action', {name: 'view', ext: ext.view[j], urlsrc: urlTemplateView}).up(); + xmlApp.ele('action', {name: 'embedview', ext: ext.view[j], urlsrc: urlTemplateEmbedView}).up(); if (-1 === cfgWopiPdfView.indexOf(ext.view[j])) { let urlConvert = `${templateStart}/convert-and-edit/${ext.view[j]}/${ext.targetext}?${templateEnd}`; - output += ``; + xmlApp.ele('action', {name: 'convert', ext: ext.view[j], targetext: ext.targetext, requires: 'update', urlsrc: urlConvert}).up(); } } for (let j = 0; j < ext.edit.length; ++j) { - output += ``; - output += ``; + xmlApp.ele('action', {name: 'view', ext: ext.edit[j], urlsrc: urlTemplateView}).up(); + xmlApp.ele('action', {name: 'embedview', ext: ext.edit[j], urlsrc: urlTemplateEmbedView}).up(); if ("oform" !== ext.edit[j]) { //todo config - output += ``; + xmlApp.ele('action', {name: 'editnew', ext: ext.edit[j], requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); } - output += ``; + xmlApp.ele('action', {name: 'edit', ext: ext.edit[j], default: 'true', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); } - output +=``; + xmlApp.up(); } //end section for MS WOPI connectors //start section for collabora nexcloud connectors @@ -162,14 +164,14 @@ function discovery(req, res) { let mimeTypes = mimeTypesByExt[ext.view[j]]; if (mimeTypes) { mimeTypes.forEach((value) => { - output += ``; - output += ``; - output += ``; + let xmlApp = xmlZone.ele('app', {name: value}); + xmlApp.ele('action', {name: 'view', ext: '', default: 'true', urlsrc: urlTemplateView}).up(); + xmlApp.ele('action', {name: 'embedview', ext: '', urlsrc: urlTemplateEmbedView}).up(); if (-1 === cfgWopiPdfView.indexOf(ext.view[j])) { let urlConvert = `${templateStart}/convert-and-edit/${ext.view[j]}/${ext.targetext}?${templateEnd}`; - output += ``; + xmlApp.ele('action', {name: 'convert', ext: '', targetext: ext.targetext, requires: 'update', urlsrc: urlConvert}).up(); } - output += ``; + xmlApp.up(); }); } } @@ -177,29 +179,30 @@ function discovery(req, res) { let mimeTypes = mimeTypesByExt[ext.edit[j]]; if (mimeTypes) { mimeTypes.forEach((value) => { - output +=``; - output += ``; - output +=``; + let xmlApp = xmlZone.ele('app', {name: value}); + xmlApp.ele('action', {name: 'edit', ext: '', default: 'true', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); + xmlApp.up(); }); } } } - output += ``; - output += ``; - output += ``; + let xmlApp = xmlZone.ele('app', {name: 'Capabilities'}); + xmlApp.ele('action', {ext: '', name: 'getinfo', urlsrc: 'locks,update', urlsrc: `${baseUrl}/hosting/capabilities`}).up(); + xmlApp.up(); //end section for collabora nexcloud connectors - let proofKey = ``; + let xmlDiscovery = xmlZone.up(); if (cfgWopiPublicKeyOld && cfgWopiPublicKey) { - proofKey += ``; + xmlDiscovery.ele('proof-key', { + oldvalue: cfgWopiPublicKeyOld, oldmodulus: cfgWopiModulusOld, oldexponent: cfgWopiExponentOld, + value: cfgWopiPublicKey, modulus: cfgWopiModulus, exponent: cfgWopiExponent + }).up(); } - output += `${proofKey}`; + xmlDiscovery.up(); } catch (err) { ctx.logger.error('wopiDiscovery error:%s', err.stack); } finally { res.setHeader('Content-Type', 'text/xml'); - res.send(output); + res.send(xml.end()); ctx.logger.info('wopiDiscovery end'); } }); From 53853debfc80275955eabc0a7592f25fdc464eac Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Tue, 6 Dec 2022 16:21:44 +0300 Subject: [PATCH 31/45] [bug] Allow only http and https as x-forwarded-proto values --- Common/sources/constants.js | 1 + Common/sources/utils.js | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Common/sources/constants.js b/Common/sources/constants.js index 17c5f889..b9c29069 100644 --- a/Common/sources/constants.js +++ b/Common/sources/constants.js @@ -49,6 +49,7 @@ exports.CHANGES_NAME = 'changes'; exports.VIEWER_ONLY = /^(?:(pdf|djvu|xps|oxps))$/; exports.DEFAULT_DOC_ID = 'docId'; exports.DEFAULT_USER_ID = 'userId'; +exports.ALLOWED_PROTO = /^https?$/i; exports.RIGHTS = { None : 0, diff --git a/Common/sources/utils.js b/Common/sources/utils.js index 66ff3e94..e510fa8d 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -669,9 +669,9 @@ function getDomain(hostHeader, forwardedHostHeader) { }; function getBaseUrl(protocol, hostHeader, forwardedProtoHeader, forwardedHostHeader, forwardedPrefixHeader) { var url = ''; - if (forwardedProtoHeader) { + if (forwardedProtoHeader && constants.ALLOWED_PROTO.test(forwardedProtoHeader)) { url += forwardedProtoHeader; - } else if (protocol) { + } else if (protocol && constants.ALLOWED_PROTO.test(protocol)) { url += protocol; } else { url += 'http'; From 2f3883adbcbeb76f6f0c9c099a4645920c6791f7 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Wed, 7 Dec 2022 16:25:33 +0300 Subject: [PATCH 32/45] [npm] Bump rhea version --- Common/npm-shrinkwrap.json | 12 ++++++------ Common/package.json | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Common/npm-shrinkwrap.json b/Common/npm-shrinkwrap.json index 8023ddf1..62104bf1 100644 --- a/Common/npm-shrinkwrap.json +++ b/Common/npm-shrinkwrap.json @@ -228,9 +228,9 @@ "integrity": "sha512-7P3FyqDcfeznLZp2b+OMitV9Sz2lUnsT87WaTat9nVwqsBkTzPG3lPLNwW3en6F4pHUiWzr6vb8CLhjdK9bcxQ==" }, "debug": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", - "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", "requires": { "ms": "^2.1.1" } @@ -782,9 +782,9 @@ "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==" }, "rhea": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/rhea/-/rhea-0.3.9.tgz", - "integrity": "sha512-16mqaARtj9ZgiYmD5F9uwcgvy5pOB64mWIbkBgrje4zEByIBjHdYB25EyeWb3baiI9OYAYMWb2rnqBfyv5weOg==", + "version": "1.0.24", + "resolved": "https://registry.npmjs.org/rhea/-/rhea-1.0.24.tgz", + "integrity": "sha512-PEl62U2EhxCO5wMUZ2/bCBcXAVKN9AdMSNQOrp3+R5b77TEaOSiy16MQ0sIOmzj/iqsgIAgPs1mt3FYfu1vIXA==", "requires": { "debug": "0.8.0 - 3.5.0" } diff --git a/Common/package.json b/Common/package.json index f90e0e4e..9070f933 100644 --- a/Common/package.json +++ b/Common/package.json @@ -24,7 +24,7 @@ "openpgp": "^4.10.8", "request": "^2.88.0", "request-filtering-agent": "^1.0.5", - "rhea": "^0.3.9", + "rhea": "^1.0.24", "uri-js": "^4.2.2", "win-ca": "^3.5.0" } From 077fc48501e5aeaa9f51978a3040bfc6878c8101 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Thu, 8 Dec 2022 18:41:15 +0300 Subject: [PATCH 33/45] [feature] Allow CloudFront-Forwarded-Proto header --- Common/sources/utils.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Common/sources/utils.js b/Common/sources/utils.js index e510fa8d..6fc7edbe 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -685,10 +685,12 @@ function getBaseUrl(protocol, hostHeader, forwardedProtoHeader, forwardedHostHea } function getBaseUrlByConnection(conn) { conn = conn.request; - return getBaseUrl('', conn.headers['host'], conn.headers['x-forwarded-proto'], conn.headers['x-forwarded-host'], conn.headers['x-forwarded-prefix']); + let proto = conn.headers['x-forwarded-proto'] || conn.headers['cloudfront-forwarded-proto']; + return getBaseUrl('', conn.headers['host'], proto, conn.headers['x-forwarded-host'], conn.headers['x-forwarded-prefix']); } function getBaseUrlByRequest(req) { - return getBaseUrl(req.protocol, req.get('host'), req.get('x-forwarded-proto'), req.get('x-forwarded-host'), req.get('x-forwarded-prefix')); + let proto = req.get('x-forwarded-proto') || req.get('cloudfront-forwarded-proto'); + return getBaseUrl(req.protocol, req.get('host'), proto, req.get('x-forwarded-host'), req.get('x-forwarded-prefix')); } exports.getBaseUrlByConnection = getBaseUrlByConnection; exports.getBaseUrlByRequest = getBaseUrlByRequest; From 73972657bec1792751abccd0b2dbe5e3db075d62 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Sun, 11 Dec 2022 16:43:37 +0300 Subject: [PATCH 34/45] [feature] Add capability option for open_sender, open_receiver calls --- Common/sources/activeMQCore.js | 10 +---- Common/sources/taskqueueRabbitMQ.js | 55 +++++++++++++++++++++++++--- DocService/sources/pubsubRabbitMQ.js | 19 +++++++++- 3 files changed, 68 insertions(+), 16 deletions(-) diff --git a/Common/sources/activeMQCore.js b/Common/sources/activeMQCore.js index 26c37475..ccbed752 100644 --- a/Common/sources/activeMQCore.js +++ b/Common/sources/activeMQCore.js @@ -75,19 +75,13 @@ function connetPromise(reconnectOnConnectionError, closeCallback) { startConnect(); }); } -function openSenderPromise(conn, name) { +function openSenderPromise(conn, options) { return new Promise(function(resolve, reject) { - let options = {target: name}; resolve(conn.open_sender(options)); }); } -function openReceiverPromise(conn, name, autoaccept) { +function openReceiverPromise(conn, options) { return new Promise(function(resolve, reject) { - let options = {source: name}; - if (!autoaccept) { - options.credit_window = 0; - options.autoaccept = false; - } resolve(conn.open_receiver(options)); }); } diff --git a/Common/sources/taskqueueRabbitMQ.js b/Common/sources/taskqueueRabbitMQ.js index 9d7091a4..2cb3b623 100644 --- a/Common/sources/taskqueueRabbitMQ.js +++ b/Common/sources/taskqueueRabbitMQ.js @@ -176,13 +176,34 @@ function initActive(taskqueue, isAddTask, isAddResponse, isAddTaskReceive, isAdd }); taskqueue.connection = conn; if (isAddTask) { - taskqueue.channelConvertTask = yield activeMQCore.openSenderPromise(conn, cfgActiveQueueConvertTask); + //https://github.com/amqp/rhea/issues/251#issuecomment-535076570 + let optionsConvertTask = { + target: { + address: cfgActiveQueueConvertTask, + capabilities: ['queue'] + } + }; + taskqueue.channelConvertTask = yield activeMQCore.openSenderPromise(conn, optionsConvertTask); } if (isAddResponse) { - taskqueue.channelConvertResponse = yield activeMQCore.openSenderPromise(conn, cfgActiveQueueConvertResponse); + let optionsConvertResponse = { + target: { + address: cfgActiveQueueConvertResponse, + capabilities: ['queue'] + } + }; + taskqueue.channelConvertResponse = yield activeMQCore.openSenderPromise(conn, optionsConvertResponse); } if (isAddTaskReceive) { - let receiver = yield activeMQCore.openReceiverPromise(conn, cfgActiveQueueConvertTask, false); + let optionsConvertTask = { + source: { + address: cfgActiveQueueConvertTask, + capabilities: ['queue'] + }, + credit_window: 0, + autoaccept: false + }; + let receiver = yield activeMQCore.openReceiverPromise(conn, optionsConvertTask); //todo ?consumer.dispatchAsync=false&consumer.prefetchSize=1 receiver.add_credit(1); receiver.on("message", function(context) { @@ -202,7 +223,15 @@ function initActive(taskqueue, isAddTask, isAddResponse, isAddTaskReceive, isAdd taskqueue.channelConvertTaskReceive = receiver; } if (isAddResponseReceive) { - let receiver = yield activeMQCore.openReceiverPromise(conn, cfgActiveQueueConvertResponse, false); + let optionsConvertResponse = { + source: { + address: cfgActiveQueueConvertResponse, + capabilities: ['queue'] + }, + credit_window: 0, + autoaccept: false + }; + let receiver = yield activeMQCore.openReceiverPromise(conn, optionsConvertResponse); //todo ?consumer.dispatchAsync=false&consumer.prefetchSize=1 receiver.add_credit(1); receiver.on("message", function(context) { @@ -216,10 +245,24 @@ function initActive(taskqueue, isAddTask, isAddResponse, isAddTaskReceive, isAdd taskqueue.channelConvertResponseReceive = receiver; } if (isAddDelayed) { - taskqueue.channelDelayed = yield activeMQCore.openSenderPromise(conn, cfgActiveQueueDelayed); + let optionsDelayed = { + target: { + address: cfgActiveQueueDelayed, + capabilities: ['queue'] + } + }; + taskqueue.channelDelayed = yield activeMQCore.openSenderPromise(conn, optionsDelayed); } if (isEmitDead) { - let receiver = yield activeMQCore.openReceiverPromise(conn, cfgActiveQueueConvertDead, false); + let optionsConvertDead = { + source: { + address: cfgActiveQueueConvertDead, + capabilities: ['queue'] + }, + credit_window: 0, + autoaccept: false + }; + let receiver = yield activeMQCore.openReceiverPromise(conn, optionsConvertDead); //todo ?consumer.dispatchAsync=false&consumer.prefetchSize=1 receiver.add_credit(1); receiver.on("message", function(context) { diff --git a/DocService/sources/pubsubRabbitMQ.js b/DocService/sources/pubsubRabbitMQ.js index f37a5ab2..746230a1 100644 --- a/DocService/sources/pubsubRabbitMQ.js +++ b/DocService/sources/pubsubRabbitMQ.js @@ -92,9 +92,24 @@ function initActive(pubsub, callback) { } }); pubsub.connection = conn; - pubsub.channelPublish = yield activeMQCore.openSenderPromise(conn, cfgActiveTopicPubSub); + //https://github.com/amqp/rhea/issues/251#issuecomment-535076570 + let optionsPubSubSender = { + target: { + address: cfgActiveTopicPubSub, + capabilities: ['topic'] + } + }; + pubsub.channelPublish = yield activeMQCore.openSenderPromise(conn, optionsPubSubSender); - let receiver = yield activeMQCore.openReceiverPromise(conn, cfgActiveTopicPubSub, false); + let optionsPubSubReceiver = { + source: { + address: cfgActiveTopicPubSub, + capabilities: ['topic'] + }, + credit_window: 0, + autoaccept: false + }; + let receiver = yield activeMQCore.openReceiverPromise(conn, optionsPubSubReceiver); //todo ?consumer.dispatchAsync=false&consumer.prefetchSize=1 receiver.add_credit(1); receiver.on("message", function(context) { From 43fceef3cb636261882543ed42858074cc148204 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Sun, 11 Dec 2022 16:48:17 +0300 Subject: [PATCH 35/45] [comment] Add comments to getBaseUrl --- Common/sources/utils.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Common/sources/utils.js b/Common/sources/utils.js index 6fc7edbe..9b67c04c 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -685,10 +685,12 @@ function getBaseUrl(protocol, hostHeader, forwardedProtoHeader, forwardedHostHea } function getBaseUrlByConnection(conn) { conn = conn.request; + //Header names are lower-cased. https://nodejs.org/api/http.html#messageheaders let proto = conn.headers['x-forwarded-proto'] || conn.headers['cloudfront-forwarded-proto']; return getBaseUrl('', conn.headers['host'], proto, conn.headers['x-forwarded-host'], conn.headers['x-forwarded-prefix']); } function getBaseUrlByRequest(req) { + //case-insensitive match. https://expressjs.com/en/api.html#req.get let proto = req.get('x-forwarded-proto') || req.get('cloudfront-forwarded-proto'); return getBaseUrl(req.protocol, req.get('host'), proto, req.get('x-forwarded-host'), req.get('x-forwarded-prefix')); } From 5bef7e484fef2cdb2db4e6c1d1d9804241609470 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 12 Dec 2022 17:56:04 +0300 Subject: [PATCH 36/45] [bug] Fix after merge of hotfix --- DocService/sources/DocsCoServer.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 846d9cc8..01e97cc4 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -1443,12 +1443,6 @@ exports.install = function(server, callbackFunction) { conn.sessionIsSendWarning = false; conn.sessionTimeLastAction = new Date().getTime() - data.idletime; break; - case 'clientLog': - let level = data.level?.toLowerCase(); - if("trace" === level || "debug" === level || "info" === level || "warn" === level || "error" === level || "fatal" === level) { - ctx.logger[level]("clientLog: %s", data.msg); - } - break; case 'forceSaveStart' : var forceSaveRes; if (conn.user) { From d38fd4d640c80dd93c2369e78508a2837bc40a55 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 12 Dec 2022 19:52:30 +0300 Subject: [PATCH 37/45] [bug] Fix bug in cleanDocumentOnExit --- DocService/sources/DocsCoServer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 01e97cc4..1291bf98 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -1110,7 +1110,7 @@ function* cleanDocumentOnExit(ctx, docId, deleteChanges, opt_userIndex) { yield editorData.cleanDocumentOnExit(ctx, docId); //remove changes if (deleteChanges) { - yield taskResult.restoreInitialPassword(ctx.tenant, docId); + yield taskResult.restoreInitialPassword(ctx, docId); sqlBase.deleteChanges(ctx, docId, null); //delete forgotten after successful send on callbackUrl yield storage.deletePath(ctx, docId, cfgForgottenFiles); From 9d5944f0fdee921b5066166ff92d2ef38470c1ee Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Fri, 16 Dec 2022 16:30:36 +0300 Subject: [PATCH 38/45] [bug] Change conn.close to conn.disconnect --- DocService/sources/DocsCoServer.js | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 1291bf98..4322ffc8 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -1036,7 +1036,7 @@ function closeUsersConnection(docId, usersMap, isOriginalId, code, description) conn = connections[i]; if (conn.docId === docId) { if (isOriginalId ? usersMap[conn.user.idOriginal] : usersMap[conn.user.id]) { - conn.close(code, description); + conn.disconnect(code, description); } } } @@ -1370,13 +1370,13 @@ exports.install = function(server, callbackFunction) { if (conn.isCiriticalError && ('message' == data.type || 'getLock' == data.type || 'saveChanges' == data.type || 'isSaveLock' == data.type)) { ctx.logger.warn("conn.isCiriticalError send command: type = %s", data.type); - conn.close(constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); + conn.disconnect(constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); return; } if ((conn.isCloseCoAuthoring || (conn.user && conn.user.view)) && ('getLock' == data.type || 'saveChanges' == data.type || 'isSaveLock' == data.type)) { ctx.logger.warn("conn.user.view||isCloseCoAuthoring access deny: type = %s", data.type); - conn.close(constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); + conn.disconnect(constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); return; } yield encryptPasswordParams(ctx, data); @@ -1386,7 +1386,7 @@ exports.install = function(server, callbackFunction) { yield* auth(ctx, conn, data); } catch(err){ ctx.logger.error('auth error: %s', err.stack); - conn.close(constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); + conn.disconnect(constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); return; } break; @@ -2197,7 +2197,7 @@ exports.install = function(server, callbackFunction) { fillDataFromJwtRes = fillDataFromJwt(ctx, decoded, data); } else if (cfgTokenRequiredParams) { ctx.logger.error("auth missing required parameter %s (since 7.1 version)", validationErr); - conn.close(constants.JWT_ERROR_CODE, constants.JWT_ERROR_REASON); + conn.disconnect(constants.JWT_ERROR_CODE, constants.JWT_ERROR_REASON); return; } else { ctx.logger.warn("auth missing required parameter %s (since 7.1 version)", validationErr); @@ -2206,11 +2206,11 @@ exports.install = function(server, callbackFunction) { } if(!fillDataFromJwtRes) { ctx.logger.warn("fillDataFromJwt return false"); - conn.close(constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); + conn.disconnect(constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); return; } } else { - conn.close(checkJwtRes.code, checkJwtRes.description); + conn.disconnect(checkJwtRes.code, checkJwtRes.description); return; } } @@ -2239,7 +2239,7 @@ exports.install = function(server, callbackFunction) { let filterStatus = yield* utils.checkHostFilter(ctx, documentCallback.hostname); if (0 !== filterStatus) { ctx.logger.warn('checkIpFilter error: url = %s', data.documentCallbackUrl); - conn.close(constants.DROP_CODE, constants.DROP_REASON); + conn.disconnect(constants.DROP_CODE, constants.DROP_REASON); return; } } @@ -3429,8 +3429,7 @@ exports.install = function(server, callbackFunction) { }); } else if (nowMs - conn.sessionTimeConnect > cfgExpSessionAbsolute) { ctx.logger.debug('expireDoc close absolute session'); - conn.close(constants.SESSION_ABSOLUTE_CODE, constants.SESSION_ABSOLUTE_REASON); - continue; + conn.disconnect(constants.SESSION_ABSOLUTE_CODE, constants.SESSION_ABSOLUTE_REASON); continue; } } if (cfgExpSessionIdle > 0 && !(conn.user?.view || conn.isCloseCoAuthoring)) { @@ -3443,8 +3442,7 @@ exports.install = function(server, callbackFunction) { }); } else if (nowMs - conn.sessionTimeLastAction > cfgExpSessionIdle) { ctx.logger.debug('expireDoc close idle session'); - conn.close(constants.SESSION_IDLE_CODE, constants.SESSION_IDLE_REASON); - continue; + conn.disconnect(constants.SESSION_IDLE_CODE, constants.SESSION_IDLE_REASON); continue; } } if (constants.CONN_CLOSED === conn.conn.readyState) { From 90fcb6c6f95b1c214fbf62706708fff144e916a6 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Thu, 12 Jan 2023 23:08:47 +0300 Subject: [PATCH 39/45] [npm] Bump jsonwebtoken version to 9.0.0; to fix vulnerability --- Common/npm-shrinkwrap.json | 99 +++++++++++++++----------------------- Common/package.json | 2 +- 2 files changed, 41 insertions(+), 60 deletions(-) diff --git a/Common/npm-shrinkwrap.json b/Common/npm-shrinkwrap.json index 62104bf1..00f01e26 100644 --- a/Common/npm-shrinkwrap.json +++ b/Common/npm-shrinkwrap.json @@ -163,7 +163,7 @@ "buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", - "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" }, "buffer-more-ints": { "version": "1.0.0", @@ -259,9 +259,9 @@ } }, "ecdsa-sig-formatter": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.10.tgz", - "integrity": "sha1-HFlQAPBKiJffuFAAiSoPTDOvhsM=", + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", "requires": { "safe-buffer": "^5.0.1" } @@ -445,19 +445,14 @@ } }, "jsonwebtoken": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.3.0.tgz", - "integrity": "sha512-oge/hvlmeJCH+iIz1DwcO7vKPkNGJHhgkspk8OH3VKlw+mbi42WtD4ig1+VXRln765vxptAv+xT26Fd3cteqag==", + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz", + "integrity": "sha512-tuGfYXxkQGDPnLJ7SibiQgVgeDgfbPq2k2ICcbgqW8WxWLBAxKQM/ZCu/IT8SOSwmaYl4dpTFCW5xZv7YbbWUw==", "requires": { - "jws": "^3.1.5", - "lodash.includes": "^4.3.0", - "lodash.isboolean": "^3.0.3", - "lodash.isinteger": "^4.0.4", - "lodash.isnumber": "^3.0.3", - "lodash.isplainobject": "^4.0.6", - "lodash.isstring": "^4.0.1", - "lodash.once": "^4.0.0", - "ms": "^2.1.1" + "jws": "^3.2.2", + "lodash": "^4.17.21", + "ms": "^2.1.1", + "semver": "^7.3.8" } }, "jsprim": { @@ -479,21 +474,21 @@ } }, "jwa": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.1.6.tgz", - "integrity": "sha512-tBO/cf++BUsJkYql/kBbJroKOgHWEigTKBAjjBEmrMGYd1QMBC74Hr4Wo2zCZw6ZrVhlJPvoMrkcOnlWR/DJfw==", + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", + "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", "requires": { "buffer-equal-constant-time": "1.0.1", - "ecdsa-sig-formatter": "1.0.10", + "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "jws": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/jws/-/jws-3.1.5.tgz", - "integrity": "sha512-GsCSexFADNQUr8T5HPJvayTjvPIfoyJPtLQBwn5a4WZQchcrPMPMAWcC1AzJVRDKyD6ZPROPAxgv6rfHViO4uQ==", + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", + "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", "requires": { - "jwa": "^1.1.5", + "jwa": "^1.4.1", "safe-buffer": "^5.0.1" } }, @@ -515,41 +510,6 @@ "lodash._baseclone": "~4.5.0" } }, - "lodash.includes": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", - "integrity": "sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8=" - }, - "lodash.isboolean": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", - "integrity": "sha1-bC4XHbKiV82WgC/UOwGyDV9YcPY=" - }, - "lodash.isinteger": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", - "integrity": "sha1-YZwK89A/iwTDH1iChAt3sRzWg0M=" - }, - "lodash.isnumber": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", - "integrity": "sha1-POdoEMWSjQM1IwGsKHMX8RwLH/w=" - }, - "lodash.isplainobject": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", - "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=" - }, - "lodash.isstring": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", - "integrity": "sha1-1SfftUVuynzJu5XV2ur4i6VKVFE=" - }, - "lodash.once": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", - "integrity": "sha1-DdOXEhPHxW34gJd9UEyI+0cal6w=" - }, "log4js": { "version": "6.4.1", "resolved": "https://registry.npmjs.org/log4js/-/log4js-6.4.1.tgz", @@ -577,6 +537,14 @@ } } }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "requires": { + "yallist": "^4.0.0" + } + }, "make-dir": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", @@ -804,6 +772,14 @@ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" }, + "semver": { + "version": "7.3.8", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", + "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "requires": { + "lru-cache": "^6.0.0" + } + }, "slide": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/slide/-/slide-1.1.6.tgz", @@ -1032,6 +1008,11 @@ "version": "9.0.7", "resolved": "http://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" } } } diff --git a/Common/package.json b/Common/package.json index 9070f933..303e6e28 100644 --- a/Common/package.json +++ b/Common/package.json @@ -14,7 +14,7 @@ "forwarded": "^0.1.2", "fs-extra": "^7.0.0", "ipaddr.js": "^1.8.1", - "jsonwebtoken": "^8.3.0", + "jsonwebtoken": "^9.0.0", "log4js": "^6.4.1", "mime": "^2.3.1", "mkdirp": "^0.5.1", From 33ba4bca6a83dd5395b97f8a65ef8f38ce3ba365 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Thu, 12 Jan 2023 23:35:55 +0300 Subject: [PATCH 40/45] [npm] Bump pg version for postgres 13 and later --- DocService/npm-shrinkwrap.json | 61 +++++++++++++--------------------- DocService/package.json | 2 +- 2 files changed, 24 insertions(+), 39 deletions(-) diff --git a/DocService/npm-shrinkwrap.json b/DocService/npm-shrinkwrap.json index 93752ab3..514d293b 100644 --- a/DocService/npm-shrinkwrap.json +++ b/DocService/npm-shrinkwrap.json @@ -968,23 +968,23 @@ "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" }, "pg": { - "version": "8.5.1", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.5.1.tgz", - "integrity": "sha512-9wm3yX9lCfjvA98ybCyw2pADUivyNWT/yIP4ZcDVpMN0og70BUWYEGXPCTAQdGTAqnytfRADb7NERrY1qxhIqw==", + "version": "8.8.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.8.0.tgz", + "integrity": "sha512-UXYN0ziKj+AeNNP7VDMwrehpACThH7LUl/p8TDFpEUuSejCUIwGSfxpHsPvtM6/WXFy6SU4E5RG4IJV/TZAGjw==", "requires": { "buffer-writer": "2.0.0", "packet-reader": "1.0.0", - "pg-connection-string": "^2.4.0", - "pg-pool": "^3.2.2", - "pg-protocol": "^1.4.0", + "pg-connection-string": "^2.5.0", + "pg-pool": "^3.5.2", + "pg-protocol": "^1.5.0", "pg-types": "^2.1.0", "pgpass": "1.x" } }, "pg-connection-string": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.4.0.tgz", - "integrity": "sha512-3iBXuv7XKvxeMrIgym7njT+HlZkwZqqGX4Bu9cci8xHZNT+Um1gWKqCsAzcC0d95rcKMU5WBg6YRUcHyV0HZKQ==" + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz", + "integrity": "sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==" }, "pg-int8": { "version": "1.0.1", @@ -992,14 +992,14 @@ "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" }, "pg-pool": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.2.2.tgz", - "integrity": "sha512-ORJoFxAlmmros8igi608iVEbQNNZlp89diFVx6yV5v+ehmpMY9sK6QgpmgoXbmkNaBAx8cOOZh9g80kJv1ooyA==" + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.5.2.tgz", + "integrity": "sha512-His3Fh17Z4eg7oANLob6ZvH8xIVen3phEZh2QuyrIl4dQSDVEabNducv6ysROKpDNPSD+12tONZVWfSgMvDD9w==" }, "pg-protocol": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.4.0.tgz", - "integrity": "sha512-El+aXWcwG/8wuFICMQjM5ZSAm6OWiJicFdNYo+VY3QP+8vI4SvLIWVe51PppTzMhikUJR+PsyIFKqfdXPz/yxA==" + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.5.0.tgz", + "integrity": "sha512-muRttij7H8TqRNu/DxrAJQITO4Ac7RmX3Klyr/9mJEOBeIpgnF8f9jAfRz5d3XwQZl5qBjF9gLsUtMPJE0vezQ==" }, "pg-types": { "version": "2.2.0", @@ -1014,11 +1014,11 @@ } }, "pgpass": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.4.tgz", - "integrity": "sha512-YmuA56alyBq7M59vxVBfPJrGSozru8QAdoNlWuW3cz8l+UX3cWge0vTvjKhsSHSJpo3Bom8/Mm6hf0TR5GY0+w==", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", "requires": { - "split2": "^3.1.1" + "split2": "^4.1.0" } }, "postgres-array": { @@ -1029,7 +1029,7 @@ "postgres-bytea": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", - "integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU=" + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==" }, "postgres-date": { "version": "1.0.7", @@ -1295,24 +1295,9 @@ } }, "split2": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz", - "integrity": "sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==", - "requires": { - "readable-stream": "^3.0.0" - }, - "dependencies": { - "readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - } - } + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.1.0.tgz", + "integrity": "sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ==" }, "sprintf-js": { "version": "1.0.3", diff --git a/DocService/package.json b/DocService/package.json index 2cfc489c..8c6cf390 100644 --- a/DocService/package.json +++ b/DocService/package.json @@ -31,7 +31,7 @@ "multi-integer-range": "^4.0.7", "multiparty": "^4.2.1", "mysql2": "^2.3.3", - "pg": "^8.5.1", + "pg": "^8.8.0", "redis": "^2.8.0", "retry": "^0.12.0", "socket.io": "^4.5.2", From fee5cfc4439689b825f1f5384bae080aab37aa36 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Fri, 13 Jan 2023 09:41:33 +0300 Subject: [PATCH 41/45] [npm] npm audit fix --- Common/npm-shrinkwrap.json | 6 +- DocService/npm-shrinkwrap.json | 727 ++++++++++++++++++++---------- DocService/package.json | 8 +- FileConverter/npm-shrinkwrap.json | 12 +- npm-shrinkwrap.json | 110 ++--- package.json | 2 +- 6 files changed, 554 insertions(+), 311 deletions(-) diff --git a/Common/npm-shrinkwrap.json b/Common/npm-shrinkwrap.json index 00f01e26..d17c03ff 100644 --- a/Common/npm-shrinkwrap.json +++ b/Common/npm-shrinkwrap.json @@ -429,9 +429,9 @@ "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" }, "json5": { - "version": "1.0.1", - "resolved": "http://registry.npmjs.org/json5/-/json5-1.0.1.tgz", - "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz", + "integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==", "requires": { "minimist": "^1.2.0" } diff --git a/DocService/npm-shrinkwrap.json b/DocService/npm-shrinkwrap.json index 514d293b..14e2bf5e 100644 --- a/DocService/npm-shrinkwrap.json +++ b/DocService/npm-shrinkwrap.json @@ -91,6 +91,14 @@ "uri-js": "^4.2.2" } }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "requires": { + "color-convert": "^2.0.1" + } + }, "apicache": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/apicache/-/apicache-1.6.2.tgz", @@ -112,7 +120,17 @@ "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" + }, + "async": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", + "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" + }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, "base64-stream": { "version": "1.0.0", @@ -125,26 +143,63 @@ "integrity": "sha512-lGe34o6EHj9y3Kts9R4ZYs/Gr+6N7MCaMlIFA3F1R2O5/m7K06AxfSeO5530PEERE6/WyEg3lsuyw4GHlPZHog==" }, "body-parser": { - "version": "1.18.3", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz", - "integrity": "sha1-WykhmP/dVTs6DyDe0FkrlWlVyLQ=", + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", + "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", "requires": { - "bytes": "3.0.0", + "bytes": "3.1.2", "content-type": "~1.0.4", "debug": "2.6.9", - "depd": "~1.1.2", - "http-errors": "~1.6.3", - "iconv-lite": "0.4.23", - "on-finished": "~2.3.0", - "qs": "6.5.2", - "raw-body": "2.3.3", - "type-is": "~1.6.16" + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.11.0", + "raw-body": "2.5.1", + "type-is": "~1.6.18", + "unpipe": "1.0.0" }, "dependencies": { "bytes": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", - "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" + }, + "depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" + }, + "mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" + }, + "mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "requires": { + "mime-db": "1.52.0" + } + }, + "on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "requires": { + "ee-first": "1.1.1" + } + }, + "type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + } } } }, @@ -153,6 +208,14 @@ "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==" }, + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "requires": { + "balanced-match": "^1.0.0" + } + }, "buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", @@ -205,11 +268,47 @@ "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" }, + "call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "requires": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } + }, + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, "co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=" }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + }, "concat-stream": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", @@ -230,11 +329,18 @@ } }, "content-disposition": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", - "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==", + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", "requires": { - "safe-buffer": "5.1.2" + "safe-buffer": "5.2.1" + }, + "dependencies": { + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + } } }, "content-type": { @@ -243,14 +349,14 @@ "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" }, "cookie": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", - "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==" + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", + "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==" }, "cookie-signature": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" }, "core-util-is": { "version": "1.0.2", @@ -285,7 +391,7 @@ "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" } } }, @@ -305,9 +411,9 @@ "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" }, "destroy": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", - "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==" }, "dicer": { "version": "0.2.5", @@ -360,19 +466,22 @@ "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, "ejs": { - "version": "2.5.9", - "resolved": "https://registry.npmjs.org/ejs/-/ejs-2.5.9.tgz", - "integrity": "sha512-GJCAeDBKfREgkBtgrYSf9hQy9kTb3helv0zGdzqhM7iAkW8FA/ZF97VQDbwFiwIT8MQLLOe5VlPZOEvZAqtUAQ==" + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.8.tgz", + "integrity": "sha512-/sXZeMlhS0ArkfX2Aw780gJzXSMPnKjtspYZv+f3NiKLlubezAHDU5+9xz6gd3/NhG3txQCo6xlglmTS+oTGEQ==", + "requires": { + "jake": "^10.8.5" + } }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==" }, "engine.io": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.2.0.tgz", - "integrity": "sha512-4KzwW3F3bk+KlzSOY57fj/Jx6LyRQ1nbcyIadehl+AnXjKT7gDO0ORdRi/84ixvMKTym6ZKuxvbzN62HDDU1Lg==", + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.2.1.tgz", + "integrity": "sha512-ECceEFcAaNRybd3lsGQKas3ZlMVjN3cyWwMP25D2i0zWfyiytVbTpRPa34qrr+FHddtpBVOmq4H/DCv1O0lZRA==", "requires": { "@types/cookie": "^0.4.1", "@types/cors": "^2.8.12", @@ -414,7 +523,7 @@ "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" }, "esprima": { "version": "4.0.1", @@ -424,120 +533,100 @@ "etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==" }, "express": { - "version": "4.17.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", - "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==", + "version": "4.18.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", + "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", "requires": { - "accepts": "~1.3.7", + "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.19.0", - "content-disposition": "0.5.3", + "body-parser": "1.20.1", + "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.4.0", + "cookie": "0.5.0", "cookie-signature": "1.0.6", "debug": "2.6.9", - "depd": "~1.1.2", + "depd": "2.0.0", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "etag": "~1.8.1", - "finalhandler": "~1.1.2", + "finalhandler": "1.2.0", "fresh": "0.5.2", + "http-errors": "2.0.0", "merge-descriptors": "1.0.1", "methods": "~1.1.2", - "on-finished": "~2.3.0", + "on-finished": "2.4.1", "parseurl": "~1.3.3", "path-to-regexp": "0.1.7", - "proxy-addr": "~2.0.5", - "qs": "6.7.0", + "proxy-addr": "~2.0.7", + "qs": "6.11.0", "range-parser": "~1.2.1", - "safe-buffer": "5.1.2", - "send": "0.17.1", - "serve-static": "1.14.1", - "setprototypeof": "1.1.1", - "statuses": "~1.5.0", + "safe-buffer": "5.2.1", + "send": "0.18.0", + "serve-static": "1.15.0", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" }, "dependencies": { - "body-parser": { - "version": "1.19.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", - "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==", + "accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", "requires": { - "bytes": "3.1.0", - "content-type": "~1.0.4", - "debug": "2.6.9", - "depd": "~1.1.2", - "http-errors": "1.7.2", - "iconv-lite": "0.4.24", - "on-finished": "~2.3.0", - "qs": "6.7.0", - "raw-body": "2.4.0", - "type-is": "~1.6.17" + "mime-types": "~2.1.34", + "negotiator": "0.6.3" } }, - "bytes": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", - "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==" - }, - "http-errors": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", - "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==", - "requires": { - "depd": "~1.1.2", - "inherits": "2.0.3", - "setprototypeof": "1.1.1", - "statuses": ">= 1.5.0 < 2", - "toidentifier": "1.0.0" - } - }, - "iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "requires": { - "safer-buffer": ">= 2.1.2 < 3" - } + "depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" }, "mime-db": { - "version": "1.46.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.46.0.tgz", - "integrity": "sha512-svXaP8UQRZ5K7or+ZmfNhg2xX3yKDMUzqadsSqi4NCH/KomcH75MAMYAGVlvXn4+b/xOPhS3I2uHKRUzvjY7BQ==" + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" }, "mime-types": { - "version": "2.1.29", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.29.tgz", - "integrity": "sha512-Y/jMt/S5sR9OaqteJtslsFZKWOIIqMACsJSiHghlCAyhf7jfVYjKBmLiX8OgpWeW+fjJ2b+Az69aPFPkUOY6xQ==", + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", "requires": { - "mime-db": "1.46.0" + "mime-db": "1.52.0" } }, - "qs": { - "version": "6.7.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", - "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==" + "negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" }, - "raw-body": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", - "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==", + "on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", "requires": { - "bytes": "3.1.0", - "http-errors": "1.7.2", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" + "ee-first": "1.1.1" } }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, "setprototypeof": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", - "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==" }, "type-is": { "version": "1.6.18", @@ -583,29 +672,67 @@ "pend": "~1.2.0" } }, + "filelist": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", + "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==", + "requires": { + "minimatch": "^5.0.1" + }, + "dependencies": { + "minimatch": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.2.tgz", + "integrity": "sha512-bNH9mmM9qsJ2X4r2Nat1B//1dJVcn3+iBLa3IgqJ7EbGaDNepL9QSHOxN4ng33s52VMMhhIfgCYDk3C4ZmlDAg==", + "requires": { + "brace-expansion": "^2.0.1" + } + } + } + }, "finalhandler": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", - "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", + "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", "requires": { "debug": "2.6.9", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", - "on-finished": "~2.3.0", + "on-finished": "2.4.1", "parseurl": "~1.3.3", - "statuses": "~1.5.0", + "statuses": "2.0.1", "unpipe": "~1.0.0" + }, + "dependencies": { + "on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "requires": { + "ee-first": "1.1.1" + } + }, + "statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==" + } } }, "forwarded": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", - "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==" }, "fresh": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==" + }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, "generate-function": { "version": "2.3.1", @@ -615,21 +742,77 @@ "is-property": "^1.0.2" } }, - "http-errors": { - "version": "1.6.3", - "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", - "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", + "get-intrinsic": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz", + "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==", "requires": { - "depd": "~1.1.2", - "inherits": "2.0.3", - "setprototypeof": "1.1.0", - "statuses": ">= 1.4.0 < 2" + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + } + }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "requires": { + "function-bind": "^1.1.1" + } + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" + }, + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" + }, + "http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "requires": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "dependencies": { + "depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==" + }, + "toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" + } } }, "iconv-lite": { - "version": "0.4.23", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz", - "integrity": "sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA==", + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", "requires": { "safer-buffer": ">= 2.1.2 < 3" } @@ -654,6 +837,17 @@ "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, + "jake": { + "version": "10.8.5", + "resolved": "https://registry.npmjs.org/jake/-/jake-10.8.5.tgz", + "integrity": "sha512-sVpxYeuAhWt0OTWITwT98oyV0GsXyMlXCF+3L1SuafBVUIr/uILGRB+NqwkzhgXKvoJpDIpQvqkUALgdmQsQxw==", + "requires": { + "async": "^3.2.3", + "chalk": "^4.0.2", + "filelist": "^1.0.1", + "minimatch": "^3.0.4" + } + }, "js-yaml": { "version": "3.14.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.0.tgz", @@ -669,27 +863,32 @@ "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" }, "json5": { - "version": "1.0.1", - "resolved": "http://registry.npmjs.org/json5/-/json5-1.0.1.tgz", - "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz", + "integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==", "requires": { "minimist": "^1.2.0" } }, "jsonwebtoken": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.3.0.tgz", - "integrity": "sha512-oge/hvlmeJCH+iIz1DwcO7vKPkNGJHhgkspk8OH3VKlw+mbi42WtD4ig1+VXRln765vxptAv+xT26Fd3cteqag==", + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz", + "integrity": "sha512-tuGfYXxkQGDPnLJ7SibiQgVgeDgfbPq2k2ICcbgqW8WxWLBAxKQM/ZCu/IT8SOSwmaYl4dpTFCW5xZv7YbbWUw==", "requires": { - "jws": "^3.1.5", - "lodash.includes": "^4.3.0", - "lodash.isboolean": "^3.0.3", - "lodash.isinteger": "^4.0.4", - "lodash.isnumber": "^3.0.3", - "lodash.isplainobject": "^4.0.6", - "lodash.isstring": "^4.0.1", - "lodash.once": "^4.0.0", - "ms": "^2.1.1" + "jws": "^3.2.2", + "lodash": "^4.17.21", + "ms": "^2.1.1", + "semver": "^7.3.8" + }, + "dependencies": { + "semver": { + "version": "7.3.8", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", + "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "requires": { + "lru-cache": "^6.0.0" + } + } } }, "jwa": { @@ -703,48 +902,38 @@ } }, "jws": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/jws/-/jws-3.1.5.tgz", - "integrity": "sha512-GsCSexFADNQUr8T5HPJvayTjvPIfoyJPtLQBwn5a4WZQchcrPMPMAWcC1AzJVRDKyD6ZPROPAxgv6rfHViO4uQ==", + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", + "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", "requires": { - "jwa": "^1.1.5", + "jwa": "^1.4.1", "safe-buffer": "^5.0.1" + }, + "dependencies": { + "ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "requires": { + "safe-buffer": "^5.0.1" + } + }, + "jwa": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", + "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", + "requires": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + } } }, - "lodash.includes": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", - "integrity": "sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8=" - }, - "lodash.isboolean": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", - "integrity": "sha1-bC4XHbKiV82WgC/UOwGyDV9YcPY=" - }, - "lodash.isinteger": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", - "integrity": "sha1-YZwK89A/iwTDH1iChAt3sRzWg0M=" - }, - "lodash.isnumber": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", - "integrity": "sha1-POdoEMWSjQM1IwGsKHMX8RwLH/w=" - }, - "lodash.isplainobject": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", - "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=" - }, - "lodash.isstring": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", - "integrity": "sha1-1SfftUVuynzJu5XV2ur4i6VKVFE=" - }, - "lodash.once": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", - "integrity": "sha1-DdOXEhPHxW34gJd9UEyI+0cal6w=" + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "long": { "version": "4.0.0", @@ -767,12 +956,12 @@ "merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", - "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" }, "methods": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==" }, "mime": { "version": "2.3.1", @@ -799,6 +988,25 @@ } } }, + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "requires": { + "brace-expansion": "^1.1.7" + }, + "dependencies": { + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + } + } + }, "minimist": { "version": "1.2.6", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", @@ -813,14 +1021,14 @@ } }, "moment": { - "version": "2.29.2", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.2.tgz", - "integrity": "sha512-UgzG4rvxYpN15jgCmVJwac49h9ly9NurikMWGPdVxm8GZD6XjkKPxDTjQQ43gtGgnV3X0cAyWDdP2Wexoquifg==" + "version": "2.29.4", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz", + "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==" }, "moment-timezone": { - "version": "0.5.23", - "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.23.tgz", - "integrity": "sha512-WHFH85DkCfiNMDX5D3X7hpNH3/PUhjTGcD0U1SgfBGZxJ3qUmJh5FdvaFjcClxOvB3rzdfj4oRffbI38jEnC1w==", + "version": "0.5.40", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.40.tgz", + "integrity": "sha512-tWfmNkRYmBkPJz5mr9GVDn9vRlVZOTe6yqY92rFxiOdWXbjaR0+9LwQnZGGuNR63X456NqmEkbskte8tWL5ePg==", "requires": { "moment": ">= 2.9.0" } @@ -939,6 +1147,11 @@ "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" }, + "object-inspect": { + "version": "1.12.3", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", + "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==" + }, "on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", @@ -960,7 +1173,7 @@ "path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" }, "pend": { "version": "1.2.0", @@ -1050,11 +1263,11 @@ "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" }, "proxy-addr": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", - "integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", "requires": { - "forwarded": "~0.1.2", + "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, @@ -1069,9 +1282,12 @@ "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" }, "qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "requires": { + "side-channel": "^1.0.4" + } }, "random-bytes": { "version": "1.0.0", @@ -1084,20 +1300,20 @@ "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" }, "raw-body": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.3.tgz", - "integrity": "sha512-9esiElv1BrZoI3rCDuOuKCBRbuApGGaDPQfjSflGxdy4oyzqghxu6klEkkVIvBje+FF0BX9coEv8KqW6X/7njw==", + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", + "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", "requires": { - "bytes": "3.0.0", - "http-errors": "1.6.3", - "iconv-lite": "0.4.23", + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", "unpipe": "1.0.0" }, "dependencies": { "bytes": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", - "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" } } }, @@ -1168,51 +1384,52 @@ "integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8=" }, "send": { - "version": "0.17.1", - "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz", - "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==", + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", + "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", "requires": { "debug": "2.6.9", - "depd": "~1.1.2", - "destroy": "~1.0.4", + "depd": "2.0.0", + "destroy": "1.2.0", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "etag": "~1.8.1", "fresh": "0.5.2", - "http-errors": "~1.7.2", + "http-errors": "2.0.0", "mime": "1.6.0", - "ms": "2.1.1", - "on-finished": "~2.3.0", + "ms": "2.1.3", + "on-finished": "2.4.1", "range-parser": "~1.2.1", - "statuses": "~1.5.0" + "statuses": "2.0.1" }, "dependencies": { - "http-errors": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.3.tgz", - "integrity": "sha512-ZTTX0MWrsQ2ZAhA1cejAwDLycFsd7I7nVtnkT3Ol0aqodaKW+0CTZDQ1uBv5whptCnc8e8HeRRJxRs0kmm/Qfw==", - "requires": { - "depd": "~1.1.2", - "inherits": "2.0.4", - "setprototypeof": "1.1.1", - "statuses": ">= 1.5.0 < 2", - "toidentifier": "1.0.0" - } - }, - "inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + "depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" }, "mime": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" }, - "setprototypeof": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", - "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "requires": { + "ee-first": "1.1.1" + } + }, + "statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==" } } }, @@ -1222,14 +1439,14 @@ "integrity": "sha1-1WgS4cAXpuTnw+Ojeh2m143TyT4=" }, "serve-static": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", - "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==", + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", "requires": { "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "parseurl": "~1.3.3", - "send": "0.17.1" + "send": "0.18.0" } }, "setprototypeof": { @@ -1237,6 +1454,16 @@ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==" }, + "side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "requires": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + } + }, "socket.io": { "version": "4.5.2", "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.5.2.tgz", @@ -1322,6 +1549,14 @@ "safe-buffer": "~5.1.0" } }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "requires": { + "has-flag": "^4.0.0" + } + }, "toidentifier": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", @@ -1357,7 +1592,7 @@ "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==" }, "uri-js": { "version": "4.4.1", @@ -1383,7 +1618,7 @@ "utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==" }, "vary": { "version": "1.1.2", diff --git a/DocService/package.json b/DocService/package.json index 8c6cf390..db75eab0 100644 --- a/DocService/package.json +++ b/DocService/package.json @@ -12,17 +12,17 @@ "ajv": "^8.9.0", "apicache": "^1.6.2", "base64-stream": "^1.0.0", - "body-parser": "^1.18.3", + "body-parser": "^1.20.1", "bottleneck": "^2.19.5", "bytes": "^3.0.0", "co": "^4.6.0", "config": "^2.0.1", "cron": "^1.5.0", "deep-equal": "^1.0.1", - "ejs": "~2.5.1", - "express": "^4.17.1", + "ejs": "^3.1.8", + "express": "^4.18.2", "fakeredis": "^2.0.0", - "jsonwebtoken": "^8.3.0", + "jsonwebtoken": "^9.0.0", "jwa": "^1.1.6", "mime": "^2.3.1", "mime-db": "^1.49.0", diff --git a/FileConverter/npm-shrinkwrap.json b/FileConverter/npm-shrinkwrap.json index 350d9e68..e57a7f59 100644 --- a/FileConverter/npm-shrinkwrap.json +++ b/FileConverter/npm-shrinkwrap.json @@ -51,9 +51,9 @@ "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" }, "json5": { - "version": "1.0.1", - "resolved": "http://registry.npmjs.org/json5/-/json5-1.0.1.tgz", - "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz", + "integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==", "requires": { "minimist": "^1.2.0" } @@ -76,9 +76,9 @@ } }, "minimist": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", - "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.7.tgz", + "integrity": "sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==" }, "pseudomap": { "version": "1.0.2", diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index feced006..1e210aab 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -30,7 +30,7 @@ "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" } } }, @@ -52,7 +52,7 @@ "array-each": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/array-each/-/array-each-1.0.1.tgz", - "integrity": "sha1-p5SvDAWrF1KEbudTofIRoFugxE8=" + "integrity": "sha512-zHjL5SZa68hkKHBFBK6DJCTtr9sfTCPCaph/L7tMSLcTFgy+zX7E+6q5UArbtOtMBCtxdICpfTCspRse+ywyXA==" }, "array-slice": { "version": "1.1.0", @@ -70,9 +70,9 @@ "integrity": "sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c=" }, "async": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.3.tgz", - "integrity": "sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g==" + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", + "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" }, "atob": { "version": "2.1.2", @@ -80,9 +80,9 @@ "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==" }, "balanced-match": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, "base": { "version": "0.11.2", @@ -334,7 +334,7 @@ "colors": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz", - "integrity": "sha1-FopHAXVran9RoSzgyXv6KMCE7WM=" + "integrity": "sha512-ENwblkFQpqqia6b++zLD/KUWafYlVY/UNnAp7oz7LY7E924wmpye416wBOmvv/HMWzl8gL1kJlfvId/1Dg176w==" }, "component-emitter": { "version": "1.3.0", @@ -344,7 +344,7 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" }, "copy-descriptor": { "version": "0.1.1", @@ -424,12 +424,12 @@ "eventemitter2": { "version": "0.4.14", "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz", - "integrity": "sha1-j2G3XN4BKy6esoTUVFWDtWQ7Yas=" + "integrity": "sha512-K7J4xq5xAD5jHsGM5ReWXRTFa3JRGofHiMcVgQ8PRwgWxzjHpMWCIzsmyf60+mh8KLsqYPcjUMa0AC4hd6lPyQ==" }, "exit": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", - "integrity": "sha1-BjJjj42HfMghB9MKD/8aF8uhzQw=" + "integrity": "sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==" }, "expand-brackets": { "version": "2.1.4", @@ -583,7 +583,7 @@ "findup-sync": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/findup-sync/-/findup-sync-0.3.0.tgz", - "integrity": "sha1-N5MKpdgWt3fANEXhlmzGeQpMCxY=", + "integrity": "sha512-z8Nrwhi6wzxNMIbxlrTzuUW6KWuKkogZ/7OdDVq+0+kxn77KUH1nipx8iU6suqkHqc4y6n7a9A8IpmxY/pTjWg==", "requires": { "glob": "~5.0.0" }, @@ -591,7 +591,7 @@ "glob": { "version": "5.0.15", "resolved": "https://registry.npmjs.org/glob/-/glob-5.0.15.tgz", - "integrity": "sha1-G8k2ueAvSmA/zCIuz3Yz0wuLk7E=", + "integrity": "sha512-c9IPMazfRITpmAAKi22dK1VKxGDX9ehhqfABDriL/lzO92xcUKEJPQHrVA/2YHSNFB4iFlykVmWvwo48nr3OxA==", "requires": { "inflight": "^1.0.4", "inherits": "2", @@ -627,7 +627,7 @@ "for-own": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/for-own/-/for-own-1.0.0.tgz", - "integrity": "sha1-xjMy9BXO3EsE2/5wz4NklMU8tEs=", + "integrity": "sha512-0OABksIGrxKK8K4kynWkQ7y1zounQxP+CWnyclVwj81KW3vlLlGUx57DKGcP/LH216GzqnstnPocF16Nxs0Ycg==", "requires": { "for-in": "^1.0.1" } @@ -701,16 +701,16 @@ "integrity": "sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ==" }, "grunt": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/grunt/-/grunt-1.4.1.tgz", - "integrity": "sha512-ZXIYXTsAVrA7sM+jZxjQdrBOAg7DyMUplOMhTaspMRExei+fD0BTwdWXnn0W5SXqhb/Q/nlkzXclSi3IH55PIA==", + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/grunt/-/grunt-1.5.3.tgz", + "integrity": "sha512-mKwmo4X2d8/4c/BmcOETHek675uOqw0RuA/zy12jaspWqvTp4+ZeQF1W+OTpcbncnaBsfbQJ6l0l4j+Sn/GmaQ==", "requires": { "dateformat": "~3.0.3", "eventemitter2": "~0.4.13", "exit": "~0.1.2", "findup-sync": "~0.3.0", "glob": "~7.1.6", - "grunt-cli": "~1.4.2", + "grunt-cli": "~1.4.3", "grunt-known-options": "~2.0.0", "grunt-legacy-log": "~3.0.0", "grunt-legacy-util": "~2.0.1", @@ -744,6 +744,14 @@ } } } + }, + "minimatch": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.8.tgz", + "integrity": "sha512-6FsRAQsxQ61mw+qP1ZzbL9Bc78x2p5OqNgNpnoAFLTrX8n5Kxph0CsnhmKKNXTWjXqU5L0pGPR7hYk+XWZr60Q==", + "requires": { + "brace-expansion": "^1.1.7" + } } } }, @@ -950,7 +958,7 @@ "hooker": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/hooker/-/hooker-0.2.3.tgz", - "integrity": "sha1-uDT3I8xKJCqmWWNFnfbZhMXT2Vk=" + "integrity": "sha512-t+UerCsQviSymAInD01Pw+Dn/usmz1sRO+3Zk1+lx8eg+WKpD2ulcwWqHHL0+aseRBr+3+vIhiG1K1JTwaIcTA==" }, "iconv-lite": { "version": "0.4.24", @@ -982,7 +990,7 @@ "interpret": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.1.0.tgz", - "integrity": "sha1-ftGxQQxqDg94z5XTuEQMY/eLhhQ=" + "integrity": "sha512-CLM8SNMDu7C5psFCn6Wg/tgpj/bKAg7hc2gWqcuR9OD5Ft9PhBpIu8PLicPeis+xDd6YX2ncI8MCA64I9tftIA==" }, "is-absolute": { "version": "1.0.0", @@ -1017,9 +1025,9 @@ "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" }, "is-core-module": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.8.1.tgz", - "integrity": "sha512-SdNCUs284hr40hFTFP6l0IfZ/RSrMXF3qgoRHd3/79unUTvrFO/JoXwkGm+5J/Oe3E/b5GsnG330uUNgRpu1PA==", + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.11.0.tgz", + "integrity": "sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw==", "requires": { "has": "^1.0.3" } @@ -1209,12 +1217,12 @@ "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==" }, "micromatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.4.tgz", - "integrity": "sha512-pRmzw/XUcwXGpD9aI9q/0XOwLNygjETJ8y0ao0wdqprrzDa4YnxLcz7fQRZr8voh8V10kGhABbNcHVk5wHgWwg==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", + "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", "requires": { - "braces": "^3.0.1", - "picomatch": "^2.2.3" + "braces": "^3.0.2", + "picomatch": "^2.3.1" } }, "to-regex-range": { @@ -1284,9 +1292,9 @@ } }, "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "requires": { "brace-expansion": "^1.1.7" } @@ -1321,9 +1329,9 @@ "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==" }, "mout": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/mout/-/mout-1.2.3.tgz", - "integrity": "sha512-vtE+eZcSj/sBkIp6gxB87MznryWP+gHIp0XX9SKrzA5TAkvz6y7VTuNruBjYdJozd8NY5i9XVIsn8cn3SwNjzg==" + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/mout/-/mout-1.2.4.tgz", + "integrity": "sha512-mZb9uOruMWgn/fw28DG4/yE3Kehfk1zKCLhuDU2O3vlKdnBBr4XaOCqVTflJ5aODavGUPqFHZgrFX3NJVuxGhQ==" }, "ms": { "version": "2.0.0", @@ -1351,7 +1359,7 @@ "nopt": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", - "integrity": "sha1-xkZdvwirzU2zWTF/eaxopkayj/k=", + "integrity": "sha512-4GUt3kSEYmk4ITxzB/b9vaIDfUVWN/Ml1Fwl11IlnIG2iaJ9O6WXZ9SrYM9NLI8OCBieN2Y8SWC2oJV0RQ7qYg==", "requires": { "abbrev": "1" } @@ -1395,7 +1403,7 @@ "object.defaults": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/object.defaults/-/object.defaults-1.1.0.tgz", - "integrity": "sha1-On+GgzS0B96gbaFtiNXNKeQ1/s8=", + "integrity": "sha512-c/K0mw/F11k4dEUBMW8naXUuBuhxRCfG7W+yFy8EcijU/rSmazOUd1XAEEe6bC0OuXY4HUKjTJv7xbxIMqdxrA==", "requires": { "array-each": "^1.0.1", "array-slice": "^1.0.0", @@ -1406,7 +1414,7 @@ "object.map": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/object.map/-/object.map-1.0.1.tgz", - "integrity": "sha1-z4Plncj8wK1fQlDh94s7gb2AHTc=", + "integrity": "sha512-3+mAJu2PLfnSVGHwIWubpOFLscJANBKuB/6A4CxBstc4aqwQY0FWcsppuy4jU5GSB95yES5JHSI+33AWuS4k6w==", "requires": { "for-own": "^1.0.0", "make-iterator": "^1.0.0" @@ -1450,7 +1458,7 @@ "parse-filepath": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/parse-filepath/-/parse-filepath-1.0.2.tgz", - "integrity": "sha1-pjISf1Oq89FYdvWHLz/6x2PWyJE=", + "integrity": "sha512-FwdRXKCohSVeXqwtYonZTXtbGJKrn+HNyWDYVcp5yuJlesTwNH4rsmRZ+GrKAPJ5bLpRxESMeS+Rl0VCHRvB2Q==", "requires": { "is-absolute": "^1.0.0", "map-cache": "^0.2.0", @@ -1480,7 +1488,7 @@ "path-root": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/path-root/-/path-root-0.1.1.tgz", - "integrity": "sha1-mkpoFMrBwM1zNgqV8yCDyOpHRbc=", + "integrity": "sha512-QLcPegTHF11axjfojBIoDygmS2E3Lf+8+jI6wOVmNVenrKSo3mFdSGiIgdSHenczw3wPtlVMQaFVwGmM7BJdtg==", "requires": { "path-root-regex": "^0.1.0" } @@ -1488,7 +1496,7 @@ "path-root-regex": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/path-root-regex/-/path-root-regex-0.1.2.tgz", - "integrity": "sha1-v8zcjfWxLcUsi0PsONGNcsBLqW0=" + "integrity": "sha512-4GlJ6rZDhQZFE0DPVKh0e9jmZ5egZfxTkp7bcRDuPlJXbAwhxcl2dINPUAsjLdejqaLsCeg8axcLjIbvBjN4pQ==" }, "picomatch": { "version": "2.3.1", @@ -1528,11 +1536,11 @@ "integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc=" }, "resolve": { - "version": "1.21.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.21.0.tgz", - "integrity": "sha512-3wCbTpk5WJlyE4mSOtDLhqQmGFi0/TD9VPwmiolnk8U0wRgMEktqCXd3vy5buTO3tljvalNvKrjHEfrd2WpEKA==", + "version": "1.22.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.1.tgz", + "integrity": "sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw==", "requires": { - "is-core-module": "^2.8.0", + "is-core-module": "^2.9.0", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" } @@ -1813,14 +1821,14 @@ "unc-path-regex": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/unc-path-regex/-/unc-path-regex-0.1.2.tgz", - "integrity": "sha1-5z3T17DXxe2G+6xrCufYxqadUPo=" + "integrity": "sha512-eXL4nmJT7oCpkZsHZUOJo8hcX3GbsiDOa0Qu9F646fi8dT3XuSVopVqAcEiVzSKKH7UoDti23wNX3qGFxcW5Qg==" }, "underscore.string": { - "version": "3.3.5", - "resolved": "https://registry.npmjs.org/underscore.string/-/underscore.string-3.3.5.tgz", - "integrity": "sha512-g+dpmgn+XBneLmXXo+sGlW5xQEt4ErkS3mgeN2GFbremYeMBSJKr9Wf2KJplQVaiPY/f7FN6atosWYNm9ovrYg==", + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/underscore.string/-/underscore.string-3.3.6.tgz", + "integrity": "sha512-VoC83HWXmCrF6rgkyxS9GHv8W9Q5nhMKho+OadDJGzL2oDYbYEppBaCMH6pFlwLeqj2QS+hhkw2kpXkSdD1JxQ==", "requires": { - "sprintf-js": "^1.0.3", + "sprintf-js": "^1.1.1", "util-deprecate": "^1.0.2" } }, @@ -1892,7 +1900,7 @@ "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, "v8flags": { "version": "3.2.0", diff --git a/package.json b/package.json index 5a67c940..ff6a0889 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "./build/server/Metrics" ], "dependencies": { - "grunt": "^1.4.1", + "grunt": "^1.5.3", "grunt-banner": "^0.6.0", "grunt-check-dependencies": "^1.0.0", "grunt-contrib-clean": "^2.0.0", From a7880c29e35deaef8522b5073cc69dd398b77d17 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Sun, 15 Jan 2023 22:40:01 +0300 Subject: [PATCH 42/45] [bug] Fix wopi discovery xml --- DocService/sources/wopiClient.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DocService/sources/wopiClient.js b/DocService/sources/wopiClient.js index bfd4a5c3..5bd80874 100644 --- a/DocService/sources/wopiClient.js +++ b/DocService/sources/wopiClient.js @@ -187,7 +187,7 @@ function discovery(req, res) { } } let xmlApp = xmlZone.ele('app', {name: 'Capabilities'}); - xmlApp.ele('action', {ext: '', name: 'getinfo', urlsrc: 'locks,update', urlsrc: `${baseUrl}/hosting/capabilities`}).up(); + xmlApp.ele('action', {ext: '', name: 'getinfo', requires: 'locks,update', urlsrc: `${baseUrl}/hosting/capabilities`}).up(); xmlApp.up(); //end section for collabora nexcloud connectors let xmlDiscovery = xmlZone.up(); From 2c9b15a4f607ccf8f742169d9f383abfc0c4b948 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Wed, 18 Jan 2023 10:55:24 +0300 Subject: [PATCH 43/45] [bug] Check authorization for bug 60619 --- DocService/sources/DocsCoServer.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 4322ffc8..46135337 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -1317,6 +1317,7 @@ exports.install = function(server, callbackFunction) { io.use((socket, next) => { co(function*(){ let ctx = new operationContext.Context(); + let res; let checkJwtRes; try { ctx.initFromConnection(socket); @@ -1324,12 +1325,16 @@ exports.install = function(server, callbackFunction) { let handshake = socket.handshake; if (cfgTokenEnableBrowser) { checkJwtRes = yield checkJwt(ctx, handshake?.auth?.token, commonDefines.c_oAscSecretType.Browser); + if (!checkJwtRes.decoded) { + res = new Error("not authorized"); + res.data = { code: checkJwtRes.code, description: checkJwtRes.description }; + } } } catch (err) { - ctx.logger.info('io.use error: %s', err.stack); + ctx.logger.error('io.use error: %s', err.stack); } finally { ctx.logger.info('io.use end'); - next((checkJwtRes && !checkJwtRes.decoded) ? new Error("not authorized") : undefined); + next(res); } }); }); From 6647ccc73e35b7b138659cf3a58ad1ab0d5c3518 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Thu, 19 Jan 2023 08:23:50 +0300 Subject: [PATCH 44/45] [feature] For reference data --- Common/config/default.json | 2 +- DocService/sources/canvasservice.js | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Common/config/default.json b/Common/config/default.json index 0067bfe2..85767337 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -141,7 +141,7 @@ "editorDataStorage": "editorDataMemory", "assemblyFormatAsOrigin": true, "newFileTemplate" : "../../document-templates/new", - "downloadFileAllowExt": ["pdf"], + "downloadFileAllowExt": ["pdf", "xlsx"], "tokenRequiredParams": true }, "requestDefaults": { diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index 6567f4ec..35164978 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -1539,6 +1539,8 @@ exports.downloadFile = function(req, res) { url = decoded.changesUrl; } else if (decoded.document && -1 !== cfgDownloadFileAllowExt.indexOf(decoded.document.fileType)) { url = decoded.document.url; + } else if (decoded.url && -1 !== cfgDownloadFileAllowExt.indexOf(decoded.fileType)) { + url = decoded.url; } else { errorDescription = 'access deny'; } From cff1455a6df94d3b97f2175a4479aa5448261a82 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 23 Jan 2023 17:31:06 +0300 Subject: [PATCH 45/45] [bug] Fix disconnect reason for socket.io --- DocService/sources/DocsCoServer.js | 49 +++++++++++++++++++----------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 46135337..2bce64e5 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -536,6 +536,13 @@ function sendDataRefreshToken(ctx, conn, msg) { function sendDataRpc(ctx, conn, responseKey, data) { sendData(ctx, conn, {type: "rpc", responseKey: responseKey, data: data}); } +function sendDataDrop(ctx, conn, code, description) { + sendData(ctx, conn, {type: "drop", code: code, description: description}); +} +function sendDataDisconnectReason(ctx, conn, code, description) { + sendData(ctx, conn, {type: "disconnectReason", code: code, description: description}); +} + function sendReleaseLock(ctx, conn, userLocks) { sendData(ctx, conn, {type: "releaseLock", locks: _.map(userLocks, function(e) { return { @@ -1029,14 +1036,15 @@ function* publishCloseUsersConnection(ctx, docId, users, isOriginalId, code, des }); } } -function closeUsersConnection(docId, usersMap, isOriginalId, code, description) { +function closeUsersConnection(ctx, docId, usersMap, isOriginalId, code, description) { //close let conn; for (let i = connections.length - 1; i >= 0; --i) { conn = connections[i]; if (conn.docId === docId) { if (isOriginalId ? usersMap[conn.user.idOriginal] : usersMap[conn.user.id]) { - conn.disconnect(code, description); + sendDataDisconnectReason(ctx, conn, code, description); + conn.disconnect(true); } } } @@ -1052,11 +1060,7 @@ function dropUserFromDocument(ctx, docId, userId, description) { for (var i = 0, length = connections.length; i < length; ++i) { elConnection = connections[i]; if (elConnection.docId === docId && userId === elConnection.user.idOriginal && !elConnection.isCloseCoAuthoring) { - sendData(ctx, elConnection, - { - type: "drop", - description: description - });//Or 0 if fails + sendDataDrop(ctx, elConnection, description); } } } @@ -1375,13 +1379,15 @@ exports.install = function(server, callbackFunction) { if (conn.isCiriticalError && ('message' == data.type || 'getLock' == data.type || 'saveChanges' == data.type || 'isSaveLock' == data.type)) { ctx.logger.warn("conn.isCiriticalError send command: type = %s", data.type); - conn.disconnect(constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); + sendDataDisconnectReason(ctx, conn, constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); + conn.disconnect(true); return; } if ((conn.isCloseCoAuthoring || (conn.user && conn.user.view)) && ('getLock' == data.type || 'saveChanges' == data.type || 'isSaveLock' == data.type)) { ctx.logger.warn("conn.user.view||isCloseCoAuthoring access deny: type = %s", data.type); - conn.disconnect(constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); + sendDataDisconnectReason(ctx, conn, constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); + conn.disconnect(true); return; } yield encryptPasswordParams(ctx, data); @@ -1391,7 +1397,8 @@ exports.install = function(server, callbackFunction) { yield* auth(ctx, conn, data); } catch(err){ ctx.logger.error('auth error: %s', err.stack); - conn.disconnect(constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); + sendDataDisconnectReason(ctx, conn, constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); + conn.disconnect(true); return; } break; @@ -2202,7 +2209,8 @@ exports.install = function(server, callbackFunction) { fillDataFromJwtRes = fillDataFromJwt(ctx, decoded, data); } else if (cfgTokenRequiredParams) { ctx.logger.error("auth missing required parameter %s (since 7.1 version)", validationErr); - conn.disconnect(constants.JWT_ERROR_CODE, constants.JWT_ERROR_REASON); + sendDataDisconnectReason(ctx, conn, constants.JWT_ERROR_CODE, constants.JWT_ERROR_REASON); + conn.disconnect(true); return; } else { ctx.logger.warn("auth missing required parameter %s (since 7.1 version)", validationErr); @@ -2211,11 +2219,13 @@ exports.install = function(server, callbackFunction) { } if(!fillDataFromJwtRes) { ctx.logger.warn("fillDataFromJwt return false"); - conn.disconnect(constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); + sendDataDisconnectReason(ctx, conn, constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); + conn.disconnect(true); return; } } else { - conn.disconnect(checkJwtRes.code, checkJwtRes.description); + sendDataDisconnectReason(ctx, conn, checkJwtRes.code, checkJwtRes.description); + conn.disconnect(true); return; } } @@ -2244,7 +2254,8 @@ exports.install = function(server, callbackFunction) { let filterStatus = yield* utils.checkHostFilter(ctx, documentCallback.hostname); if (0 !== filterStatus) { ctx.logger.warn('checkIpFilter error: url = %s', data.documentCallbackUrl); - conn.disconnect(constants.DROP_CODE, constants.DROP_REASON); + sendDataDisconnectReason(ctx, conn, constants.DROP_CODE, constants.DROP_REASON); + conn.disconnect(true); return; } } @@ -3208,7 +3219,7 @@ exports.install = function(server, callbackFunction) { } break; case commonDefines.c_oPublishType.closeConnection: - closeUsersConnection(data.docId, data.usersMap, data.isOriginalId, data.code, data.description); + closeUsersConnection(ctx, data.docId, data.usersMap, data.isOriginalId, data.code, data.description); break; case commonDefines.c_oPublishType.releaseLock: participants = getParticipants(data.docId, true, data.userId, true); @@ -3434,7 +3445,9 @@ exports.install = function(server, callbackFunction) { }); } else if (nowMs - conn.sessionTimeConnect > cfgExpSessionAbsolute) { ctx.logger.debug('expireDoc close absolute session'); - conn.disconnect(constants.SESSION_ABSOLUTE_CODE, constants.SESSION_ABSOLUTE_REASON); continue; + sendDataDisconnectReason(ctx, conn, constants.SESSION_ABSOLUTE_CODE, constants.SESSION_ABSOLUTE_REASON); + conn.disconnect(true); + continue; } } if (cfgExpSessionIdle > 0 && !(conn.user?.view || conn.isCloseCoAuthoring)) { @@ -3447,7 +3460,9 @@ exports.install = function(server, callbackFunction) { }); } else if (nowMs - conn.sessionTimeLastAction > cfgExpSessionIdle) { ctx.logger.debug('expireDoc close idle session'); - conn.disconnect(constants.SESSION_IDLE_CODE, constants.SESSION_IDLE_REASON); continue; + sendDataDisconnectReason(ctx, conn, constants.SESSION_IDLE_CODE, constants.SESSION_IDLE_REASON); + conn.disconnect(true); + continue; } } if (constants.CONN_CLOSED === conn.conn.readyState) {