Merge branch 'hotfix/v8.1.1' into develop

# Conflicts:
#	DocService/sources/utilsDocService.js

Common/config/default.json

@@ -245,7 +245,7 @@
 			"utils": {
 				"utils_common_fontdir": "null",
 				"utils_fonts_search_patterns": "*.ttf;*.ttc;*.otf",
-				"limits_image_types_upload": "jpg;jpeg;jpe;png;gif;bmp;svg"
+				"limits_image_types_upload": "jpg;jpeg;jpe;png;gif;bmp;svg;tiff;tif"
 			},
 			"sql": {
 				"type": "postgres",

Common/sources/storage-fs.js

@@ -151,7 +151,11 @@ async function getSignedUrl(ctx, storageCfg, baseUrl, strPath, urlType, optFilen
   } else if (ctx.wopiSrc) {
     wopiSrcCached = ctx.wopiSrc;
     url += `&${constants.SHARD_KEY_WOPI_NAME}=${encodeURIComponent(ctx.wopiSrc)}`;
+  } else if (process.env.DEFAULT_SHARD_KEY) {
+    //Set DEFAULT_SHARD_KEY from environment as shardkey in case of integrator did not pass this param
+    url += `&${constants.SHARD_KEY_API_NAME}=${encodeURIComponent(process.env.DEFAULT_SHARD_KEY)}`;
   } else if (shardKeyCached) {
+    //Add stubs for shardkey params until integrators pass these parameters to all requests
     url += `&${constants.SHARD_KEY_API_NAME}=${encodeURIComponent(shardKeyCached)}`;
   } else if (wopiSrcCached) {
     url += `&${constants.SHARD_KEY_WOPI_NAME}=${encodeURIComponent(wopiSrcCached)}`;

DocService/sources/DocsCoServer.js

@@ -836,7 +836,8 @@ function* setForceSave(ctx, docId, forceSave, cmd, success, url) {
   convertInfo.setResponseKey(undefined);
   convertInfo.setFormData(undefined);
   if (convertInfo.getForceSave()) {
-    convertInfo.getForceSave().setType(undefined);
+    //type must be saved to distinguish c_oAscForceSaveTypes.Form
+    //convertInfo.getForceSave().setType(undefined);
     convertInfo.getForceSave().setAuthorUserId(undefined);
     convertInfo.getForceSave().setAuthorUserIndex(undefined);
   }
@@ -884,7 +885,13 @@ async function applyForceSaveCache(ctx, docId, forceSave, type, opt_userConnecti
   let forceSaveCache = await checkForceSaveCache(ctx, forceSave.convertInfo);
   if (forceSaveCache.hasCache || forceSave.ended) {
     if (commonDefines.c_oAscForceSaveTypes.Form === type || commonDefines.c_oAscForceSaveTypes.Internal === type || !forceSave.ended) {
-      if (forceSaveCache.hasValidCache) {
+      //c_oAscForceSaveTypes.Form has uniqueue options {'documentLayout': {'isPrint': true}}; dont use it for other types
+      let forceSaveCached = forceSaveCache.cmd?.getForceSave()?.getType();
+      let cacheHasSameOptions = (commonDefines.c_oAscForceSaveTypes.Form === type &&
+          commonDefines.c_oAscForceSaveTypes.Form === forceSaveCached) ||
+        (commonDefines.c_oAscForceSaveTypes.Form !== type &&
+          commonDefines.c_oAscForceSaveTypes.Form !== forceSaveCached)
+      if (forceSaveCache.hasValidCache && cacheHasSameOptions) {
         let cmd = forceSaveCache.cmd;
         cmd.setUserConnectionDocId(opt_userConnectionDocId);
         cmd.setUserConnectionId(opt_userConnectionId);
@@ -899,7 +906,7 @@ async function applyForceSaveCache(ctx, docId, forceSave, type, opt_userConnecti
         await canvasService.commandSfcCallback(ctx, cmd, true, false);
         res.ok = true;
       } else {
-        await editorData.checkAndSetForceSave(ctx, docId, forceSave.getTime(), forceSave.getIndex(), false, false, null);
+        await editorData.checkAndSetForceSave(ctx, docId, forceSave.time, forceSave.index, false, false, null);
         res.startedForceSave = await editorData.checkAndStartForceSave(ctx, docId);
         res.ok = !!res.startedForceSave;
       }
@@ -4359,6 +4366,9 @@ exports.commandFromServer = function (req, res) {
       output.error = validateInputParams(ctx, authRes, params);
       if (output.error === commonDefines.c_oAscServerCommandErrors.NoError) {
         ctx.logger.debug('commandFromServer: c = %s', params.c);
+        if (params.key && !req.query[constants.SHARD_KEY_API_NAME] && !req.query[constants.SHARD_KEY_WOPI_NAME] && process.env.DEFAULT_SHARD_KEY) {
+          ctx.logger.warn('commandFromServer. Pass query string parameter "%s" to correctly process commands with "key" in sharded cluster', constants.SHARD_KEY_API_NAME);
+        }
         yield *commandHandle(ctx, params, req, output);
       }
     } catch (err) {

DocService/sources/canvasservice.js

@@ -167,11 +167,11 @@ function getOpenedAtJSONParams(row) {
   return undefined;
 }
 
-var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAdditionalOutput, opt_bIsRestore) {
+async function getOutputData(ctx, cmd, outputData, key, optConn, optAdditionalOutput, opt_bIsRestore) {
   const tenExpUpdateVersionStatus = ms(ctx.getCfg('services.CoAuthoring.expire.updateVersionStatus', cfgExpUpdateVersionStatus));
 
   let status, statusInfo, password, creationDate, openedAt, row;
-  let selectRes = yield taskResult.select(ctx, key);
+  let selectRes = await taskResult.select(ctx, key);
   if (selectRes.length > 0) {
     row = selectRes[0];
     status = row.status;
@@ -184,31 +184,29 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd
     case commonDefines.FileStatus.SaveVersion:
     case commonDefines.FileStatus.UpdateVersion:
     case commonDefines.FileStatus.Ok:
-      if(commonDefines.FileStatus.Ok == status) {
+      if(commonDefines.FileStatus.Ok === status) {
         outputData.setStatus('ok');
-      } else if (commonDefines.FileStatus.SaveVersion == status ||
+      } else if (optConn && (optConn.user.view || optConn.isCloseCoAuthoring)) {
+        outputData.setStatus('ok');
+      } else if (commonDefines.FileStatus.SaveVersion === status ||
         (!opt_bIsRestore && commonDefines.FileStatus.UpdateVersion === status &&
         Date.now() - statusInfo * 60000 > tenExpUpdateVersionStatus)) {
-        if (optConn && (optConn.user.view || optConn.isCloseCoAuthoring)) {
+        if (commonDefines.FileStatus.UpdateVersion === status) {
-          outputData.setStatus(constants.FILE_STATUS_UPDATE_VERSION);
+          ctx.logger.warn("UpdateVersion expired");
+        }
+        var updateMask = new taskResult.TaskResultData();
+        updateMask.tenant = ctx.tenant;
+        updateMask.key = key;
+        updateMask.status = status;
+        updateMask.statusInfo = statusInfo;
+        var updateTask = new taskResult.TaskResultData();
+        updateTask.status = commonDefines.FileStatus.Ok;
+        updateTask.statusInfo = constants.NO_ERROR;
+        var updateIfRes = await taskResult.updateIf(ctx, updateTask, updateMask);
+        if (updateIfRes.affectedRows > 0) {
+          outputData.setStatus('ok');
         } else {
-          if (commonDefines.FileStatus.UpdateVersion === status) {
+          outputData.setStatus(constants.FILE_STATUS_UPDATE_VERSION);
-            ctx.logger.warn("UpdateVersion expired");
-          }
-          var updateMask = new taskResult.TaskResultData();
-          updateMask.tenant = ctx.tenant;
-          updateMask.key = key;
-          updateMask.status = status;
-          updateMask.statusInfo = statusInfo;
-          var updateTask = new taskResult.TaskResultData();
-          updateTask.status = commonDefines.FileStatus.Ok;
-          updateTask.statusInfo = constants.NO_ERROR;
-          var updateIfRes = yield taskResult.updateIf(ctx, updateTask, updateMask);
-          if (updateIfRes.affectedRows > 0) {
-            outputData.setStatus('ok');
-          } else {
-            outputData.setStatus(constants.FILE_STATUS_UPDATE_VERSION);
-          }
         }
       } else {
         outputData.setStatus(constants.FILE_STATUS_UPDATE_VERSION);
@@ -219,9 +217,9 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd
         if (optConn) {
           let url;
           if(cmd.getInline()) {
-            url = yield getPrintFileUrl(ctx, key, optConn.baseUrl, cmd.getTitle());
+            url = await getPrintFileUrl(ctx, key, optConn.baseUrl, cmd.getTitle());
           } else {
-            url = yield storage.getSignedUrl(ctx, optConn.baseUrl, strPath, commonDefines.c_oAscUrlTypes.Temporary,
+            url = await storage.getSignedUrl(ctx, optConn.baseUrl, strPath, commonDefines.c_oAscUrlTypes.Temporary,
                                                  cmd.getTitle());
           }
           outputData.setData(url);
@@ -237,8 +235,8 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd
         let decryptedPassword;
         let isCorrectPassword;
         if (password && encryptedUserPassword) {
-          decryptedPassword = yield utils.decryptPassword(ctx, password);
+          decryptedPassword = await utils.decryptPassword(ctx, password);
-          userPassword = yield utils.decryptPassword(ctx, encryptedUserPassword);
+          userPassword = await utils.decryptPassword(ctx, encryptedUserPassword);
           isCorrectPassword = decryptedPassword === userPassword;
         }
         if(password && !isCorrectPassword) {
@@ -252,7 +250,7 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd
           }
         } else if (optConn) {
           outputData.setOpenedAt(openedAt);
-          outputData.setData(yield storage.getSignedUrls(ctx, optConn.baseUrl, key, commonDefines.c_oAscUrlTypes.Session, creationDate));
+          outputData.setData(await storage.getSignedUrls(ctx, optConn.baseUrl, key, commonDefines.c_oAscUrlTypes.Session, creationDate));
         } else if (optAdditionalOutput) {
           optAdditionalOutput.needUrlKey = key;
           optAdditionalOutput.needUrlMethod = 0;
@@ -267,7 +265,7 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd
       outputData.setStatus('needparams');
       var settingsPath = key + '/' + 'origin.' + cmd.getFormat();
       if (optConn) {
-        let url = yield storage.getSignedUrl(ctx, optConn.baseUrl, settingsPath, commonDefines.c_oAscUrlTypes.Temporary);
+        let url = await storage.getSignedUrl(ctx, optConn.baseUrl, settingsPath, commonDefines.c_oAscUrlTypes.Temporary);
         outputData.setData(url);
       } else if (optAdditionalOutput) {
         optAdditionalOutput.needUrlKey = settingsPath;
@@ -286,7 +284,7 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd
     case commonDefines.FileStatus.ErrToReload:
       outputData.setStatus('err');
       outputData.setData(statusInfo);
-      yield cleanupErrToReload(ctx, key);
+      await cleanupErrToReload(ctx, key);
       break;
     case commonDefines.FileStatus.None:
       //this status has no handler
@@ -300,7 +298,7 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd
       break;
   }
   return status;
-});
+}
 function* addRandomKeyTaskCmd(ctx, cmd) {
   var task = yield* taskResult.addRandomKeyTask(ctx, cmd.getDocId());
   cmd.setSaveKey(task.key);
@@ -771,6 +769,11 @@ function* commandImgurls(ctx, conn, cmd, outputData) {
           }
         }
         if (isAllow) {
+          if (format === constants.AVS_OFFICESTUDIO_FILE_IMAGE_TIFF) {
+            data = yield utilsDocService.convertImageToPng(ctx, data);
+            format = constants.AVS_OFFICESTUDIO_FILE_IMAGE_PNG;
+            formatStr = formatChecker.getStringFromFormat(format);
+          }
           let strLocalPath = 'media/' + crypto.randomBytes(16).toString("hex") + '_';
           if (urlParsed) {
             var urlBasename = pathModule.basename(urlParsed.pathname);

DocService/sources/converterservice.js

@@ -386,7 +386,10 @@ function convertRequest(req, res, isJson) {
         cmd.setTitle(path.basename(params.title, path.extname(params.title)) + '.' + outputExt);
       }
       var async = (typeof params.async === 'string') ? 'true' == params.async : params.async;
-
+      if (async && !req.query[constants.SHARD_KEY_API_NAME] && !req.query[constants.SHARD_KEY_WOPI_NAME] && process.env.DEFAULT_SHARD_KEY) {
+        ctx.logger.warn('convertRequest set async=false. Pass query string parameter "%s" to correctly process request in sharded cluster', constants.SHARD_KEY_API_NAME);
+        async = false;
+      }
       if (constants.AVS_OFFICESTUDIO_FILE_UNKNOWN !== cmd.getOutputFormat()) {
         let fileTo = constants.OUTPUT_NAME + '.' + outputExt;
         var status = yield* convertByCmd(ctx, cmd, async, fileTo, undefined, undefined, undefined, undefined, true);
@@ -460,6 +463,10 @@ function builderRequest(req, res) {
           yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
         }
         let async = (typeof params.async === 'string') ? 'true' === params.async : params.async;
+        if (async && !req.query[constants.SHARD_KEY_API_NAME] && !req.query[constants.SHARD_KEY_WOPI_NAME] && process.env.DEFAULT_SHARD_KEY) {
+          ctx.logger.warn('builderRequest set async=false. Pass query string parameter "%s" to correctly process request in sharded cluster', constants.SHARD_KEY_API_NAME);
+          async = false;
+        }
         let status = yield* convertByCmd(ctx, cmd, async, undefined, undefined, constants.QUEUE_PRIORITY_LOW);
         end = status.end;
         error = status.err;

DocService/sources/fileuploaderservice.js

@@ -252,6 +252,11 @@ exports.uploadImageFile = function(req, res) {
           var supportedFormats = tenTypesUpload || 'jpg';
           let formatLimit = formatStr && -1 !== supportedFormats.indexOf(formatStr);
           if (formatLimit) {
+            if (format === constants.AVS_OFFICESTUDIO_FILE_IMAGE_TIFF) {
+              buffer = yield utilsDocService.convertImageToPng(ctx, buffer);
+              format = constants.AVS_OFFICESTUDIO_FILE_IMAGE_PNG;
+              formatStr = formatChecker.getStringFromFormat(format);
+            }
             //a hash is written at the beginning to avoid errors during parallel upload in co-editing
             var strImageName = crypto.randomBytes(16).toString("hex");
             var strPathRel = 'media/' + strImageName + '.' + formatStr;

DocService/sources/utilsDocService.js

@@ -53,20 +53,32 @@ async function fixImageExifRotation(ctx, buffer) {
     let exif = parser.parse();
     if (exif.tags?.Orientation > 1) {
       ctx.logger.debug('fixImageExifRotation remove exif and rotate:%j', exif);
-      let image = await Jimp.read(buffer);
+      buffer = convertImageTo(ctx, buffer, Jimp.AUTO);
-      //remove exif
-      image.bitmap.exifBuffer = undefined;
-      //set jpeg and png quality
-      //https://www.imagemagick.org/script/command-line-options.php#quality
-      image.quality(90);
-      image.deflateLevel(7);
-      buffer = await image.getBufferAsync(Jimp.AUTO);
     }
   } catch (e) {
     ctx.logger.debug('fixImageExifRotation error:%s', e.stack);
   }
   return buffer;
 }
+async function convertImageToPng(ctx, buffer) {
+  return await convertImageTo(ctx, buffer, Jimp.MIME_PNG);
+}
+async function convertImageTo(ctx, buffer, mime) {
+  try {
+    ctx.logger.debug('convertImageTo %s', mime);
+    let image = await Jimp.read(buffer);
+    //remove exif
+    image.bitmap.exifBuffer = undefined;
+    //set jpeg and png quality
+    //https://www.imagemagick.org/script/command-line-options.php#quality
+    image.quality(90);
+    image.deflateLevel(7);
+    buffer = await image.getBufferAsync(mime);
+  } catch (e) {
+    ctx.logger.debug('convertImageTo error:%s', e.stack);
+  }
+  return buffer;
+}
 /**
  *
  * @param {string} lang
@@ -120,5 +132,6 @@ function notifyLicenseExpiration(ctx, endDate) {
 }
 
 module.exports.fixImageExifRotation = fixImageExifRotation;
+module.exports.convertImageToPng = convertImageToPng;
 module.exports.localeToLCID = localeToLCID;
 module.exports.notifyLicenseExpiration = notifyLicenseExpiration;

package.json

@@ -73,6 +73,7 @@
   "scripts": {
     "perf-expired": "cd ./DocService&& cross-env NODE_ENV=development-windows NODE_CONFIG_DIR=../Common/config node ../tests/perf/checkFileExpire.js",
     "perf-exif": "cd ./DocService&& cross-env NODE_ENV=development-windows NODE_CONFIG_DIR=../Common/config node ../tests/perf/fixImageExifRotation.js",
+    "perf-png": "cd ./DocService&& cross-env NODE_ENV=development-windows NODE_CONFIG_DIR=../Common/config node ../tests/perf/convertImageToPng.js",
     "unit tests": "cd ./DocService && jest unit --inject-globals=false --config=../tests/jest.config.js",
     "integration tests with server instance": "cd ./DocService && jest integration/withServerInstance --inject-globals=false --config=../tests/jest.config.js",
     "integration database tests": "cd ./DocService && jest integration/databaseTests --inject-globals=false --config=../tests/jest.config.js",

tests/perf/convertImageToPng.js

@@ -0,0 +1,127 @@
+/*
+ * (c) Copyright Ascensio System SIA 2010-2023
+ *
+ * This program is a free software product. You can redistribute it and/or
+ * modify it under the terms of the GNU Affero General Public License (AGPL)
+ * version 3 as published by the Free Software Foundation. In accordance with
+ * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect
+ * that Ascensio System SIA expressly excludes the warranty of non-infringement
+ * of any third-party rights.
+ *
+ * This program is distributed WITHOUT ANY WARRANTY; without even the implied
+ * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE. For
+ * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
+ *
+ * You can contact Ascensio System SIA at 20A-6 Ernesta Birznieka-Upish
+ * street, Riga, Latvia, EU, LV-1050.
+ *
+ * The  interactive user interfaces in modified source and object code versions
+ * of the Program must display Appropriate Legal Notices, as required under
+ * Section 5 of the GNU AGPL version 3.
+ *
+ * Pursuant to Section 7(b) of the License you must retain the original Product
+ * logo when distributing the program. Pursuant to Section 7(e) we decline to
+ * grant you any rights under trademark law for use of our trademarks.
+ *
+ * All the Product's GUI elements, including illustrations and icon sets, as
+ * well as technical writing content are licensed under the terms of the
+ * Creative Commons Attribution-ShareAlike 4.0 International. See the License
+ * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
+ *
+ */
+
+'use strict';
+
+const {
+  createHistogram,
+  performance,
+  PerformanceObserver,
+} = require('node:perf_hooks');
+
+const { readdir, mkdir, readFile, writeFile } = require("node:fs/promises");
+const path = require("path");
+// const Jimp = require('Jimp');
+const utils = require('./../../Common/sources/utils');
+const operationContext = require('./../../Common/sources/operationContext');
+const utilsDocService = require("./../../DocService/sources/utilsDocService");
+
+let ctx = operationContext.global;
+
+let histograms = {};
+
+async function beforeStart() {
+  let timerify = function (func) {
+    let histogram = createHistogram();
+    histograms[func.name] = histogram;
+    return performance.timerify(func, {histogram: histogram});
+  }
+  utilsDocService.convertImageToPng = timerify(utilsDocService.convertImageToPng);
+  // Jimp.read = timerify(Jimp.read);
+
+  const obs = new PerformanceObserver((list) => {
+    const entries = list.getEntries();
+    entries.forEach((entry) => {
+      let duration = Math.round(entry.duration * 1000) / 1000;
+      console.log(`${entry.name}:${duration}ms`);
+    });
+  });
+  obs.observe({ entryTypes: ['function']});
+}
+
+async function beforeEnd() {
+  let logHistogram = function (histogram, name) {
+    let mean = Math.round(histogram.mean / 1000) / 1000;
+    let min = Math.round(histogram.min / 1000) / 1000;
+    let max = Math.round(histogram.max / 1000) / 1000;
+    let count = histogram.count;
+    ctx.logger.info(`histogram ${name}: count=${count}, mean=${mean}ms, min=${min}ms, max=${max}ms`);
+  }
+  await utils.sleep(1000);
+  for (let name in histograms) {
+    logHistogram(histograms[name], name);
+  }
+}
+
+async function fixInDir(dirIn, dirOut) {
+  ctx.logger.info("dirIn:%s", dirIn);
+  ctx.logger.info("dirOut:%s", dirOut);
+  let dirents = await readdir(dirIn, {withFileTypes : true, recursive: true});
+  for (let dirent of dirents) {
+    if (dirent.isFile()) {
+      let file = dirent.name;
+      ctx.logger.info("fixInDir:%s", file);
+      let buffer = await readFile(path.join(dirent.path, file));
+      let bufferNew = await utilsDocService.convertImageToPng(ctx, buffer);
+      if (buffer !== bufferNew) {
+        let outputPath = path.join(dirOut, dirent.path.substring(dirIn.length), path.basename(file, path.extname(file)) + '.png');
+        await mkdir(path.dirname(outputPath), {recursive: true});
+        await writeFile(outputPath, bufferNew);
+      }
+    }
+  }
+}
+
+async function startTest() {
+  let args = process.argv.slice(2);
+  if (args.length < 2) {
+    ctx.logger.error('missing arguments.USAGE: convertImageToPng.js "dirIn" "dirOut"');
+    return;
+  }
+  ctx.logger.info("test started");
+  await beforeStart();
+
+
+  await fixInDir(args[0], args[1]);
+
+  await beforeEnd();
+  ctx.logger.info("test finished");
+}
+
+startTest().then(()=>{
+  //delay to log observer events
+  return utils.sleep(1000);
+}).catch((err) => {
+  ctx.logger.error(err.stack);
+}).finally(() => {
+  process.exit(0);
+});

tests/perf/fixImageExifRotation.js

@@ -43,7 +43,7 @@ const path = require("path");
 // const Jimp = require('Jimp');
 const utils = require('./../../Common/sources/utils');
 const operationContext = require('./../../Common/sources/operationContext');
-const docsCoServer = require("./../../DocService/sources/DocsCoServer");
+const utilsDocService = require("./../../DocService/sources/utilsDocService");
 
 let ctx = operationContext.global;
 
@@ -55,7 +55,7 @@ async function beforeStart() {
     histograms[func.name] = histogram;
     return performance.timerify(func, {histogram: histogram});
   }
-  docsCoServer.fixImageExifRotation = timerify(docsCoServer.fixImageExifRotation);
+  utilsDocService.fixImageExifRotation = timerify(utilsDocService.fixImageExifRotation);
   // Jimp.read = timerify(Jimp.read);
 
   const obs = new PerformanceObserver((list) => {
@@ -91,7 +91,7 @@ async function fixInDir(dirIn, dirOut) {
       let file = dirent.name;
       ctx.logger.info("fixInDir:%s", file);
       let buffer = await readFile(path.join(dirent.path, file));
-      let bufferNew = await docsCoServer.fixImageExifRotation(ctx, buffer);
+      let bufferNew = await utilsDocService.fixImageExifRotation(ctx, buffer);
       if (buffer !== bufferNew) {
         let outputPath = path.join(dirOut, dirent.path.substring(dirIn.length), file);
         await mkdir(path.dirname(outputPath), {recursive: true});