pengjunjie/server
1
0
Fork 1
mirror of https://github.com/ONLYOFFICE/server.git synced 2026-02-10 18:05:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'hotfix/v8.1.1' into develop

Browse Source 
# Conflicts:
#	DocService/sources/utilsDocService.js
This commit is contained in:
Sergey Konovalov
2024-07-26 12:21:03 +03:00
parent 9822859572 61b5e5d0b9
commit 1268de5dbd
10 changed files with 217 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Common/config/default.json
View File

@ -245,7 +245,7 @@
"utils": {
"utils_common_fontdir": "null",
"utils_fonts_search_patterns": "*.ttf;*.ttc;*.otf",
"limits_image_types_upload": "jpg;jpeg;jpe;png;gif;bmp;svg"
"limits_image_types_upload": "jpg;jpeg;jpe;png;gif;bmp;svg;tiff;tif"
},
"sql": {
"type": "postgres",

4
Common/sources/storage-fs.js
View File

@ -151,7 +151,11 @@ async function getSignedUrl(ctx, storageCfg, baseUrl, strPath, urlType, optFilen
} else if (ctx.wopiSrc) {
wopiSrcCached = ctx.wopiSrc;
url += `&${constants.SHARD_KEY_WOPI_NAME}=${encodeURIComponent(ctx.wopiSrc)}`;
} else if (process.env.DEFAULT_SHARD_KEY) {
//Set DEFAULT_SHARD_KEY from environment as shardkey in case of integrator did not pass this param
url += `&${constants.SHARD_KEY_API_NAME}=${encodeURIComponent(process.env.DEFAULT_SHARD_KEY)}`;
} else if (shardKeyCached) {
//Add stubs for shardkey params until integrators pass these parameters to all requests
url += `&${constants.SHARD_KEY_API_NAME}=${encodeURIComponent(shardKeyCached)}`;
} else if (wopiSrcCached) {
url += `&${constants.SHARD_KEY_WOPI_NAME}=${encodeURIComponent(wopiSrcCached)}`;

16
DocService/sources/DocsCoServer.js
View File

@ -836,7 +836,8 @@ function* setForceSave(ctx, docId, forceSave, cmd, success, url) {
convertInfo.setResponseKey(undefined);
convertInfo.setFormData(undefined);
if (convertInfo.getForceSave()) {
convertInfo.getForceSave().setType(undefined);
//type must be saved to distinguish c_oAscForceSaveTypes.Form
//convertInfo.getForceSave().setType(undefined);
convertInfo.getForceSave().setAuthorUserId(undefined);
convertInfo.getForceSave().setAuthorUserIndex(undefined);
}
@ -884,7 +885,13 @@ async function applyForceSaveCache(ctx, docId, forceSave, type, opt_userConnecti
let forceSaveCache = await checkForceSaveCache(ctx, forceSave.convertInfo);
if (forceSaveCache.hasCache || forceSave.ended) {
if (commonDefines.c_oAscForceSaveTypes.Form === type || commonDefines.c_oAscForceSaveTypes.Internal === type || !forceSave.ended) {
if (forceSaveCache.hasValidCache) {
//c_oAscForceSaveTypes.Form has uniqueue options {'documentLayout': {'isPrint': true}}; dont use it for other types
let forceSaveCached = forceSaveCache.cmd?.getForceSave()?.getType();
let cacheHasSameOptions = (commonDefines.c_oAscForceSaveTypes.Form === type &&
commonDefines.c_oAscForceSaveTypes.Form === forceSaveCached) ||
(commonDefines.c_oAscForceSaveTypes.Form !== type &&
commonDefines.c_oAscForceSaveTypes.Form !== forceSaveCached)
if (forceSaveCache.hasValidCache && cacheHasSameOptions) {
let cmd = forceSaveCache.cmd;
cmd.setUserConnectionDocId(opt_userConnectionDocId);
cmd.setUserConnectionId(opt_userConnectionId);
@ -899,7 +906,7 @@ async function applyForceSaveCache(ctx, docId, forceSave, type, opt_userConnecti
await canvasService.commandSfcCallback(ctx, cmd, true, false);
res.ok = true;
} else {
await editorData.checkAndSetForceSave(ctx, docId, forceSave.getTime(), forceSave.getIndex(), false, false, null);
await editorData.checkAndSetForceSave(ctx, docId, forceSave.time, forceSave.index, false, false, null);
res.startedForceSave = await editorData.checkAndStartForceSave(ctx, docId);
res.ok = !!res.startedForceSave;
}
@ -4359,6 +4366,9 @@ exports.commandFromServer = function (req, res) {
output.error = validateInputParams(ctx, authRes, params);
if (output.error === commonDefines.c_oAscServerCommandErrors.NoError) {
ctx.logger.debug('commandFromServer: c = %s', params.c);
if (params.key && !req.query[constants.SHARD_KEY_API_NAME] && !req.query[constants.SHARD_KEY_WOPI_NAME] && process.env.DEFAULT_SHARD_KEY) {
ctx.logger.warn('commandFromServer. Pass query string parameter "%s" to correctly process commands with "key" in sharded cluster', constants.SHARD_KEY_API_NAME);
}
yield *commandHandle(ctx, params, req, output);
}
} catch (err) {

65
DocService/sources/canvasservice.js
View File

@ -167,11 +167,11 @@ function getOpenedAtJSONParams(row) {
return undefined;
}
var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAdditionalOutput, opt_bIsRestore) {
async function getOutputData(ctx, cmd, outputData, key, optConn, optAdditionalOutput, opt_bIsRestore) {
const tenExpUpdateVersionStatus = ms(ctx.getCfg('services.CoAuthoring.expire.updateVersionStatus', cfgExpUpdateVersionStatus));
let status, statusInfo, password, creationDate, openedAt, row;
let selectRes = yield taskResult.select(ctx, key);
let selectRes = await taskResult.select(ctx, key);
if (selectRes.length > 0) {
row = selectRes[0];
status = row.status;
@ -184,31 +184,29 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd
case commonDefines.FileStatus.SaveVersion:
case commonDefines.FileStatus.UpdateVersion:
case commonDefines.FileStatus.Ok:
if(commonDefines.FileStatus.Ok == status) {
if(commonDefines.FileStatus.Ok === status) {
outputData.setStatus('ok');
} else if (commonDefines.FileStatus.SaveVersion == status ||
} else if (optConn && (optConn.user.view || optConn.isCloseCoAuthoring)) {
outputData.setStatus('ok');
} else if (commonDefines.FileStatus.SaveVersion === status ||
(!opt_bIsRestore && commonDefines.FileStatus.UpdateVersion === status &&
Date.now() - statusInfo * 60000 > tenExpUpdateVersionStatus)) {
if (optConn && (optConn.user.view || optConn.isCloseCoAuthoring)) {
outputData.setStatus(constants.FILE_STATUS_UPDATE_VERSION);
if (commonDefines.FileStatus.UpdateVersion === status) {
ctx.logger.warn("UpdateVersion expired");
}
var updateMask = new taskResult.TaskResultData();
updateMask.tenant = ctx.tenant;
updateMask.key = key;
updateMask.status = status;
updateMask.statusInfo = statusInfo;
var updateTask = new taskResult.TaskResultData();
updateTask.status = commonDefines.FileStatus.Ok;
updateTask.statusInfo = constants.NO_ERROR;
var updateIfRes = await taskResult.updateIf(ctx, updateTask, updateMask);
if (updateIfRes.affectedRows > 0) {
outputData.setStatus('ok');
} else {
if (commonDefines.FileStatus.UpdateVersion === status) {
ctx.logger.warn("UpdateVersion expired");
}
var updateMask = new taskResult.TaskResultData();
updateMask.tenant = ctx.tenant;
updateMask.key = key;
updateMask.status = status;
updateMask.statusInfo = statusInfo;
var updateTask = new taskResult.TaskResultData();
updateTask.status = commonDefines.FileStatus.Ok;
updateTask.statusInfo = constants.NO_ERROR;
var updateIfRes = yield taskResult.updateIf(ctx, updateTask, updateMask);
if (updateIfRes.affectedRows > 0) {
outputData.setStatus('ok');
} else {
outputData.setStatus(constants.FILE_STATUS_UPDATE_VERSION);
}
outputData.setStatus(constants.FILE_STATUS_UPDATE_VERSION);
}
} else {
outputData.setStatus(constants.FILE_STATUS_UPDATE_VERSION);
@ -219,9 +217,9 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd
if (optConn) {
let url;
if(cmd.getInline()) {
url = yield getPrintFileUrl(ctx, key, optConn.baseUrl, cmd.getTitle());
url = await getPrintFileUrl(ctx, key, optConn.baseUrl, cmd.getTitle());
} else {
url = yield storage.getSignedUrl(ctx, optConn.baseUrl, strPath, commonDefines.c_oAscUrlTypes.Temporary,
url = await storage.getSignedUrl(ctx, optConn.baseUrl, strPath, commonDefines.c_oAscUrlTypes.Temporary,
cmd.getTitle());
}
outputData.setData(url);
@ -237,8 +235,8 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd
let decryptedPassword;
let isCorrectPassword;
if (password && encryptedUserPassword) {
decryptedPassword = yield utils.decryptPassword(ctx, password);
userPassword = yield utils.decryptPassword(ctx, encryptedUserPassword);
decryptedPassword = await utils.decryptPassword(ctx, password);
userPassword = await utils.decryptPassword(ctx, encryptedUserPassword);
isCorrectPassword = decryptedPassword === userPassword;
}
if(password && !isCorrectPassword) {
@ -252,7 +250,7 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd
}
} else if (optConn) {
outputData.setOpenedAt(openedAt);
outputData.setData(yield storage.getSignedUrls(ctx, optConn.baseUrl, key, commonDefines.c_oAscUrlTypes.Session, creationDate));
outputData.setData(await storage.getSignedUrls(ctx, optConn.baseUrl, key, commonDefines.c_oAscUrlTypes.Session, creationDate));
} else if (optAdditionalOutput) {
optAdditionalOutput.needUrlKey = key;
optAdditionalOutput.needUrlMethod = 0;
@ -267,7 +265,7 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd
outputData.setStatus('needparams');
var settingsPath = key + '/' + 'origin.' + cmd.getFormat();
if (optConn) {
let url = yield storage.getSignedUrl(ctx, optConn.baseUrl, settingsPath, commonDefines.c_oAscUrlTypes.Temporary);
let url = await storage.getSignedUrl(ctx, optConn.baseUrl, settingsPath, commonDefines.c_oAscUrlTypes.Temporary);
outputData.setData(url);
} else if (optAdditionalOutput) {
optAdditionalOutput.needUrlKey = settingsPath;
@ -286,7 +284,7 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd
case commonDefines.FileStatus.ErrToReload:
outputData.setStatus('err');
outputData.setData(statusInfo);
yield cleanupErrToReload(ctx, key);
await cleanupErrToReload(ctx, key);
break;
case commonDefines.FileStatus.None:
//this status has no handler
@ -300,7 +298,7 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd
break;
}
return status;
});
}
function* addRandomKeyTaskCmd(ctx, cmd) {
var task = yield* taskResult.addRandomKeyTask(ctx, cmd.getDocId());
cmd.setSaveKey(task.key);
@ -771,6 +769,11 @@ function* commandImgurls(ctx, conn, cmd, outputData) {
}
}
if (isAllow) {
if (format === constants.AVS_OFFICESTUDIO_FILE_IMAGE_TIFF) {
data = yield utilsDocService.convertImageToPng(ctx, data);
format = constants.AVS_OFFICESTUDIO_FILE_IMAGE_PNG;
formatStr = formatChecker.getStringFromFormat(format);
}
let strLocalPath = 'media/' + crypto.randomBytes(16).toString("hex") + '_';
if (urlParsed) {
var urlBasename = pathModule.basename(urlParsed.pathname);

9
DocService/sources/converterservice.js
View File

@ -386,7 +386,10 @@ function convertRequest(req, res, isJson) {
cmd.setTitle(path.basename(params.title, path.extname(params.title)) + '.' + outputExt);
}
var async = (typeof params.async === 'string') ? 'true' == params.async : params.async;
if (async && !req.query[constants.SHARD_KEY_API_NAME] && !req.query[constants.SHARD_KEY_WOPI_NAME] && process.env.DEFAULT_SHARD_KEY) {
ctx.logger.warn('convertRequest set async=false. Pass query string parameter "%s" to correctly process request in sharded cluster', constants.SHARD_KEY_API_NAME);
async = false;
}
if (constants.AVS_OFFICESTUDIO_FILE_UNKNOWN !== cmd.getOutputFormat()) {
let fileTo = constants.OUTPUT_NAME + '.' + outputExt;
var status = yield* convertByCmd(ctx, cmd, async, fileTo, undefined, undefined, undefined, undefined, true);
@ -460,6 +463,10 @@ function builderRequest(req, res) {
yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
}
let async = (typeof params.async === 'string') ? 'true' === params.async : params.async;
if (async && !req.query[constants.SHARD_KEY_API_NAME] && !req.query[constants.SHARD_KEY_WOPI_NAME] && process.env.DEFAULT_SHARD_KEY) {
ctx.logger.warn('builderRequest set async=false. Pass query string parameter "%s" to correctly process request in sharded cluster', constants.SHARD_KEY_API_NAME);
async = false;
}
let status = yield* convertByCmd(ctx, cmd, async, undefined, undefined, constants.QUEUE_PRIORITY_LOW);
end = status.end;
error = status.err;

5
DocService/sources/fileuploaderservice.js
View File

@ -252,6 +252,11 @@ exports.uploadImageFile = function(req, res) {
var supportedFormats = tenTypesUpload || 'jpg';
let formatLimit = formatStr && -1 !== supportedFormats.indexOf(formatStr);
if (formatLimit) {
if (format === constants.AVS_OFFICESTUDIO_FILE_IMAGE_TIFF) {
buffer = yield utilsDocService.convertImageToPng(ctx, buffer);
format = constants.AVS_OFFICESTUDIO_FILE_IMAGE_PNG;
formatStr = formatChecker.getStringFromFormat(format);
}
//a hash is written at the beginning to avoid errors during parallel upload in co-editing
var strImageName = crypto.randomBytes(16).toString("hex");
var strPathRel = 'media/' + strImageName + '.' + formatStr;

29
DocService/sources/utilsDocService.js
View File

@ -53,20 +53,32 @@ async function fixImageExifRotation(ctx, buffer) {
let exif = parser.parse();
if (exif.tags?.Orientation > 1) {
ctx.logger.debug('fixImageExifRotation remove exif and rotate:%j', exif);
let image = await Jimp.read(buffer);
//remove exif
image.bitmap.exifBuffer = undefined;
//set jpeg and png quality
//https://www.imagemagick.org/script/command-line-options.php#quality
image.quality(90);
image.deflateLevel(7);
buffer = await image.getBufferAsync(Jimp.AUTO);
buffer = convertImageTo(ctx, buffer, Jimp.AUTO);
}
} catch (e) {
ctx.logger.debug('fixImageExifRotation error:%s', e.stack);
}
return buffer;
}
async function convertImageToPng(ctx, buffer) {
return await convertImageTo(ctx, buffer, Jimp.MIME_PNG);
}
async function convertImageTo(ctx, buffer, mime) {
try {
ctx.logger.debug('convertImageTo %s', mime);
let image = await Jimp.read(buffer);
//remove exif
image.bitmap.exifBuffer = undefined;
//set jpeg and png quality
//https://www.imagemagick.org/script/command-line-options.php#quality
image.quality(90);
image.deflateLevel(7);
buffer = await image.getBufferAsync(mime);
} catch (e) {
ctx.logger.debug('convertImageTo error:%s', e.stack);
}
return buffer;
}
/**
*
* @param {string} lang
@ -120,5 +132,6 @@ function notifyLicenseExpiration(ctx, endDate) {
}
module.exports.fixImageExifRotation = fixImageExifRotation;
module.exports.convertImageToPng = convertImageToPng;
module.exports.localeToLCID = localeToLCID;
module.exports.notifyLicenseExpiration = notifyLicenseExpiration;

1
package.json
View File

@ -73,6 +73,7 @@
"scripts": {
"perf-expired": "cd ./DocService&& cross-env NODE_ENV=development-windows NODE_CONFIG_DIR=../Common/config node ../tests/perf/checkFileExpire.js",
"perf-exif": "cd ./DocService&& cross-env NODE_ENV=development-windows NODE_CONFIG_DIR=../Common/config node ../tests/perf/fixImageExifRotation.js",
"perf-png": "cd ./DocService&& cross-env NODE_ENV=development-windows NODE_CONFIG_DIR=../Common/config node ../tests/perf/convertImageToPng.js",
"unit tests": "cd ./DocService && jest unit --inject-globals=false --config=../tests/jest.config.js",
"integration tests with server instance": "cd ./DocService && jest integration/withServerInstance --inject-globals=false --config=../tests/jest.config.js",
"integration database tests": "cd ./DocService && jest integration/databaseTests --inject-globals=false --config=../tests/jest.config.js",

127
tests/perf/convertImageToPng.js Normal file
View File

@ -0,0 +1,127 @@
/*
* (c) Copyright Ascensio System SIA 2010-2023
*
* This program is a free software product. You can redistribute it and/or
* modify it under the terms of the GNU Affero General Public License (AGPL)
* version 3 as published by the Free Software Foundation. In accordance with
* Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect
* that Ascensio System SIA expressly excludes the warranty of non-infringement
* of any third-party rights.
*
* This program is distributed WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For
* details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
*
* You can contact Ascensio System SIA at 20A-6 Ernesta Birznieka-Upish
* street, Riga, Latvia, EU, LV-1050.
*
* The interactive user interfaces in modified source and object code versions
* of the Program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU AGPL version 3.
*
* Pursuant to Section 7(b) of the License you must retain the original Product
* logo when distributing the program. Pursuant to Section 7(e) we decline to
* grant you any rights under trademark law for use of our trademarks.
*
* All the Product's GUI elements, including illustrations and icon sets, as
* well as technical writing content are licensed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International. See the License
* terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
*
*/
'use strict';
const {
createHistogram,
performance,
PerformanceObserver,
} = require('node:perf_hooks');
const { readdir, mkdir, readFile, writeFile } = require("node:fs/promises");
const path = require("path");
// const Jimp = require('Jimp');
const utils = require('./../../Common/sources/utils');
const operationContext = require('./../../Common/sources/operationContext');
const utilsDocService = require("./../../DocService/sources/utilsDocService");
let ctx = operationContext.global;
let histograms = {};
async function beforeStart() {
let timerify = function (func) {
let histogram = createHistogram();
histograms[func.name] = histogram;
return performance.timerify(func, {histogram: histogram});
}
utilsDocService.convertImageToPng = timerify(utilsDocService.convertImageToPng);
// Jimp.read = timerify(Jimp.read);
const obs = new PerformanceObserver((list) => {
const entries = list.getEntries();
entries.forEach((entry) => {
let duration = Math.round(entry.duration * 1000) / 1000;
console.log(`${entry.name}:${duration}ms`);
});
});
obs.observe({ entryTypes: ['function']});
}
async function beforeEnd() {
let logHistogram = function (histogram, name) {
let mean = Math.round(histogram.mean / 1000) / 1000;
let min = Math.round(histogram.min / 1000) / 1000;
let max = Math.round(histogram.max / 1000) / 1000;
let count = histogram.count;
ctx.logger.info(`histogram ${name}: count=${count}, mean=${mean}ms, min=${min}ms, max=${max}ms`);
}
await utils.sleep(1000);
for (let name in histograms) {
logHistogram(histograms[name], name);
}
}
async function fixInDir(dirIn, dirOut) {
ctx.logger.info("dirIn:%s", dirIn);
ctx.logger.info("dirOut:%s", dirOut);
let dirents = await readdir(dirIn, {withFileTypes : true, recursive: true});
for (let dirent of dirents) {
if (dirent.isFile()) {
let file = dirent.name;
ctx.logger.info("fixInDir:%s", file);
let buffer = await readFile(path.join(dirent.path, file));
let bufferNew = await utilsDocService.convertImageToPng(ctx, buffer);
if (buffer !== bufferNew) {
let outputPath = path.join(dirOut, dirent.path.substring(dirIn.length), path.basename(file, path.extname(file)) + '.png');
await mkdir(path.dirname(outputPath), {recursive: true});
await writeFile(outputPath, bufferNew);
}
}
}
}
async function startTest() {
let args = process.argv.slice(2);
if (args.length < 2) {
ctx.logger.error('missing arguments.USAGE: convertImageToPng.js "dirIn" "dirOut"');
return;
}
ctx.logger.info("test started");
await beforeStart();
await fixInDir(args[0], args[1]);
await beforeEnd();
ctx.logger.info("test finished");
}
startTest().then(()=>{
//delay to log observer events
return utils.sleep(1000);
}).catch((err) => {
ctx.logger.error(err.stack);
}).finally(() => {
process.exit(0);
});

6
tests/perf/fixImageExifRotation.js
View File

@ -43,7 +43,7 @@ const path = require("path");
// const Jimp = require('Jimp');
const utils = require('./../../Common/sources/utils');
const operationContext = require('./../../Common/sources/operationContext');
const docsCoServer = require("./../../DocService/sources/DocsCoServer");
const utilsDocService = require("./../../DocService/sources/utilsDocService");
let ctx = operationContext.global;
@ -55,7 +55,7 @@ async function beforeStart() {
histograms[func.name] = histogram;
return performance.timerify(func, {histogram: histogram});
}
docsCoServer.fixImageExifRotation = timerify(docsCoServer.fixImageExifRotation);
utilsDocService.fixImageExifRotation = timerify(utilsDocService.fixImageExifRotation);
// Jimp.read = timerify(Jimp.read);
const obs = new PerformanceObserver((list) => {
@ -91,7 +91,7 @@ async function fixInDir(dirIn, dirOut) {
let file = dirent.name;
ctx.logger.info("fixInDir:%s", file);
let buffer = await readFile(path.join(dirent.path, file));
let bufferNew = await docsCoServer.fixImageExifRotation(ctx, buffer);
let bufferNew = await utilsDocService.fixImageExifRotation(ctx, buffer);
if (buffer !== bufferNew) {
let outputPath = path.join(dirOut, dirent.path.substring(dirIn.length), file);
await mkdir(path.dirname(outputPath), {recursive: true});