Merge remote-tracking branch 'remotes/origin/release/v8.0.0' into develop

# Conflicts:
#	Common/sources/storage-fs.js
#	DocService/package.json
#	DocService/sources/DocsCoServer.js
#	DocService/sources/databaseConnectors/connectorUtilities.js
#	DocService/sources/databaseConnectors/postgreConnector.js
#	DocService/sources/gc.js
#	DocService/sources/mySqlBaseConnector.js
#	DocService/sources/wopiClient.js
This commit is contained in:
Sergey Konovalov
2024-02-05 14:43:02 +03:00
14 changed files with 181 additions and 53 deletions

View File

@ -100,7 +100,11 @@
"modulusOld": "qnro3nUUjvZK1i7UqeOlXmCrVPiDtHlRgIPReAjt2nKL1GG3SBXO6N0aPbiM5rtK0XRPUoLmKu2rYvSJ/Kmkdp14a/3uiEl788VVn0hb/l9OuQtH3HBjmM0/LKRgJQuU3LgHI67uRVZYtSJ/n9fYdZqnLfveLsrgZpgRCoabrp+H5Uem9N+x0OJR3LpToVRZhzSkYQrxnERJmF3bhR5yF8Zn+3BoSiUpVOCAvJRAYl8cAIs3BwQcTEyXJjnt+wW5Q1VyKr+bXp/39+tnugQeTe1jjdPy6rOTftQwzjro81oZpOMazwwR1aeQuQWCrmHQZqyV3Rvo6X3xYlOQnlo1/w==",
"exponentOld": "AQAB",
"privateKeyOld": "MIIEowIBAAKCAQEAqnro3nUUjvZK1i7UqeOlXmCrVPiDtHlRgIPReAjt2nKL1GG3SBXO6N0aPbiM5rtK0XRPUoLmKu2rYvSJ/Kmkdp14a/3uiEl788VVn0hb/l9OuQtH3HBjmM0/LKRgJQuU3LgHI67uRVZYtSJ/n9fYdZqnLfveLsrgZpgRCoabrp+H5Uem9N+x0OJR3LpToVRZhzSkYQrxnERJmF3bhR5yF8Zn+3BoSiUpVOCAvJRAYl8cAIs3BwQcTEyXJjnt+wW5Q1VyKr+bXp/39+tnugQeTe1jjdPy6rOTftQwzjro81oZpOMazwwR1aeQuQWCrmHQZqyV3Rvo6X3xYlOQnlo1/wIDAQABAoIBAQCKtUSBs8tNYrGTQTlBHXrwpkDg+u7WSZt5sEcfnkxA39BLtlHU8gGO0E9Ihr8GAL+oWjUsEltJ9GTtN8CJ9lFdPVS8sTiCZR/YQOggmFRZTJyVzMrkXgF7Uwwiu3+KxLiTOZx9eRhfDBlTD8W9fXaegX2i2Xp2ohUhBHthEBLdaZTWFi5Sid/Y0dDzBeP6UIJorZ5D+1ybaeIVHjndpwNsIEUGUxPFLrkeiU8Rm4MJ9ahxfywcP7DjQoPGY9Ge5cBhpxfzERWf732wUD6o3+L9tvOBU00CLVjULbGZKTVE2FJMyXK9jr6Zor9Mkhomp6/8Agkr9rp+TPyelFGYEz8hAoGBAOEc09CrL3eYBkhNEcaMQzxBLvOGpg8kaDX5SaArHfl9+U9yzRqss4ARECanp9HuHfjMQo7iejao0ngDjL7BNMSaH74QlSsPOY2iOm8Qvx8/zb7g4h9r1zLjFZb3mpSA4snRZvvdiZ9ugbuVPmhXnDzRRMg45MibJeeOTJNylofRAoGBAMHfF/WutqKDoX25qZo9m74W4bttOj6oIDk1N4/c6M1Z1v/aptYSE06bkWngj9P46kqjaay4hgMtzyGruc5aojPx5MHHf5bo14+Jv4NzYtR2llrUxO+UJX7aCfUYXI7RC93GUmhpeQ414j7SNAXec58d7e+ETw+6cHiAWO4uOSTPAoGATPq5qDLR4Zi4FUNdn8LZPyKfNqHF6YmupT5hIgd8kZO1jKiaYNPL8jBjkIRmjBBcaXcYD5p85nImvumf2J9jNxPpZOpwyC/Fo5xlVROp97qu1eY7DTmodntXJ6/2SXAlnZQhHmHsrPtyG752f+HtyJJbbgiem8cKWDu+DfHybfECgYBbSLo1WiBwgN4nHqZ3E48jgA6le5azLeKOTTpuKKwNFMIhEkj//t7MYn+jhLL0Mf3PSwZU50Vidc1To1IHkbFSGBGIFHFFEzl8QnXEZS4hr/y3o/teezj0c6HAn8nlDRUzRVBEDXWMdV6kCcGpCccTIrqHzpqTY0vV0UkOTQFnDQKBgAxSEhm/gtCYJIMCBe+KBJT9uECV5xDQopTTjsGOkd4306EN2dyPOIlAfwM6K/0qWisa0Ei5i8TbRRuBeTTdLEYLqXCJ7fj5tdD1begBdSVtHQ2WHqzPJSuImTkFi9NXxd1XUyZFM3y6YQvlssSuL7QSxUIEtZHnrJTt3QDd10dj",
"refreshLockInterval": "10m"
"refreshLockInterval": "10m",
"dummy" : {
"enable": true,
"sampleFilePath": ""
}
},
"tenants": {
"baseDir": "",

View File

@ -74,8 +74,8 @@ let configS3 = {
};
if (configS3.endpoint) {
configS3.sslEnabled = cfgSslEnabled;
configS3.s3ForcePathStyle = cfgS3ForcePathStyle;
configS3.tls = cfgSslEnabled;
configS3.forcePathStyle = cfgS3ForcePathStyle;
}
const client = new S3Client(configS3);

View File

@ -2657,9 +2657,9 @@
}
},
"oracledb": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/oracledb/-/oracledb-6.0.1.tgz",
"integrity": "sha512-N5/Y4IkCFQCumqjEXi3snrclDKjDMMeawq/z/5Ydm5+BxjV3kg9wCaTAp++JTLlCWASb4JMXqK70PctmAkZh3g=="
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/oracledb/-/oracledb-6.3.0.tgz",
"integrity": "sha512-fr3U66QxgGXb5cs/ozLBQU50TMbZcBQEWvSaj2rJAXG8KRrsZcGOK8JTlZL1yJHeW8cSjOm6n/wTw3SJksGjDg=="
},
"packet-reader": {
"version": "1.0.0",

View File

@ -36,7 +36,7 @@
"multi-integer-range": "4.0.7",
"multiparty": "4.2.1",
"mysql2": "2.3.3",
"oracledb": "6.0.1",
"oracledb": "6.3.0",
"pg": "8.11.3",
"redis": "4.6.11",
"retry": "0.12.0",

View File

@ -1396,7 +1396,7 @@ function getRequestParams(ctx, req, opt_isNotInBody) {
const tenTokenEnableRequestInbox = ctx.getCfg('services.CoAuthoring.token.enable.request.inbox', cfgTokenEnableRequestInbox);
const tenTokenRequiredParams = ctx.getCfg('services.CoAuthoring.server.tokenRequiredParams', cfgTokenRequiredParams);
let res = {code: constants.NO_ERROR, isDecoded: false, params: undefined};
let res = {code: constants.NO_ERROR, description: "", isDecoded: false, params: undefined};
if (req.body && Buffer.isBuffer(req.body) && req.body.length > 0 && !opt_isNotInBody) {
res.params = JSON.parse(req.body.toString('utf8'));
} else {
@ -1427,6 +1427,7 @@ function getRequestParams(ctx, req, opt_isNotInBody) {
} else if (constants.JWT_EXPIRED_CODE == checkJwtRes.code) {
res.code = constants.VKEY_KEY_EXPIRE;
}
res.description = checkJwtRes.description;
}
}
return res;

View File

@ -292,7 +292,7 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd
}
break;
case commonDefines.FileStatus.None:
outputData.setStatus('none');
//this status has no handler
break;
case commonDefines.FileStatus.WaitQueue:
//task in the queue. response will be after convertion
@ -302,6 +302,7 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd
outputData.setData(constants.UNKNOWN);
break;
}
return status;
});
function* addRandomKeyTaskCmd(ctx, cmd) {
var task = yield* taskResult.addRandomKeyTask(ctx, cmd.getDocId());
@ -517,8 +518,8 @@ function* commandOpen(ctx, conn, cmd, outputData, opt_upsertRes, opt_bIsRestore)
}
}
function* commandOpenFillOutput(ctx, conn, cmd, outputData, opt_bIsRestore) {
yield getOutputData(ctx, cmd, outputData, cmd.getDocId(), conn, undefined, opt_bIsRestore);
return 'none' === outputData.getStatus();
let status = yield getOutputData(ctx, cmd, outputData, cmd.getDocId(), conn, undefined, opt_bIsRestore);
return commonDefines.FileStatus.None === status;
}
function* commandReopen(ctx, conn, cmd, outputData) {
const tenOpenProtectedFile = ctx.getCfg('services.CoAuthoring.server.openProtectedFile', cfgOpenProtectedFile);
@ -1336,7 +1337,7 @@ exports.openDocument = function(ctx, conn, cmd, opt_upsertRes, opt_bIsRestore) {
outputData.setData(constants.UNKNOWN);
}
finally {
if (outputData && outputData.getStatus()) {
if (outputData?.getStatus()) {
ctx.logger.debug('Response command: %s', JSON.stringify(outputData));
docsCoServer.sendData(ctx, conn, new OutputDataWrap('documentOpen', outputData));
}
@ -1583,8 +1584,12 @@ exports.downloadFile = function(req, res) {
}
ctx.initFromRequest(req);
yield ctx.initTenantCache();
let url = decodeURI(req.get('x-url'));
ctx.setDocId(req.params.docid);
//todo remove in 8.1. For compatibility
let url = req.get('x-url');
if (url) {
url = decodeURI(url);
}
ctx.logger.info('Start downloadFile');
const tenTokenEnableBrowser = ctx.getCfg('services.CoAuthoring.token.enable.browser', cfgTokenEnableBrowser);
const tenDownloadMaxBytes = ctx.getCfg('FileConverter.converter.maxDownloadBytes', cfgDownloadMaxBytes);
@ -1593,32 +1598,35 @@ exports.downloadFile = function(req, res) {
const tenAllowPrivateIPAddressForSignedRequests = ctx.getCfg('services.CoAuthoring.server.allowPrivateIPAddressForSignedRequests', cfgAllowPrivateIPAddressForSignedRequests);
let authorization;
if (tenTokenEnableBrowser) {
let checkJwtRes = yield docsCoServer.checkJwtHeader(ctx, req, 'Authorization', 'Bearer ', commonDefines.c_oAscSecretType.Browser);
let errorDescription;
if (checkJwtRes.decoded) {
let decoded = checkJwtRes.decoded;
if (decoded.changesUrl) {
url = decoded.changesUrl;
} else if (decoded.document && -1 !== tenDownloadFileAllowExt.indexOf(decoded.document.fileType)) {
url = decoded.document.url;
} else if (decoded.url && -1 !== tenDownloadFileAllowExt.indexOf(decoded.fileType)) {
let errorDescription;
let authRes = yield docsCoServer.getRequestParams(ctx, req);
if (authRes.code === constants.NO_ERROR) {
let decoded = authRes.params;
if (decoded.changesUrl) {
url = decoded.changesUrl;
} else if (decoded.document && -1 !== tenDownloadFileAllowExt.indexOf(decoded.document.fileType)) {
url = decoded.document.url;
} else if (decoded.url && -1 !== tenDownloadFileAllowExt.indexOf(decoded.fileType)) {
url = decoded.url;
} else if (!tenTokenEnableBrowser) {
//todo token required
if (decoded.url) {
url = decoded.url;
} else {
errorDescription = 'access deny';
}
} else {
errorDescription = checkJwtRes.description;
}
if (errorDescription) {
ctx.logger.warn('Error downloadFile jwt: description = %s', errorDescription);
res.sendStatus(403);
return;
}
if (utils.canIncludeOutboxAuthorization(ctx, url)) {
let secret = yield tenantManager.getTenantSecret(ctx, commonDefines.c_oAscSecretType.Outbox);
authorization = utils.fillJwtForRequest(ctx, {url: url}, secret, false);
errorDescription = 'access deny';
}
} else {
errorDescription = authRes.description || 'need token';
}
if (errorDescription) {
ctx.logger.warn('Error downloadFile jwt: description = %s', errorDescription);
res.sendStatus(403);
return;
}
if (utils.canIncludeOutboxAuthorization(ctx, url)) {
let secret = yield tenantManager.getTenantSecret(ctx, commonDefines.c_oAscSecretType.Outbox);
authorization = utils.fillJwtForRequest(ctx, {url: url}, secret, false);
}
let urlParsed = urlModule.parse(url);
let filterStatus = yield* utils.checkHostFilter(ctx, urlParsed.hostname);

View File

@ -275,6 +275,7 @@ function convertRequest(req, res, isJson) {
cmd.setFormat(filetype);
cmd.setDocId(docId);
cmd.setOutputFormat(outputFormat);
let outputExt = formatChecker.getStringFromFormat(cmd.getOutputFormat());
cmd.setCodepage(commonDefines.c_oAscEncodingsMap[params.codePage] || commonDefines.c_oAscCodePageUtf8);
cmd.setDelimiter(parseIntParam(params.delimiter) || commonDefines.c_oAscCsvDelimiter.Comma);
@ -283,11 +284,18 @@ function convertRequest(req, res, isJson) {
if (params.region && locale[params.region.toLowerCase()]) {
cmd.setLCID(locale[params.region.toLowerCase()].id);
}
let jsonParams = {};
if (params.documentLayout) {
cmd.setJsonParams(JSON.stringify({'documentLayout': params.documentLayout}));
jsonParams['documentLayout'] = params.documentLayout;
}
if (params.spreadsheetLayout) {
cmd.setJsonParams(JSON.stringify({'spreadsheetLayout': params.spreadsheetLayout}));
jsonParams['spreadsheetLayout'] = params.spreadsheetLayout;
}
if (params.watermark) {
jsonParams['watermark'] = params.watermark;
}
if (Object.keys(jsonParams).length > 0) {
cmd.setJsonParams(JSON.stringify(jsonParams));
}
if (params.password) {
if (params.password.length > constants.PASSWORD_MAX_LENGTH) {
@ -323,8 +331,8 @@ function convertRequest(req, res, isJson) {
break;
}
cmd.setThumbnail(thumbnailData);
if (false == thumbnailData.getFirst()) {
cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_IMAGE);
if (false === thumbnailData.getFirst() && 0 !== (constants.AVS_OFFICESTUDIO_FILE_IMAGE & cmd.getOutputFormat())) {
outputExt = 'zip';
}
}
var documentRenderer = params.documentRenderer;
@ -350,7 +358,6 @@ function convertRequest(req, res, isJson) {
}
cmd.setTextParams(textParamsData);
}
let outputExt = formatChecker.getStringFromFormat(cmd.getOutputFormat());
if (params.title) {
cmd.setTitle(path.basename(params.title, path.extname(params.title)) + '.' + outputExt);
}

View File

@ -46,7 +46,7 @@ const connectionConfiguration = {
user: configSql.get('dbUser'),
password: configSql.get('dbPass'),
server: configSql.get('dbHost'),
port: configSql.get('dbPort'),
port: parseInt(configSql.get('dbPort')),
database: configSql.get('dbName'),
pool: {
max: configSql.get('connectionlimit'),

View File

@ -41,7 +41,7 @@ const cfgTableResult = config.get('services.CoAuthoring.sql.tableResult');
const pool = mysql.createPool({
host : configSql.get('dbHost'),
port : configSql.get('dbPort'),
port : parseInt(configSql.get('dbPort')),
user : configSql.get('dbUser'),
password : configSql.get('dbPass'),
database : configSql.get('dbName'),

View File

@ -117,7 +117,16 @@ async function executeQuery(ctx, sqlCommand, values = [], noModifyRes = false, n
throw error;
} finally {
connection?.close();
if (connection) {
try {
// Put the connection back in the pool
await connection.close();
} catch (error) {
if (!noLog) {
ctx.logger.error(`connection.close() error while executing query: ${sqlCommand}\n${error.stack}`);
}
}
}
}
}

View File

@ -45,7 +45,7 @@ const cfgEditor = config.get('services.CoAuthoring.editor');
let connectionConfig = {
host: configSql.get('dbHost'),
port: configSql.get('dbPort'),
port: parseInt(configSql.get('dbPort')),
user: configSql.get('dbUser'),
password: configSql.get('dbPass'),
database: configSql.get('dbName'),

View File

@ -60,6 +60,7 @@ const tenantManager = require('./../../Common/sources/tenantManager');
const configStorage = config.get('storage');
const cfgWopiEnable = config.get('wopi.enable');
const cfgWopiDummyEnable = config.get('wopi.dummy.enable');
const cfgHtmlTemplate = config.get('wopi.htmlTemplate');
const cfgTokenEnableBrowser = config.get('services.CoAuthoring.token.enable.browser');
const cfgTokenEnableRequestInbox = config.get('services.CoAuthoring.token.enable.request.inbox');
@ -249,6 +250,7 @@ docsCoServer.install(server, () => {
app.post('/savefile/:docid', rawFileParser, canvasService.saveFile);
app.get('/printfile/:docid/:filename', canvasService.printFile);
app.get('/downloadfile/:docid', canvasService.downloadFile);
app.post('/downloadfile/:docid', rawFileParser, canvasService.downloadFile);
app.get('/healthcheck', utils.checkClientIp, docsCoServer.healthCheck);
app.get('/baseurl', (req, res) => {
@ -292,6 +294,24 @@ docsCoServer.install(server, () => {
res.sendStatus(404);
});
}
function checkWopiDummyEnable(req, res, next) {
//todo may be move code into wopiClient or wopiClient.discovery...
let ctx = new operationContext.Context();
ctx.initFromRequest(req);
ctx.initTenantCache()
.then(() => {
const tenWopiEnable = ctx.getCfg('wopi.enable', cfgWopiEnable);
const tenWopiDummyEnable = ctx.getCfg('wopi.dummy.enable', cfgWopiDummyEnable);
if (tenWopiEnable && tenWopiDummyEnable) {
next();
} else {
res.sendStatus(404);
}
}).catch((err) => {
ctx.logger.error('checkWopiDummyEnable error: %s', err.stack);
res.sendStatus(404);
});
}
//todo dest
let fileForms = multer({limits: {fieldSize: cfgDownloadMaxBytes}});
app.get('/hosting/discovery', checkWopiEnable, utils.checkClientIp, wopiClient.discovery);
@ -301,8 +321,12 @@ docsCoServer.install(server, () => {
app.post('/hosting/wopi/:documentType/:mode', checkWopiEnable, urleEcodedParser, forms.none(), utils.lowercaseQueryString, wopiClient.getEditorHtml);
app.post('/hosting/wopi/convert-and-edit/:ext/:targetext', checkWopiEnable, urleEcodedParser, forms.none(), utils.lowercaseQueryString, wopiClient.getConverterHtml);
app.get('/hosting/wopi/convert-and-edit-handler', checkWopiEnable, utils.lowercaseQueryString, converterService.getConverterHtmlHandler);
app.get('/wopi/files/:docid', apicache.middleware("5 minutes"), checkWopiDummyEnable, utils.lowercaseQueryString, wopiClient.dummyCheckFileInfo);
app.post('/wopi/files/:docid', checkWopiDummyEnable, wopiClient.dummyOk);
app.get('/wopi/files/:docid/contents', apicache.middleware("5 minutes"), checkWopiDummyEnable, wopiClient.dummyGetFile);
app.post('/wopi/files/:docid/contents', checkWopiDummyEnable, wopiClient.dummyOk);
app.post('/dummyCallback', utils.checkClientIp, rawFileParser, function(req, res){
app.post('/dummyCallback', utils.checkClientIp, apicache.middleware("5 minutes"), rawFileParser, function(req, res){
let ctx = new operationContext.Context();
ctx.initFromRequest(req);
//yield ctx.initTenantCache();//no need

View File

@ -33,13 +33,14 @@
'use strict';
const path = require('path');
const { pipeline } = require('node:stream/promises');
const crypto = require('crypto');
const {URL} = require('url');
const co = require('co');
const jwt = require('jsonwebtoken');
const config = require('config');
const { createReadStream } = require('fs');
const { lstat, readdir } = require('fs/promises');
const { stat, lstat, readdir } = require('fs/promises');
const utf7 = require('utf7');
const mimeDB = require('mime-db');
const xmlbuilder2 = require('xmlbuilder2');
@ -54,6 +55,7 @@ const sqlBase = require('./databaseConnectors/baseConnector');
const taskResult = require('./taskresult');
const canvasService = require('./canvasservice');
const converterService = require('./converterservice');
const mime = require('mime');
const cfgTokenOutboxAlgorithm = config.get('services.CoAuthoring.token.outbox.algorithm');
const cfgTokenOutboxExpires = config.get('services.CoAuthoring.token.outbox.expires');
@ -83,6 +85,7 @@ const cfgWopiModulusOld = config.get('wopi.modulusOld');
const cfgWopiExponentOld = config.get('wopi.exponentOld');
const cfgWopiPrivateKeyOld = config.get('wopi.privateKeyOld');
const cfgWopiHost = config.get('wopi.host');
const cfgWopiDummySampleFilePath = config.get('wopi.dummy.sampleFilePath');
let templatesFolderLocalesCache = null;
const templateFilesSizeCache = {};
@ -862,7 +865,75 @@ function getWopiParams(lockId, fileInfo, wopiSrc, access_token, access_token_ttl
hostSessionId: null, userSessionId: null, mode: null
};
return {commonInfo: commonInfo, userAuth: userAuth, LastModifiedTime: null};
};
}
async function dummyCheckFileInfo(req, res) {
if (true) {
//static output for performance reason
res.json({
BaseFileName: "sample.docx",
OwnerId: "userId",
Size: 100,//no need to set actual size for test
UserId: "userId",//test ignores
UserFriendlyName: "user",
Version: 0,
UserCanWrite: true,
SupportsGetLock: true,
SupportsLocks: true,
SupportsUpdate: true,
});
} else {
let fileInfo;
let ctx = new operationContext.Context();
ctx.initFromRequest(req);
try {
await ctx.initTenantCache();
const tenWopiDummySampleFilePath = ctx.getCfg('wopi.dummy.sampleFilePath', cfgWopiDummySampleFilePath);
let access_token = req.query['access_token'];
ctx.logger.debug('dummyCheckFileInfo access_token:%s', access_token);
let sampleFileStat = await stat(tenWopiDummySampleFilePath);
fileInfo = JSON.parse(Buffer.from(access_token, 'base64').toString('ascii'));
fileInfo.BaseFileName = path.basename(tenWopiDummySampleFilePath);
fileInfo.Size = sampleFileStat.size;
} catch (err) {
ctx.logger.error('dummyCheckFileInfo error:%s', err.stack);
} finally {
if (fileInfo) {
res.json(fileInfo);
} else {
res.sendStatus(400)
}
}
}
}
async function dummyGetFile(req, res) {
let ctx = new operationContext.Context();
ctx.initFromRequest(req);
try {
await ctx.initTenantCache();
const tenWopiDummySampleFilePath = ctx.getCfg('wopi.dummy.sampleFilePath', cfgWopiDummySampleFilePath);
let sampleFileStat = await stat(tenWopiDummySampleFilePath);
res.setHeader('Content-Length', sampleFileStat.size);
res.setHeader('Content-Type', mime.getType(tenWopiDummySampleFilePath));
await pipeline(
createReadStream(tenWopiDummySampleFilePath),
res,
);
} catch (err) {
ctx.logger.error('dummyGetFile error:%s', err.stack);
} finally {
if (!res.headersSent) {
res.sendStatus(400);
}
}
}
function dummyOk(req, res) {
res.sendStatus(200);
}
exports.discovery = discovery;
exports.collaboraCapabilities = collaboraCapabilities;
@ -880,3 +951,6 @@ exports.fillStandardHeaders = fillStandardHeaders;
exports.getWopiUnlockMarker = getWopiUnlockMarker;
exports.getWopiModifiedMarker = getWopiModifiedMarker;
exports.getFileTypeByInfo = getFileTypeByInfo;
exports.dummyCheckFileInfo = dummyCheckFileInfo;
exports.dummyGetFile = dummyGetFile;
exports.dummyOk = dummyOk;

View File

@ -763,8 +763,11 @@ function* streamEnd(streamObj, text) {
streamObj.writeStream.end(text, 'utf8');
yield utils.promiseWaitClose(streamObj.writeStream);
}
function* processUploadToStorage(ctx, dir, storagePath, calcChecksum, opt_specialDirDst) {
function* processUploadToStorage(ctx, dir, storagePath, calcChecksum, opt_specialDirDst, opt_ignorPrefix) {
var list = yield utils.listObjects(dir);
if (opt_ignorPrefix) {
list = list.filter((dir) => !dir.startsWith(opt_ignorPrefix));
}
if (list.length < MAX_OPEN_FILES) {
yield* processUploadToStorageChunk(ctx, list, dir, storagePath, calcChecksum, opt_specialDirDst);
} else {
@ -804,13 +807,11 @@ function* processUploadToStorageErrorFile(ctx, dataConvert, tempDirs, childRes,
let outputPath = path.join(tempDirs.temp, 'console.txt');
fs.writeFileSync(outputPath, output, {encoding: 'utf8'});
//remove result dir with temp dir inside(see m_sTempDir param) to reduce the amount of data transferred
//todo filter on upload
fs.rmSync(tempDirs.result, { recursive: true, force: true });
//ignore result dir with temp dir inside(see m_sTempDir param) to reduce the amount of data transferred
let ignorePrefix = path.normalize(tempDirs.result);
let format = path.extname(dataConvert.fileFrom).substring(1) || "unknown";
yield* processUploadToStorage(ctx, tempDirs.temp, format + '/' + dataConvert.key , false, tenErrorFiles);
yield* processUploadToStorage(ctx, tempDirs.temp, format + '/' + dataConvert.key , false, tenErrorFiles, ignorePrefix);
ctx.logger.debug('processUploadToStorage error complete(id=%s)', dataConvert.key);
}
function writeProcessOutputToLog(ctx, childRes, isDebug) {