[config] Add externalRequest options to separate requests; for bug 63590

This commit is contained in:
Sergey Konovalov
2024-02-15 13:29:07 +03:00
parent 087bf48c9e
commit b8fbbaf180
6 changed files with 137 additions and 33 deletions

View File

@ -119,6 +119,23 @@
"useClones": false
}
},
"externalRequest": {
"directIfIn" : {
"allowList": [],
"jwtToken": true
},
"action": {
"allow": true,
"blockPrivateIP": true,
"proxyUrl": "",
"proxyAuth": {
"username": "",
"password": ""
},
"proxyHeaders": {
}
}
},
"services": {
"CoAuthoring": {
"server": {

View File

@ -48,8 +48,8 @@
"dbPass": "onlyoffice"
},
"request-filtering-agent" : {
"allowPrivateIPAddress": true,
"allowMetaIPAddress": true
"allowPrivateIPAddress": false,
"allowMetaIPAddress": false
},
"sockjs": {
"sockjs_url": "/web-apps/vendor/sockjs/sockjs.min.js"

View File

@ -84,7 +84,10 @@ const cfgPasswordEncrypt = config.get('openpgpjs.encrypt');
const cfgPasswordDecrypt = config.get('openpgpjs.decrypt');
const cfgPasswordConfig = config.get('openpgpjs.config');
const cfgRequesFilteringAgent = config.get('services.CoAuthoring.request-filtering-agent');
const cfgAllowPrivateIPAddressForSignedRequests = config.get('services.CoAuthoring.server.allowPrivateIPAddressForSignedRequests');
const cfgStorageExternalHost = config.get('storage.externalHost');
const cfgExternalRequestDirectIfIn = config.get('externalRequest.directIfIn');
const cfgExternalRequestAction = config.get('externalRequest.action');
const dnscache = getDnsCache(cfgDnsCache);
@ -266,6 +269,54 @@ function raiseErrorObj(ro, error) {
function isRedirectResponse(response) {
return response && response.statusCode >= 300 && response.statusCode < 400 && response.caseless.has('location');
}
function isAllowDirectRequest(ctx, uri, isInJwtToken) {
let res = false;
const tenExternalRequestDirectIfIn = ctx.getCfg('externalRequest.directIfIn', cfgExternalRequestDirectIfIn);
const tenAllowPrivateIPAddressForSignedRequests = ctx.getCfg('services.CoAuthoring.server.allowPrivateIPAddressForSignedRequests', cfgAllowPrivateIPAddressForSignedRequests);
let allowList = tenExternalRequestDirectIfIn.allowList;
if (allowList.length > 0) {
let allowIndex = allowList.findIndex((allowPrefix) => {
return uri.startsWith(allowPrefix);
}, uri);
res = -1 !== allowIndex;
ctx.logger.debug("isAllowDirectRequest check allow list res=%s", res);
} else if (tenExternalRequestDirectIfIn.jwtToken && tenAllowPrivateIPAddressForSignedRequests) {
res = isInJwtToken;
ctx.logger.debug("isAllowDirectRequest url in jwt token res=%s", res);
}
return res;
}
function addExternalRequestOptions(ctx, uri, isInJwtToken, options) {
let res = false;
const tenExternalRequestAction = ctx.getCfg('externalRequest.action', cfgExternalRequestAction);
const tenRequesFilteringAgent = ctx.getCfg('services.CoAuthoring.request-filtering-agent', cfgRequesFilteringAgent);
if (isAllowDirectRequest(ctx, uri, isInJwtToken)) {
res = true;
} else if (tenExternalRequestAction.allow) {
res = true;
if (tenExternalRequestAction.blockPrivateIP) {
const agentOptions = Object.assign({}, https.globalAgent.options, tenRequesFilteringAgent);
options.agent = getRequestFilterAgent(uri, agentOptions);
}
if (tenExternalRequestAction.proxyUrl) {
options.proxy = tenExternalRequestAction.proxyUrl;
}
if (tenExternalRequestAction.proxyUser?.username) {
let user = tenExternalRequestAction.proxyUser.username;
let pass = tenExternalRequestAction.proxyUser.password;
options.headers = {'proxy-authorization': `${user}:${pass}`};
}
if (tenExternalRequestAction.proxyHeaders) {
if (!options.headers) {
options.headers = {};
}
Object.assign(options.headers, tenExternalRequestAction.proxyHeaders);
}
}
return res;
}
function downloadUrlPromise(ctx, uri, optTimeout, optLimit, opt_Authorization, opt_filterPrivate, opt_headers, opt_streamWriter) {
//todo replace deprecated request module
const tenTenantRequestDefaults = ctx.getCfg('services.CoAuthoring.requestDefaults', cfgRequestDefaults);
@ -298,7 +349,6 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A
const tenTenantRequestDefaults = ctx.getCfg('services.CoAuthoring.requestDefaults', cfgRequestDefaults);
const tenTokenOutboxHeader = ctx.getCfg('services.CoAuthoring.token.outbox.header', cfgTokenOutboxHeader);
const tenTokenOutboxPrefix = ctx.getCfg('services.CoAuthoring.token.outbox.prefix', cfgTokenOutboxPrefix);
const tenRequesFilteringAgent = ctx.getCfg('services.CoAuthoring.request-filtering-agent', cfgRequesFilteringAgent);
//IRI to URI
uri = URI.serialize(URI.parse(uri));
var urlParsed = url.parse(uri);
@ -309,10 +359,12 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A
let connectionAndInactivity = optTimeout && optTimeout.connectionAndInactivity && ms(optTimeout.connectionAndInactivity);
let options = config.util.extendDeep({}, tenTenantRequestDefaults);
Object.assign(options, {uri: urlParsed, encoding: null, timeout: connectionAndInactivity, followRedirect: false});
if (opt_filterPrivate) {
const agentOptions = Object.assign({}, https.globalAgent.options, tenRequesFilteringAgent);
options.agent = getRequestFilterAgent(uri, agentOptions);
} else {
if (!addExternalRequestOptions(ctx, uri, opt_filterPrivate, options)) {
reject(new Error('Block external request. See externalRequest config options'));
return;
}
if (!options.agent) {
//baseRequest creates new agent(win-ca injects in globalAgent)
options.agentOptions = https.globalAgent.options;
}

View File

@ -70,7 +70,6 @@ const cfgAssemblyFormatAsOrigin = config.get('services.CoAuthoring.server.assemb
const cfgDownloadMaxBytes = config.get('FileConverter.converter.maxDownloadBytes');
const cfgDownloadTimeout = config.get('FileConverter.converter.downloadTimeout');
const cfgDownloadFileAllowExt = config.get('services.CoAuthoring.server.downloadFileAllowExt');
const cfgAllowPrivateIPAddressForSignedRequests = config.get('services.CoAuthoring.server.allowPrivateIPAddressForSignedRequests');
var SAVE_TYPE_PART_START = 0;
var SAVE_TYPE_PART = 1;
@ -658,11 +657,11 @@ function* commandImgurls(ctx, conn, cmd, outputData) {
const tenImageSize = ctx.getCfg('services.CoAuthoring.server.limits_image_size', cfgImageSize);
const tenImageDownloadTimeout = ctx.getCfg('services.CoAuthoring.server.limits_image_download_timeout', cfgImageDownloadTimeout);
const tenTokenEnableBrowser = ctx.getCfg('services.CoAuthoring.token.enable.browser', cfgTokenEnableBrowser);
const tenAllowPrivateIPAddressForSignedRequests = ctx.getCfg('services.CoAuthoring.server.allowPrivateIPAddressForSignedRequests', cfgAllowPrivateIPAddressForSignedRequests);
var errorCode = constants.NO_ERROR;
let urls = cmd.getData();
let authorizations = [];
let isInJwtToken = false;
let token = cmd.getTokenDownload();
if (tenTokenEnableBrowser && token) {
let checkJwtRes = yield docsCoServer.checkJwt(ctx, token, commonDefines.c_oAscSecretType.Browser);
@ -681,6 +680,7 @@ function* commandImgurls(ctx, conn, cmd, outputData) {
authorizations[i] = [utils.fillJwtForRequest(ctx, {url: urls[i]}, secret, false)];
}
}
isInJwtToken = true;
} else {
ctx.logger.warn('Error commandImgurls jwt: %s', checkJwtRes.description);
errorCode = constants.VKEY_ENCRYPT;
@ -723,8 +723,7 @@ function* commandImgurls(ctx, conn, cmd, outputData) {
}
}
//todo stream
const filterPrivate = !authorizations[i] || !tenAllowPrivateIPAddressForSignedRequests;
let getRes = yield utils.downloadUrlPromise(ctx, urlSource, tenImageDownloadTimeout, tenImageSize, authorizations[i], filterPrivate);
let getRes = yield utils.downloadUrlPromise(ctx, urlSource, tenImageDownloadTimeout, tenImageSize, authorizations[i], isInJwtToken);
data = getRes.body;
urlParsed = urlModule.parse(urlSource);
} catch (e) {
@ -1595,23 +1594,27 @@ exports.downloadFile = function(req, res) {
const tenDownloadMaxBytes = ctx.getCfg('FileConverter.converter.maxDownloadBytes', cfgDownloadMaxBytes);
const tenDownloadTimeout = ctx.getCfg('FileConverter.converter.downloadTimeout', cfgDownloadTimeout);
const tenDownloadFileAllowExt = ctx.getCfg('services.CoAuthoring.server.downloadFileAllowExt', cfgDownloadFileAllowExt);
const tenAllowPrivateIPAddressForSignedRequests = ctx.getCfg('services.CoAuthoring.server.allowPrivateIPAddressForSignedRequests', cfgAllowPrivateIPAddressForSignedRequests);
let authorization;
let isInJwtToken = false;
let errorDescription;
let authRes = yield docsCoServer.getRequestParams(ctx, req);
if (authRes.code === constants.NO_ERROR) {
let decoded = authRes.params;
if (decoded.changesUrl) {
url = decoded.changesUrl;
isInJwtToken = true;
} else if (decoded.document && -1 !== tenDownloadFileAllowExt.indexOf(decoded.document.fileType)) {
url = decoded.document.url;
isInJwtToken = true;
} else if (decoded.url && -1 !== tenDownloadFileAllowExt.indexOf(decoded.fileType)) {
url = decoded.url;
isInJwtToken = true;
} else if (!tenTokenEnableBrowser) {
//todo token required
if (decoded.url) {
url = decoded.url;
isInJwtToken = true;
}
} else {
errorDescription = 'access deny';
@ -1642,8 +1645,7 @@ exports.downloadFile = function(req, res) {
}
}
const filterPrivate = !authorization || !tenAllowPrivateIPAddressForSignedRequests;
yield utils.downloadUrlPromise(ctx, url, tenDownloadTimeout, tenDownloadMaxBytes, authorization, filterPrivate, headers, res);
yield utils.downloadUrlPromise(ctx, url, tenDownloadTimeout, tenDownloadMaxBytes, authorization, isInJwtToken, headers, res);
if (clientStatsD) {
clientStatsD.timing('coauth.downloadFile', new Date() - startDate);

View File

@ -61,7 +61,6 @@ const cfgTokenOutboxAlgorithm = config.get('services.CoAuthoring.token.outbox.al
const cfgTokenOutboxExpires = config.get('services.CoAuthoring.token.outbox.expires');
const cfgTokenEnableBrowser = config.get('services.CoAuthoring.token.enable.browser');
const cfgCallbackRequestTimeout = config.get('services.CoAuthoring.server.callbackRequestTimeout');
const cfgAllowPrivateIPAddressForSignedRequests = config.get('services.CoAuthoring.server.allowPrivateIPAddressForSignedRequests');
const cfgNewFileTemplate = config.get('services.CoAuthoring.server.newFileTemplate');
const cfgDownloadTimeout = config.get('FileConverter.converter.downloadTimeout');
const cfgWopiFileInfoBlockList = config.get('wopi.fileInfoBlockList');
@ -701,7 +700,6 @@ function checkFileInfo(ctx, wopiSrc, access_token, opt_sc) {
let fileInfo = undefined;
try {
ctx.logger.info('wopi checkFileInfo start');
const tenAllowPrivateIPAddressForSignedRequests = ctx.getCfg('services.CoAuthoring.server.allowPrivateIPAddressForSignedRequests', cfgAllowPrivateIPAddressForSignedRequests);
const tenDownloadTimeout = ctx.getCfg('FileConverter.converter.downloadTimeout', cfgDownloadTimeout);
let uri = `${encodeURI(wopiSrc)}?access_token=${encodeURIComponent(access_token)}`;
@ -715,8 +713,9 @@ function checkFileInfo(ctx, wopiSrc, access_token, opt_sc) {
}
fillStandardHeaders(ctx, headers, uri, access_token);
ctx.logger.debug('wopi checkFileInfo request uri=%s headers=%j', uri, headers);
const filterPrivate = !tenAllowPrivateIPAddressForSignedRequests;
let getRes = yield utils.downloadUrlPromise(ctx, uri, tenDownloadTimeout, undefined, undefined, filterPrivate, headers);
//todo false?
let isInJwtToken = true;
let getRes = yield utils.downloadUrlPromise(ctx, uri, tenDownloadTimeout, undefined, undefined, isInJwtToken, headers);
ctx.logger.debug(`wopi checkFileInfo headers=%j body=%s`, getRes.response.headers, getRes.body);
fileInfo = JSON.parse(getRes.body);
} catch (err) {

View File

@ -74,8 +74,9 @@ const cfgForgottenFiles = config.get('services.CoAuthoring.server.forgottenfiles
const cfgForgottenFilesName = config.get('services.CoAuthoring.server.forgottenfilesname');
const cfgNewFileTemplate = config.get('services.CoAuthoring.server.newFileTemplate');
const cfgEditor = config.get('services.CoAuthoring.editor');
const cfgAllowPrivateIPAddressForSignedRequests = config.get('services.CoAuthoring.server.allowPrivateIPAddressForSignedRequests');
const cfgRequesFilteringAgent = config.get('services.CoAuthoring.request-filtering-agent');
const cfgExternalRequestDirectIfIn = config.get('externalRequest.directIfIn');
const cfgExternalRequestAction = config.get('externalRequest.action');
//windows limit 512(2048) https://msdn.microsoft.com/en-us/library/6e3b887c.aspx
//Ubuntu 14.04 limit 4096 http://underyx.me/2015/05/18/raising-the-maximum-number-of-file-descriptors.html
@ -164,7 +165,7 @@ TaskQueueDataConvert.prototype = {
xml += this.serializeXmlProp('m_bIsNoBase64', this.noBase64);
xml += this.serializeXmlProp('m_sConvertToOrigin', this.convertToOrigin);
xml += this.serializeLimit(ctx);
xml += this.serializeOptions(ctx);
xml += this.serializeOptions(ctx, false);
xml += '</TaskQueueDataConvert>';
fs.writeFileSync(fsPath, xml, {encoding: 'utf8'});
},
@ -187,12 +188,45 @@ TaskQueueDataConvert.prototype = {
return xml;
});
},
serializeOptions: function (ctx) {
serializeOptions: function (ctx, isInJwtToken) {
const tenRequesFilteringAgent = ctx.getCfg('services.CoAuthoring.request-filtering-agent', cfgRequesFilteringAgent);
const tenExternalRequestDirectIfIn = ctx.getCfg('externalRequest.directIfIn', cfgExternalRequestDirectIfIn);
const tenExternalRequestAction = ctx.getCfg('externalRequest.action', cfgExternalRequestAction);
let allowList = tenExternalRequestDirectIfIn.allowList;
let allowNetworkRequest = tenExternalRequestAction.allow;
let allowPrivateIP = !tenExternalRequestAction.blockPrivateIP && tenRequesFilteringAgent.allowPrivateIPAddress;
let proxyUrl = tenExternalRequestAction.proxyUrl;
let proxyUser = tenExternalRequestAction.proxyUser;
let proxyHeaders = tenExternalRequestAction.proxyHeaders;
if (allowList.length === 0 && tenExternalRequestDirectIfIn.jwtToken && isInJwtToken) {
allowNetworkRequest = true;
allowPrivateIP = true;
proxyUrl = "";
proxyUser = null;
proxyHeaders = {};
}
let xml = "";
xml += '<options>';
xml += this.serializeXmlProp('allowNetworkRequest', true);
xml += this.serializeXmlProp('allowPrivateIP', tenRequesFilteringAgent.allowPrivateIPAddress);
if (allowList.length > 0) {
xml += this.serializeXmlProp('allowList', allowList.join(';'));
}
xml += this.serializeXmlProp('allowNetworkRequest', allowNetworkRequest);
xml += this.serializeXmlProp('allowPrivateIP', allowPrivateIP);
if (proxyUrl) {
xml += this.serializeXmlProp('proxy', proxyUrl);
}
if (proxyUser) {
let user = proxyUser.username;
let pass = proxyUser.password;
xml += this.serializeXmlProp('proxyUser', `${user}:${pass}`);
}
let proxyHeadersStr= [];
for (let name in proxyHeaders) {
proxyHeadersStr.push(`${name}:${proxyHeaders[name]}`);
}
if (proxyHeadersStr.length > 0) {
xml += this.serializeXmlProp('proxyHeader', proxyHeadersStr.join(';'));
}
xml += '</options>';
return xml;
},
@ -257,6 +291,7 @@ TaskQueueDataConvert.prototype = {
},
serializeXmlProp: function(name, value) {
var xml = '';
//todo check empty and undefined (password?)
if (null != value) {
xml += '<' + name + '>';
xml += utils.encodeXml(value.toString());
@ -346,7 +381,7 @@ function* replaceEmptyFile(ctx, fileFrom, ext, _lcid) {
}
}
}
function* downloadFile(ctx, uri, fileFrom, withAuthorization, filterPrivate, opt_headers) {
function* downloadFile(ctx, uri, fileFrom, withAuthorization, isInJwtToken, opt_headers) {
const tenMaxDownloadBytes = ctx.getCfg('FileConverter.converter.maxDownloadBytes', cfgMaxDownloadBytes);
const tenDownloadTimeout = ctx.getCfg('FileConverter.converter.downloadTimeout', cfgDownloadTimeout);
const tenDownloadAttemptMaxCount = ctx.getCfg('FileConverter.converter.downloadAttemptMaxCount', cfgDownloadAttemptMaxCount);
@ -365,7 +400,7 @@ function* downloadFile(ctx, uri, fileFrom, withAuthorization, filterPrivate, opt
let secret = yield tenantManager.getTenantSecret(ctx, commonDefines.c_oAscSecretType.Outbox);
authorization = utils.fillJwtForRequest(ctx, {url: uri}, secret, false);
}
let getRes = yield utils.downloadUrlPromise(ctx, uri, tenDownloadTimeout, tenMaxDownloadBytes, authorization, filterPrivate, opt_headers);
let getRes = yield utils.downloadUrlPromise(ctx, uri, tenDownloadTimeout, tenMaxDownloadBytes, authorization, isInJwtToken, opt_headers);
data = getRes.body;
sha256 = getRes.sha256;
res = constants.NO_ERROR;
@ -897,7 +932,7 @@ function* postProcess(ctx, cmd, dataConvert, tempDirs, childRes, error, isTimeou
return queueData;
}
function* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, getTaskTime, task) {
function* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, getTaskTime, task, isInJwtToken) {
const tenX2tPath = ctx.getCfg('FileConverter.converter.x2tPath', cfgX2tPath);
const tenDocbuilderPath = ctx.getCfg('FileConverter.converter.docbuilderPath', cfgDocbuilderPath);
const tenArgs = ctx.getCfg('FileConverter.converter.args', cfgArgs);
@ -926,7 +961,7 @@ function* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, g
if (builderParams.argument) {
childArgs.push(`--argument=${JSON.stringify(builderParams.argument)}`);
}
childArgs.push('--options=' + dataConvert.serializeOptions(ctx));
childArgs.push('--options=' + dataConvert.serializeOptions(ctx, isInJwtToken));
childArgs.push(dataConvert.fileFrom);
}
let timeoutId;
@ -970,7 +1005,6 @@ function* ExecuteTask(ctx, task) {
const tenMaxDownloadBytes = ctx.getCfg('FileConverter.converter.maxDownloadBytes', cfgMaxDownloadBytes);
const tenForgottenFiles = ctx.getCfg('services.CoAuthoring.server.forgottenfiles', cfgForgottenFiles);
const tenForgottenFilesName = ctx.getCfg('services.CoAuthoring.server.forgottenfilesname', cfgForgottenFilesName);
const tenAllowPrivateIPAddressForSignedRequests = ctx.getCfg('services.CoAuthoring.server.allowPrivateIPAddressForSignedRequests', cfgAllowPrivateIPAddressForSignedRequests);
var startDate = null;
var curDate = null;
if(clientStatsD) {
@ -988,6 +1022,7 @@ function* ExecuteTask(ctx, task) {
dataConvert.fileTo = fileTo ? path.join(tempDirs.result, fileTo) : '';
let builderParams = cmd.getBuilderParams();
let authorProps = {lastModifiedBy: null, modified: null};
let isInJwtToken = cmd.getWithAuthorization();
error = yield* isUselessConvertion(ctx, task, cmd);
if (constants.NO_ERROR !== error) {
;
@ -997,13 +1032,12 @@ function* ExecuteTask(ctx, task) {
if (utils.checkPathTraversal(ctx, dataConvert.key, tempDirs.source, dataConvert.fileFrom)) {
let url = cmd.getUrl();
let withAuthorization = cmd.getWithAuthorization();
let filterPrivate = !withAuthorization || !tenAllowPrivateIPAddressForSignedRequests;
let headers;
let fileSize;
let wopiParams = cmd.getWopiParams();
if (wopiParams) {
withAuthorization = false;
filterPrivate = !tenAllowPrivateIPAddressForSignedRequests;
isInJwtToken = true;
let fileInfo = wopiParams.commonInfo?.fileInfo;
let userAuth = wopiParams.userAuth;
fileSize = fileInfo?.Size;
@ -1020,7 +1054,7 @@ function* ExecuteTask(ctx, task) {
ctx.logger.debug('wopi url=%s; headers=%j', url, headers);
}
if (undefined === fileSize || fileSize > 0) {
error = yield* downloadFile(ctx, url, dataConvert.fileFrom, withAuthorization, filterPrivate, headers);
error = yield* downloadFile(ctx, url, dataConvert.fileFrom, withAuthorization, isInJwtToken, headers);
}
if (constants.NO_ERROR === error) {
yield* replaceEmptyFile(ctx, dataConvert.fileFrom, format, cmd.getLCID());
@ -1066,7 +1100,7 @@ function* ExecuteTask(ctx, task) {
let childRes = null;
let isTimeout = false;
if (constants.NO_ERROR === error) {
({childRes, isTimeout} = yield* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, getTaskTime, task));
({childRes, isTimeout} = yield* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, getTaskTime, task, isInJwtToken));
if (childRes && 0 !== childRes.status && !isTimeout && task.getFromChanges()
&& constants.AVS_OFFICESTUDIO_FILE_OTHER_OOXML !== dataConvert.formatTo
&& !formatChecker.isOOXFormat(dataConvert.formatTo) && !cmd.getWopiParams()) {
@ -1075,7 +1109,7 @@ function* ExecuteTask(ctx, task) {
let extNew = '.' + formatChecker.getStringFromFormat(constants.AVS_OFFICESTUDIO_FILE_OTHER_OOXML);
dataConvert.formatTo = constants.AVS_OFFICESTUDIO_FILE_OTHER_OOXML;
dataConvert.fileTo = dataConvert.fileTo.slice(0, -extOld.length) + extNew;
({childRes, isTimeout} = yield* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, getTaskTime, task));
({childRes, isTimeout} = yield* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, getTaskTime, task, isInJwtToken));
}
if(clientStatsD) {
clientStatsD.timing('conv.spawnSync', new Date() - curDate);