From 22fbfd20e393dfacbb1c93b9ae5f3c3c14b40d2a Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 4 Dec 2023 21:24:49 +0300 Subject: [PATCH] [feature] Use built-in nodejs fs functions instead of fs-extra and mkdirp --- Common/npm-shrinkwrap.json | 38 --- Common/package.json | 2 - Common/sources/storage-base.js | 11 +- Common/sources/storage-fs.js | 235 +++++++----------- Common/sources/storage-s3.js | 142 ++++++----- DocService/sources/DocsCoServer.js | 2 +- DocService/sources/canvasservice.js | 12 +- .../forgottenFilesCommnads.tests.js | 8 +- tests/integration/storage.tests.js | 81 ++++-- tests/perf/checkFileExpire.js | 2 +- 10 files changed, 245 insertions(+), 288 deletions(-) diff --git a/Common/npm-shrinkwrap.json b/Common/npm-shrinkwrap.json index 5173fdfc..94110ddb 100644 --- a/Common/npm-shrinkwrap.json +++ b/Common/npm-shrinkwrap.json @@ -1472,16 +1472,6 @@ "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" }, - "fs-extra": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.0.tgz", - "integrity": "sha512-EglNDLRpmaTWiD/qraZn6HREAEAHJcJOmxNEYwq6xeMKnVMAy3GUcFB+wXt2C6k4CNvB/mP1y/U3dzvKKj5OtQ==", - "requires": { - "graceful-fs": "^4.1.2", - "jsonfile": "^4.0.0", - "universalify": "^0.1.0" - } - }, "getpass": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", @@ -1577,14 +1567,6 @@ "minimist": "^1.2.0" } }, - "jsonfile": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", - "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", - "requires": { - "graceful-fs": "^4.1.6" - } - }, "jsonwebtoken": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz", @@ -1722,21 +1704,6 @@ "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, - "mkdirp": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha512-SknJC52obPfGQPnjIkXbmA6+5H15E+fR+E4iR2oQ3zzCLbd7/ONua69R/Gw7AgkTLsRG+r5fzksYwWe1AgTyWA==", - "requires": { - "minimist": "0.0.8" - }, - "dependencies": { - "minimist": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha512-miQKw5Hv4NS1Psg2517mV4e4dYNaO3++hjAvLOAzKqZ61rH8NS1SK+vbfBWZ5PY/Me/bEWhUwqMghEW5Fb9T7Q==" - } - } - }, "ms": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", @@ -2050,11 +2017,6 @@ "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" }, - "universalify": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", - "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==" - }, "uri-js": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", diff --git a/Common/package.json b/Common/package.json index 4d9971ba..8fdcf927 100644 --- a/Common/package.json +++ b/Common/package.json @@ -13,12 +13,10 @@ "dnscache": "1.0.1", "escape-string-regexp": "1.0.5", "forwarded": "0.1.2", - "fs-extra": "7.0.0", "ipaddr.js": "1.8.1", "jsonwebtoken": "9.0.0", "log4js": "6.4.1", "mime": "2.3.1", - "mkdirp": "0.5.1", "ms": "2.1.1", "node-cache": "4.2.1", "node-statsd": "0.1.1", diff --git a/Common/sources/storage-base.js b/Common/sources/storage-base.js index 1a85bf31..48ff2ad3 100644 --- a/Common/sources/storage-base.js +++ b/Common/sources/storage-base.js @@ -87,17 +87,8 @@ exports.listObjects = function(ctx, strPath, opt_specialDir) { exports.deleteObject = function(ctx, strPath, opt_specialDir) { return storage.deleteObject(getStoragePath(ctx, strPath, opt_specialDir)); }; -exports.deleteObjects = function(ctx, strPaths, opt_specialDir) { - var StoragePaths = strPaths.map(function(curValue) { - return getStoragePath(ctx, curValue, opt_specialDir); - }); - return storage.deleteObjects(StoragePaths); -}; exports.deletePath = function(ctx, strPath, opt_specialDir) { - let storageSrc = getStoragePath(ctx, strPath, opt_specialDir); - return storage.listObjects(storageSrc).then(function(list) { - return storage.deleteObjects(list); - }); + return storage.deletePath(getStoragePath(ctx, strPath, opt_specialDir)); }; exports.getSignedUrl = function(ctx, baseUrl, strPath, urlType, optFilename, opt_creationDate, opt_specialDir) { return storage.getSignedUrl(ctx, baseUrl, getStoragePath(ctx, strPath, opt_specialDir), urlType, optFilename, opt_creationDate); diff --git a/Common/sources/storage-fs.js b/Common/sources/storage-fs.js index efbfd3ed..d4aad1fc 100644 --- a/Common/sources/storage-fs.js +++ b/Common/sources/storage-fs.js @@ -32,10 +32,9 @@ 'use strict'; -var fs = require('fs'); -const fse = require('fs-extra') +const { cp, rm, mkdir } = require('fs/promises'); +const { stat, readFile, writeFile } = require('fs/promises'); var path = require('path'); -var mkdirp = require('mkdirp'); var utils = require("./utils"); var crypto = require('crypto'); const ms = require('ms'); @@ -57,152 +56,104 @@ function getFilePath(strPath) { function getOutputPath(strPath) { return strPath.replace(/\\/g, '/'); } -function removeEmptyParent(strPath, done) { - if (cfgStorageFolderPath.length + 1 >= strPath.length) { - done(); + +async function headObject(strPath) { + let fsPath = getFilePath(strPath); + let stats = await stat(fsPath); + return {ContentLength: stats.size}; +} + +async function getObject(strPath) { + let fsPath = getFilePath(strPath); + return await readFile(fsPath); +} + +async function createReadStream(strPath) { + let fsPath = getFilePath(strPath); + let stats = await stat(fsPath); + let contentLength = stats.size; + let readStream = await utils.promiseCreateReadStream(fsPath); + return { + contentLength: contentLength, + readStream: readStream + }; +} + +async function putObject(strPath, buffer, contentLength) { + var fsPath = getFilePath(strPath); + await mkdir(path.dirname(fsPath), {recursive: true}); + + if (Buffer.isBuffer(buffer)) { + await writeFile(fsPath, buffer); } else { - fs.readdir(strPath, function(err, list) { - if (err) { - //we do not react to the error, because most likely this folder was deleted in a neighboring thread - done(); - } else { - if (list.length > 0) { - done(); - } else { - fs.rmdir(strPath, function(err) { - if (err) { - //we do not react to the error, because most likely this folder was deleted in a neighboring thread - done(); - } else { - removeEmptyParent(path.dirname(strPath), function(err) { - done(err); - }); - } - }); - } - } - }); + let writable = await utils.promiseCreateWriteStream(fsPath); + await utils.pipeStreams(buffer, writable, true); } } -exports.headObject = function(strPath) { - return utils.fsStat(getFilePath(strPath)).then(function(stats) { - return {ContentLength: stats.size}; - }); -}; -exports.getObject = function(strPath) { - return utils.readFile(getFilePath(strPath)); -}; -exports.createReadStream = function(strPath) { +async function uploadObject(strPath, filePath) { let fsPath = getFilePath(strPath); - let contentLength; - return new Promise(function(resolve, reject) { - fs.stat(fsPath, function(err, stats) { - if (err) { - reject(err); - } else { - resolve(stats); - } - }); - }).then(function(stats){ - contentLength = stats.size; - return utils.promiseCreateReadStream(fsPath); - }).then(function(readStream, stats){ - return { - contentLength: contentLength, - readStream: readStream - }; - }); -}; + await cp(filePath, fsPath, {force: true, recursive: true}); +} -exports.putObject = function(strPath, buffer, contentLength) { - return new Promise(function(resolve, reject) { - var fsPath = getFilePath(strPath); - mkdirp(path.dirname(fsPath), function(err) { - if (err) { - reject(err); - } else { - //todo 0666 - if (Buffer.isBuffer(buffer)) { - fs.writeFile(fsPath, buffer, function(err) { - if (err) { - reject(err); - } else { - resolve(); - } - }); - } else { - utils.promiseCreateWriteStream(fsPath).then(function(writable) { - buffer.pipe(writable); - }).catch(function(err) { - reject(err); - }); - } - } - }); - }); -}; -exports.uploadObject = function(strPath, filePath) { - let fsPath = getFilePath(strPath); - return fse.copy(filePath, fsPath); -}; -exports.copyObject = function(sourceKey, destinationKey) { +async function copyObject(sourceKey, destinationKey) { let fsPathSource = getFilePath(sourceKey); - let fsPathSestination = getFilePath(destinationKey); - return fse.copy(fsPathSource, fsPathSestination); -}; -exports.listObjects = function(strPath) { - return utils.listObjects(getFilePath(strPath)).then(function(values) { - return values.map(function(curvalue) { - return getOutputPath(curvalue.substring(cfgStorageFolderPath.length + 1)); - }); - }); -}; -exports.deleteObject = function(strPath) { - return new Promise(function(resolve, reject) { - const fsPath = getFilePath(strPath); - fs.unlink(fsPath, function(err) { - if (err) { - reject(err); - } else { - //resolve(); - removeEmptyParent(path.dirname(fsPath), function(err) { - if (err) { - reject(err); - } else { - resolve(); - } - }); - } - }); - }); -}; -exports.deleteObjects = function(strPaths) { - return Promise.all(strPaths.map(exports.deleteObject)); -}; -exports.getSignedUrl = function(ctx, baseUrl, strPath, urlType, optFilename, opt_creationDate) { - return new Promise(function(resolve, reject) { - //replace '/' with %2f before encodeURIComponent becase nginx determine %2f as '/' and get wrong system path - var userFriendlyName = optFilename ? encodeURIComponent(optFilename.replace(/\//g, "%2f")) : path.basename(strPath); - var uri = '/' + cfgBucketName + '/' + cfgStorageFolderName + '/' + strPath + '/' + userFriendlyName; - //RFC 1123 does not allow underscores https://stackoverflow.com/questions/2180465/can-domain-name-subdomains-have-an-underscore-in-it - var url = utils.checkBaseUrl(ctx, baseUrl).replace(/_/g, "%5f"); - url += uri; + let fsPathDestination = getFilePath(destinationKey); + await cp(fsPathSource, fsPathDestination, {force: true, recursive: true}); +} - var date = Date.now(); - let creationDate = opt_creationDate || date; - let expiredAfter = (commonDefines.c_oAscUrlTypes.Session === urlType ? (cfgExpSessionAbsolute / 1000) : cfgStorageUrlExpires) || 31536000; - //todo creationDate can be greater because mysql CURRENT_TIMESTAMP uses local time, not UTC - var expires = creationDate + Math.ceil(Math.abs(date - creationDate)/expiredAfter) * expiredAfter; - expires = Math.ceil(expires / 1000); - expires += expiredAfter; - - var md5 = crypto.createHash('md5').update(expires + decodeURIComponent(uri) + cfgStorageSecretString).digest("base64"); - md5 = md5.replace(/\+/g, "-").replace(/\//g, "_").replace(/=/g, ""); - - url += '?md5=' + encodeURIComponent(md5); - url += '&expires=' + encodeURIComponent(expires); - url += '&filename=' + userFriendlyName; - resolve(url); +async function listObjects(strPath) { + let fsPath = getFilePath(strPath); + let values = await utils.listObjects(fsPath); + return values.map(function(curvalue) { + return getOutputPath(curvalue.substring(cfgStorageFolderPath.length + 1)); }); +} + +async function deleteObject(strPath) { + const fsPath = getFilePath(strPath); + return rm(fsPath, {force: true, recursive: true}); +} + +async function deletePath(strPath) { + const fsPath = getFilePath(strPath); + return rm(fsPath, {force: true, recursive: true}); +} + +async function getSignedUrl(ctx, baseUrl, strPath, urlType, optFilename, opt_creationDate) { + //replace '/' with %2f before encodeURIComponent becase nginx determine %2f as '/' and get wrong system path + var userFriendlyName = optFilename ? encodeURIComponent(optFilename.replace(/\//g, "%2f")) : path.basename(strPath); + var uri = '/' + cfgBucketName + '/' + cfgStorageFolderName + '/' + strPath + '/' + userFriendlyName; + //RFC 1123 does not allow underscores https://stackoverflow.com/questions/2180465/can-domain-name-subdomains-have-an-underscore-in-it + var url = utils.checkBaseUrl(ctx, baseUrl).replace(/_/g, "%5f"); + url += uri; + + var date = Date.now(); + let creationDate = opt_creationDate || date; + let expiredAfter = (commonDefines.c_oAscUrlTypes.Session === urlType ? (cfgExpSessionAbsolute / 1000) : cfgStorageUrlExpires) || 31536000; + //todo creationDate can be greater because mysql CURRENT_TIMESTAMP uses local time, not UTC + var expires = creationDate + Math.ceil(Math.abs(date - creationDate) / expiredAfter) * expiredAfter; + expires = Math.ceil(expires / 1000); + expires += expiredAfter; + + var md5 = crypto.createHash('md5').update(expires + decodeURIComponent(uri) + cfgStorageSecretString).digest("base64"); + md5 = md5.replace(/\+/g, "-").replace(/\//g, "_").replace(/=/g, ""); + + url += '?md5=' + encodeURIComponent(md5); + url += '&expires=' + encodeURIComponent(expires); + url += '&filename=' + userFriendlyName; + return url; +} + +module.exports = { + headObject, + getObject, + createReadStream, + putObject, + uploadObject, + copyObject, + listObjects, + deleteObject, + deletePath, + getSignedUrl }; diff --git a/Common/sources/storage-s3.js b/Common/sources/storage-s3.js index e288a802..71d15510 100644 --- a/Common/sources/storage-s3.js +++ b/Common/sources/storage-s3.js @@ -31,30 +31,30 @@ */ 'use strict'; -var fs = require('fs'); -var url = require('url'); -var path = require('path'); +const fs = require('fs'); +const url = require('url'); +const path = require('path'); const { S3Client, ListObjectsCommand, HeadObjectCommand} = require("@aws-sdk/client-s3"); const { GetObjectCommand, PutObjectCommand, CopyObjectCommand} = require("@aws-sdk/client-s3"); const { DeleteObjectsCommand, DeleteObjectCommand } = require("@aws-sdk/client-s3"); const { getSignedUrl } = require("@aws-sdk/s3-request-presigner"); -var mime = require('mime'); -var utils = require('./utils'); +const mime = require('mime'); +const utils = require('./utils'); const ms = require('ms'); const commonDefines = require('./../../Common/sources/commondefines'); -var config = require('config'); -var configStorage = require('config').get('storage'); -var cfgRegion = configStorage.get('region'); -var cfgEndpoint = configStorage.get('endpoint'); -var cfgBucketName = configStorage.get('bucketName'); -var cfgStorageFolderName = configStorage.get('storageFolderName'); -var cfgAccessKeyId = configStorage.get('accessKeyId'); -var cfgSecretAccessKey = configStorage.get('secretAccessKey'); -var cfgSslEnabled = configStorage.get('sslEnabled'); -var cfgS3ForcePathStyle = configStorage.get('s3ForcePathStyle'); -var configFs = configStorage.get('fs'); -var cfgStorageUrlExpires = configFs.get('urlExpires'); +const config = require('config'); +const configStorage = require('config').get('storage'); +const cfgRegion = configStorage.get('region'); +const cfgEndpoint = configStorage.get('endpoint'); +const cfgBucketName = configStorage.get('bucketName'); +const cfgStorageFolderName = configStorage.get('storageFolderName'); +const cfgAccessKeyId = configStorage.get('accessKeyId'); +const cfgSecretAccessKey = configStorage.get('secretAccessKey'); +const cfgSslEnabled = configStorage.get('sslEnabled'); +const cfgS3ForcePathStyle = configStorage.get('s3ForcePathStyle'); +const configFs = configStorage.get('fs'); +const cfgStorageUrlExpires = configFs.get('urlExpires'); const cfgExpSessionAbsolute = ms(config.get('services.CoAuthoring.expire.sessionabsolute')); /** @@ -64,7 +64,7 @@ const cfgExpSessionAbsolute = ms(config.get('services.CoAuthoring.expire.session * export AWS_ACCESS_KEY_ID='AKID' * export AWS_SECRET_ACCESS_KEY='SECRET' */ -var configS3 = { +let configS3 = { region: cfgRegion, endpoint: cfgEndpoint, credentials : { @@ -80,7 +80,7 @@ if (configS3.endpoint) { const client = new S3Client(configS3); //This operation enables you to delete multiple objects from a bucket using a single HTTP request. You may specify up to 1000 keys. -var MAX_DELETE_OBJECTS = 1000; +const MAX_DELETE_OBJECTS = 1000; function getFilePath(strPath) { //todo @@ -90,20 +90,20 @@ function joinListObjects(inputArray, outputArray) { if (!inputArray) { return; } - var length = inputArray.length; - for (var i = 0; i < length; i++) { + let length = inputArray.length; + for (let i = 0; i < length; i++) { outputArray.push(inputArray[i].Key.substring((cfgStorageFolderName + '/').length)); } } async function listObjectsExec(output, params) { const data = await client.send(new ListObjectsCommand(params)); - joinListObjects(data.Contents, output); + joinListObjects(data.Contents, output); if (data.IsTruncated && (data.NextMarker || (data.Contents && data.Contents.length > 0))) { - params.Marker = data.NextMarker || data.Contents[data.Contents.length - 1].Key; + params.Marker = data.NextMarker || data.Contents[data.Contents.length - 1].Key; return await listObjectsExec(output, params); - } else { + } else { return output; - } + } } async function deleteObjectsHelp(aKeys) { //By default, the operation uses verbose mode in which the response includes the result of deletion of each key in your request. @@ -116,18 +116,19 @@ async function deleteObjectsHelp(aKeys) { } }; const command = new DeleteObjectsCommand(input); - return await client.send(command); + await client.send(command); } -exports.headObject = async function(strPath) { +async function headObject(strPath) { const input = { Bucket: cfgBucketName, Key: getFilePath(strPath) }; const command = new HeadObjectCommand(input); - return await client.send(command); -}; -exports.getObject = async function(strPath) { + let output = await client.send(command); + return {ContentLength: output.ContentLength}; +} +async function getObject(strPath) { const input = { Bucket: cfgBucketName, Key: getFilePath(strPath) @@ -136,8 +137,8 @@ exports.getObject = async function(strPath) { const output = await client.send(command); return await utils.stream2Buffer(output.Body); -}; -exports.createReadStream = async function(strPath) { +} +async function createReadStream(strPath) { const input = { Bucket: cfgBucketName, Key: getFilePath(strPath) @@ -148,8 +149,8 @@ exports.createReadStream = async function(strPath) { contentLength: output.ContentLength, readStream: output.Body }; -}; -exports.putObject = async function(strPath, buffer, contentLength) { +} +async function putObject(strPath, buffer, contentLength) { //todo consider Expires const input = { Bucket: cfgBucketName, @@ -159,9 +160,9 @@ exports.putObject = async function(strPath, buffer, contentLength) { ContentType: mime.getType(strPath) }; const command = new PutObjectCommand(input); - return await client.send(command); -}; -exports.uploadObject = async function(strPath, filePath) { + await client.send(command); +} +async function uploadObject(strPath, filePath) { const file = fs.createReadStream(filePath); //todo рассмотреть Expires const input = { @@ -171,9 +172,9 @@ exports.uploadObject = async function(strPath, filePath) { ContentType: mime.getType(strPath) }; const command = new PutObjectCommand(input); - return await client.send(command); -}; -exports.copyObject = function(sourceKey, destinationKey) { + await client.send(command); +} +async function copyObject(sourceKey, destinationKey) { //todo source bucket const input = { Bucket: cfgBucketName, @@ -181,37 +182,43 @@ exports.copyObject = function(sourceKey, destinationKey) { CopySource: `/${cfgBucketName}/${getFilePath(sourceKey)}` }; const command = new CopyObjectCommand(input); - return client.send(command); -}; -exports.listObjects = async function(strPath) { - var params = {Bucket: cfgBucketName, Prefix: getFilePath(strPath)}; - var output = []; - return await listObjectsExec(output, params); -}; -exports.deleteObject = function(strPath) { + await client.send(command); +} +async function listObjects(strPath) { + let params = { + Bucket: cfgBucketName, + Prefix: getFilePath(strPath) + }; + let output = []; + await listObjectsExec(output, params); + return output; +} +async function deleteObject(strPath) { const input = { Bucket: cfgBucketName, Key: getFilePath(strPath) }; const command = new DeleteObjectCommand(input); - return client.send(command); + await client.send(command); }; -exports.deleteObjects = function(strPaths) { - var aKeys = strPaths.map(function (currentValue) { +async function deleteObjects(strPaths) { + let aKeys = strPaths.map(function (currentValue) { return {Key: getFilePath(currentValue)}; }); - var deletePromises = []; - for (var i = 0; i < aKeys.length; i += MAX_DELETE_OBJECTS) { - deletePromises.push(deleteObjectsHelp(aKeys.slice(i, i + MAX_DELETE_OBJECTS))); + for (let i = 0; i < aKeys.length; i += MAX_DELETE_OBJECTS) { + await deleteObjectsHelp(aKeys.slice(i, i + MAX_DELETE_OBJECTS)); } - return Promise.all(deletePromises); -}; -exports.getSignedUrl = async function (ctx, baseUrl, strPath, urlType, optFilename, opt_creationDate) { - var expires = (commonDefines.c_oAscUrlTypes.Session === urlType ? cfgExpSessionAbsolute / 1000 : cfgStorageUrlExpires) || 31536000; +} +async function deletePath(strPath) { + let list = await listObjects(strPath); + await deleteObjects(list); +} +async function getSignedUrlWrapper(ctx, baseUrl, strPath, urlType, optFilename, opt_creationDate) { + let expires = (commonDefines.c_oAscUrlTypes.Session === urlType ? cfgExpSessionAbsolute / 1000 : cfgStorageUrlExpires) || 31536000; // Signature version 4 presigned URLs must have an expiration date less than one week in the future expires = Math.min(expires, 604800); - var userFriendlyName = optFilename ? optFilename.replace(/\//g, "%2f") : path.basename(strPath); - var contentDisposition = utils.getContentDisposition(userFriendlyName, null, null); + let userFriendlyName = optFilename ? optFilename.replace(/\//g, "%2f") : path.basename(strPath); + let contentDisposition = utils.getContentDisposition(userFriendlyName, null, null); const input = { Bucket: cfgBucketName, @@ -220,11 +227,24 @@ exports.getSignedUrl = async function (ctx, baseUrl, strPath, urlType, optFilena }; const command = new GetObjectCommand(input); //default Expires 900 seconds - var options = { + let options = { expiresIn: expires }; return await getSignedUrl(client, command, options); //extra query params cause SignatureDoesNotMatch //https://stackoverflow.com/questions/55503009/amazon-s3-signature-does-not-match-when-extra-query-params-ga-added-in-url // return utils.changeOnlyOfficeUrl(url, strPath, optFilename); +} + +module.exports = { + headObject, + getObject, + createReadStream, + putObject, + uploadObject, + copyObject, + listObjects, + deleteObject, + deletePath, + getSignedUrl: getSignedUrlWrapper }; diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index b1add8fd..5ae1641b 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -4306,7 +4306,7 @@ function* commandHandle(ctx, params, req, output) { break; } - yield storage.deleteObject(ctx, forgottenFile, tenForgottenFiles); + yield storage.deletePath(ctx, docId, tenForgottenFiles); break; } case 'getForgottenList': { diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index c3f8446b..377b1e89 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -416,28 +416,24 @@ function* getUpdateResponse(ctx, cmd) { var cleanupCache = co.wrap(function* (ctx, docId) { //todo redis ? var res = false; - let list = []; var removeRes = yield taskResult.remove(ctx, docId); if (removeRes.affectedRows > 0) { - list = yield storage.listObjects(ctx, docId); - yield storage.deleteObjects(ctx, list); + yield storage.deletePath(ctx, docId); res = true; } - ctx.logger.debug("cleanupCache docId=%s db.affectedRows=%d list.length=%d", docId, removeRes.affectedRows, list.length); + ctx.logger.debug("cleanupCache docId=%s db.affectedRows=%d", docId, removeRes.affectedRows); return res; }); var cleanupCacheIf = co.wrap(function* (ctx, mask) { //todo redis ? var res = false; - let list = []; var removeRes = yield taskResult.removeIf(ctx, mask); if (removeRes.affectedRows > 0) { sqlBase.deleteChanges(ctx, mask.key, null); - list = yield storage.listObjects(ctx, mask.key); - yield storage.deleteObjects(ctx, list); + yield storage.deletePath(ctx, mask.key); res = true; } - ctx.logger.debug("cleanupCacheIf db.affectedRows=%d list.length=%d", removeRes.affectedRows, list.length); + ctx.logger.debug("cleanupCacheIf db.affectedRows=%d", removeRes.affectedRows); return res; }); diff --git a/tests/integration/forgottenFilesCommnads.tests.js b/tests/integration/forgottenFilesCommnads.tests.js index 42bd9764..e9ae4073 100644 --- a/tests/integration/forgottenFilesCommnads.tests.js +++ b/tests/integration/forgottenFilesCommnads.tests.js @@ -85,9 +85,11 @@ beforeAll(async function () { afterAll(async function () { const keys = await storage.listObjects(ctx, '', cfgForgottenFiles); - const deletePromises = keys.filter(key => key.includes('DocService-DocsCoServer-forgottenFilesCommands')) - .map(filteredKey => storage.deleteObject(ctx, filteredKey, cfgForgottenFiles)); - + const keysDirectories = getKeysDirectories(keys); + const deletePromises = keysDirectories.filter(key => key.includes('DocService-DocsCoServer-forgottenFilesCommands')) + .map(filteredKey => storage.deletePath(ctx, filteredKey, cfgForgottenFiles)); + console.log(`keys:`+JSON.stringify(keys)); + console.log(`keysDirectories:`+JSON.stringify(keysDirectories)); return Promise.allSettled(deletePromises); }); diff --git a/tests/integration/storage.tests.js b/tests/integration/storage.tests.js index 836bede0..6ec1048c 100644 --- a/tests/integration/storage.tests.js +++ b/tests/integration/storage.tests.js @@ -2,6 +2,18 @@ const {jest, describe, test, expect} = require('@jest/globals'); const http = require('http'); const https = require('https'); const fs = require('fs'); +const { Readable } = require('stream'); + +let testFileData1 = "test1"; +let testFileData2 = "test22"; +let testFileData3 = "test333"; +let testFileData4 = testFileData3; + +jest.mock("fs/promises", () => ({ + ...jest.requireActual('fs/promises'), + cp: jest.fn().mockImplementation((from, to) => fs.writeFileSync(to, testFileData3)) +})); +const { cp } = require('fs/promises'); const operationContext = require('../../Common/sources/operationContext'); const storage = require('../../Common/sources/storage-base'); @@ -19,9 +31,7 @@ const urlType = commonDefines.c_oAscUrlTypes.Session; let testFile1 = testDir + "/test1.txt"; let testFile2 = testDir + "/test2.txt"; let testFile3 = testDir + "/test3.txt"; -let testFileData1 = "test1"; -let testFileData2 = "test2"; -let testFileData3 = testFileData2; +let testFile4 = testDir + "/test4.txt"; console.debug(`testDir: ${testDir}`) @@ -42,43 +52,58 @@ function runTestForDir(specialDir) { }); test("putObject", async () => { let buffer = Buffer.from(testFileData1); - await storage.putObject(ctx, testFile1, buffer, buffer.length, specialDir); + let res = await storage.putObject(ctx, testFile1, buffer, buffer.length, specialDir); + expect(res).toEqual(undefined); let list = await storage.listObjects(ctx, testDir, specialDir); expect(list.sort()).toEqual([testFile1].sort()); }); + test("putObject-stream", async () => { + let buffer = Buffer.from(testFileData2); + const stream = Readable.from(buffer); + let res = await storage.putObject(ctx, testFile2, stream, buffer.length, specialDir); + expect(res).toEqual(undefined); + let list = await storage.listObjects(ctx, testDir, specialDir); + expect(list.sort()).toEqual([testFile1, testFile2].sort()); + }); if ("storage-fs" === cfgStorageName) { - test("todo UploadObject in fs", async () => { - let buffer = Buffer.from(testFileData2); - await storage.putObject(ctx, testFile2, buffer, buffer.length, specialDir); + test("UploadObject", async () => { + let res = await storage.uploadObject(ctx, testFile3, "createReadStream.txt", specialDir); + expect(res).toEqual(undefined); + expect(cp).toHaveBeenCalled(); let list = await storage.listObjects(ctx, testDir, specialDir); - expect(list.sort()).toEqual([testFile1, testFile2].sort()); + expect(list.sort()).toEqual([testFile1, testFile2, testFile3].sort()); }); } else { test("uploadObject", async () => { - const spy = jest.spyOn(fs, 'createReadStream').mockReturnValue(testFileData2); - await storage.uploadObject(ctx, testFile2, "createReadStream.txt", specialDir); + const spy = jest.spyOn(fs, 'createReadStream').mockReturnValue(testFileData3); + let res = await storage.uploadObject(ctx, testFile3, "createReadStream.txt", specialDir); + expect(res).toEqual(undefined); let list = await storage.listObjects(ctx, testDir, specialDir); expect(spy).toHaveBeenCalled(); - expect(list.sort()).toEqual([testFile1, testFile2].sort()); + expect(list.sort()).toEqual([testFile1, testFile2, testFile3].sort()); }); } test("copyObject", async () => { - await storage.copyObject(ctx, testFile2, testFile3, specialDir, specialDir); + let res = await storage.copyObject(ctx, testFile3, testFile4, specialDir, specialDir); + expect(res).toEqual(undefined); // let buffer = Buffer.from(testFileData3); // await storage.putObject(ctx, testFile3, buffer, buffer.length, specialDir); let list = await storage.listObjects(ctx, testDir, specialDir); - expect(list.sort()).toEqual([testFile1, testFile2, testFile3].sort()); + expect(list.sort()).toEqual([testFile1, testFile2, testFile3, testFile4].sort()); }); test("headObject", async () => { let output; output = await storage.headObject(ctx, testFile1, specialDir); - expect(output).toHaveProperty("ContentLength", testFileData1.length); + expect(output).toMatchObject({ContentLength: testFileData1.length}); output = await storage.headObject(ctx, testFile2, specialDir); - expect(output).toHaveProperty("ContentLength", testFileData2.length); + expect(output).toMatchObject({ContentLength: testFileData2.length}); output = await storage.headObject(ctx, testFile3, specialDir); - expect(output).toHaveProperty("ContentLength", testFileData3.length); + expect(output).toMatchObject({ContentLength: testFileData3.length}); + + output = await storage.headObject(ctx, testFile4, specialDir); + expect(output).toMatchObject({ContentLength: testFileData4.length}); }); test("getObject", async () => { let output; @@ -90,21 +115,27 @@ function runTestForDir(specialDir) { output = await storage.getObject(ctx, testFile3, specialDir); expect(output.toString("utf8")).toEqual(testFileData3); + + output = await storage.getObject(ctx, testFile4, specialDir); + expect(output.toString("utf8")).toEqual(testFileData4); }); test("createReadStream", async () => { let output, outputText; output = await storage.createReadStream(ctx, testFile1, specialDir); await utils.sleep(100); + expect(output.contentLength).toEqual(testFileData1.length); outputText = await utils.stream2Buffer(output.readStream); await utils.sleep(100); expect(outputText.toString("utf8")).toEqual(testFileData1); output = await storage.createReadStream(ctx, testFile2, specialDir); + expect(output.contentLength).toEqual(testFileData2.length); outputText = await utils.stream2Buffer(output.readStream); expect(outputText.toString("utf8")).toEqual(testFileData2); output = await storage.createReadStream(ctx, testFile3, specialDir); + expect(output.contentLength).toEqual(testFileData3.length); outputText = await utils.stream2Buffer(output.readStream); expect(outputText.toString("utf8")).toEqual(testFileData3); }); @@ -121,23 +152,29 @@ function runTestForDir(specialDir) { url = await storage.getSignedUrl(ctx, baseUrl, testFile3, urlType, undefined, undefined, specialDir); data = await request(url); expect(data).toEqual(testFileData3); + + url = await storage.getSignedUrl(ctx, baseUrl, testFile4, urlType, undefined, undefined, specialDir); + data = await request(url); + expect(data).toEqual(testFileData4); }); test("deleteObject", async () => { let list; list = await storage.listObjects(ctx, testDir, specialDir); - expect(list.sort()).toEqual([testFile1, testFile2, testFile3].sort()); + expect(list.sort()).toEqual([testFile1, testFile2, testFile3, testFile4].sort()); - await storage.deleteObject(ctx, testFile1, specialDir); + let res = await storage.deleteObject(ctx, testFile1, specialDir); + expect(res).toEqual(undefined); list = await storage.listObjects(ctx, testDir, specialDir); - expect(list.sort()).toEqual([testFile2, testFile3].sort()); + expect(list.sort()).toEqual([testFile2, testFile3, testFile4].sort()); }); - test("deleteObjects", async () => { + test("deletePath", async () => { let list; list = await storage.listObjects(ctx, testDir, specialDir); - expect(list.sort()).toEqual([testFile2, testFile3].sort()); + expect(list.sort()).toEqual([testFile2, testFile3, testFile4].sort()); - await storage.deleteObjects(ctx, list, specialDir); + let res = await storage.deletePath(ctx, testDir, specialDir); + expect(res).toEqual(undefined); list = await storage.listObjects(ctx, testDir, specialDir); expect(list.sort()).toEqual([].sort()); diff --git a/tests/perf/checkFileExpire.js b/tests/perf/checkFileExpire.js index 6a75356a..406253d1 100644 --- a/tests/perf/checkFileExpire.js +++ b/tests/perf/checkFileExpire.js @@ -68,7 +68,7 @@ async function beforeStart() { taskResult.remove = timerify(taskResult.remove, "remove"); storage.putObject = timerify(storage.putObject, "putObject"); storage.listObjects = timerify(storage.listObjects, "listObjects"); - storage.deleteObjects = timerify(storage.deleteObjects, "deleteObjects"); + storageFs.deletePath = timerify(storageFs.deletePath, "deletePath"); storageFs.deleteObject = timerify(storageFs.deleteObject, "deleteObject"); docsCoServer.getEditorsCountPromise = timerify(docsCoServer.getEditorsCountPromise, "getEditorsCountPromise");