diff --git a/Common/config/default.json b/Common/config/default.json index 9a169021..4cd62b7e 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -25,7 +25,7 @@ "endpoint": "http://localhost/s3", "bucketName": "cache", "storageFolderName": "files", - "urlExpires": 60, + "urlExpires": 604800, "accessKeyId": "AKID", "secretAccessKey": "SECRET", "useRequestToGetUrl": false, diff --git a/Common/sources/storage-fs.js b/Common/sources/storage-fs.js index b0622efa..3e212944 100644 --- a/Common/sources/storage-fs.js +++ b/Common/sources/storage-fs.js @@ -1 +1 @@ -var fs = require('fs'); var path = require('path'); var mkdirp = require('mkdirp'); var utils = require('./utils'); var crypto = require('crypto'); var configStorage = require('config').get('storage'); var cfgBucketName = configStorage.get('bucketName'); var cfgStorageFolderName = configStorage.get('storageFolderName'); var cfgStorageExternalHost = configStorage.get('externalHost'); var configFs = configStorage.get('fs'); var cfgStorageFolderPath = configFs.get('folderPath'); var cfgStorageSecretString = configFs.get('secretString'); function getFilePath(strPath) { return path.join(cfgStorageFolderPath, strPath); } function getOutputPath(strPath) { return strPath.replace(/\\/g, '/'); } function removeEmptyParent(strPath, done) { if (cfgStorageFolderPath.length + 1 >= strPath.length) { done(); } else { fs.readdir(strPath, function(err, list) { if (err) { //не реагируем на ошибку, потому скорее всего эта папка удалилась в соседнем потоке done(); } else { if (list.length > 0) { done(); } else { fs.rmdir(strPath, function(err) { if (err) { //не реагируем на ошибку, потому скорее всего эта папка удалилась в соседнем потоке done(); } else { removeEmptyParent(path.dirname(strPath), function(err) { done(err); }); } }); } } }); } } exports.getObject = function(strPath) { return new Promise(function(resolve, reject) { fs.readFile(getFilePath(strPath), function(err, data) { if (err) { reject(err); } else { resolve(data); } }); }); }; exports.putObject = function(strPath, buffer, contentLength) { return new Promise(function(resolve, reject) { var fsPath = getFilePath(strPath); mkdirp(path.dirname(fsPath), function(err) { if (err) { reject(err); } else { //todo 0666 if (Buffer.isBuffer(buffer)) { fs.writeFile(fsPath, buffer, function(err) { if (err) { reject(err); } else { resolve(); } }); } else { utils.promiseCreateWriteStream(fsPath).then(function(writable) { buffer.pipe(writable); }).catch(function(err) { reject(err); }); } } }); }); }; exports.listObjects = function(strPath) { return utils.listObjects(getFilePath(strPath)).then(function(values) { return values.map(function(curvalue) { return getOutputPath(curvalue.substring(cfgStorageFolderPath.length + 1)); }); }); }; exports.deleteObject = function(strPath) { return new Promise(function(resolve, reject) { var fsPath = getFilePath(strPath); fs.unlink(fsPath, function(err) { if (err) { reject(err); } else { //resolve(); removeEmptyParent(path.dirname(fsPath), function(err) { if (err) { reject(err); } else { resolve(); } }); } }); }); }; exports.deleteObjects = function(strPaths) { return Promise.all(strPaths.map(exports.deleteObject)); }; exports.getSignedUrl = function(baseUrl, strPath, optUrlExpires, optFilename) { return new Promise(function(resolve, reject) { var userFriendlyName = optFilename ? encodeURIComponent(optFilename) : path.basename(strPath); var uri = '/' + cfgBucketName + '/' + cfgStorageFolderName + '/' + strPath + '/' + userFriendlyName; var url = (cfgStorageExternalHost ? cfgStorageExternalHost : baseUrl) + uri; var date = new Date(); var expires = Math.ceil(date.getTime() / 1000) + (optUrlExpires || 60); // отключил время жизни т.к. существует сценарий, при котором объект // получаемый по ссылке запрашивается после того как закончилось время // его жизни. var md5 = crypto.createHash('md5').update(/*expires + */uri + cfgStorageSecretString).digest("base64"); md5 = md5.replace(/\+/g, "-"); md5 = md5.replace(/\//g, "_"); url += ('?md5=' + md5 + '&expires=' + expires); resolve(utils.changeOnlyOfficeUrl(url, strPath, optFilename)); }); }; \ No newline at end of file +var fs = require('fs'); var path = require('path'); var mkdirp = require('mkdirp'); var utils = require('./utils'); var crypto = require('crypto'); var configStorage = require('config').get('storage'); var cfgBucketName = configStorage.get('bucketName'); var cfgStorageFolderName = configStorage.get('storageFolderName'); var cfgStorageExternalHost = configStorage.get('externalHost'); var configFs = configStorage.get('fs'); var cfgStorageFolderPath = configFs.get('folderPath'); var cfgStorageSecretString = configFs.get('secretString'); function getFilePath(strPath) { return path.join(cfgStorageFolderPath, strPath); } function getOutputPath(strPath) { return strPath.replace(/\\/g, '/'); } function removeEmptyParent(strPath, done) { if (cfgStorageFolderPath.length + 1 >= strPath.length) { done(); } else { fs.readdir(strPath, function(err, list) { if (err) { //не реагируем на ошибку, потому скорее всего эта папка удалилась в соседнем потоке done(); } else { if (list.length > 0) { done(); } else { fs.rmdir(strPath, function(err) { if (err) { //не реагируем на ошибку, потому скорее всего эта папка удалилась в соседнем потоке done(); } else { removeEmptyParent(path.dirname(strPath), function(err) { done(err); }); } }); } } }); } } exports.getObject = function(strPath) { return new Promise(function(resolve, reject) { fs.readFile(getFilePath(strPath), function(err, data) { if (err) { reject(err); } else { resolve(data); } }); }); }; exports.putObject = function(strPath, buffer, contentLength) { return new Promise(function(resolve, reject) { var fsPath = getFilePath(strPath); mkdirp(path.dirname(fsPath), function(err) { if (err) { reject(err); } else { //todo 0666 if (Buffer.isBuffer(buffer)) { fs.writeFile(fsPath, buffer, function(err) { if (err) { reject(err); } else { resolve(); } }); } else { utils.promiseCreateWriteStream(fsPath).then(function(writable) { buffer.pipe(writable); }).catch(function(err) { reject(err); }); } } }); }); }; exports.listObjects = function(strPath) { return utils.listObjects(getFilePath(strPath)).then(function(values) { return values.map(function(curvalue) { return getOutputPath(curvalue.substring(cfgStorageFolderPath.length + 1)); }); }); }; exports.deleteObject = function(strPath) { return new Promise(function(resolve, reject) { var fsPath = getFilePath(strPath); fs.unlink(fsPath, function(err) { if (err) { reject(err); } else { //resolve(); removeEmptyParent(path.dirname(fsPath), function(err) { if (err) { reject(err); } else { resolve(); } }); } }); }); }; exports.deleteObjects = function(strPaths) { return Promise.all(strPaths.map(exports.deleteObject)); }; exports.getSignedUrl = function(baseUrl, strPath, optUrlExpires, optFilename) { return new Promise(function(resolve, reject) { var userFriendlyName = optFilename ? encodeURIComponent(optFilename) : path.basename(strPath); var uri = '/' + cfgBucketName + '/' + cfgStorageFolderName + '/' + strPath + '/' + userFriendlyName; var url = (cfgStorageExternalHost ? cfgStorageExternalHost : baseUrl) + uri; var date = new Date(); var expires = Math.ceil(date.getTime() / 1000) + (optUrlExpires || 604800); // отключил время жизни т.к. существует сценарий, при котором объект // получаемый по ссылке запрашивается после того как закончилось время // его жизни. var md5 = crypto.createHash('md5').update(/*expires + */uri + cfgStorageSecretString).digest("base64"); md5 = md5.replace(/\+/g, "-"); md5 = md5.replace(/\//g, "_"); url += ('?md5=' + md5 + '&expires=' + expires); resolve(utils.changeOnlyOfficeUrl(url, strPath, optFilename)); }); }; \ No newline at end of file diff --git a/Common/sources/storage-s3.js b/Common/sources/storage-s3.js index 9d856771..0bd4fbf7 100644 --- a/Common/sources/storage-s3.js +++ b/Common/sources/storage-s3.js @@ -1 +1 @@ -'use strict'; var url = require('url'); var path = require('path'); var AWS = require('aws-sdk'); var mime = require('mime'); var s3urlSigner = require('amazon-s3-url-signer'); var utils = require('./utils'); var configStorage = require('config').get('storage'); var cfgRegion = configStorage.get('region'); var cfgEndpoint = configStorage.get('endpoint'); var cfgBucketName = configStorage.get('bucketName'); var cfgStorageFolderName = configStorage.get('storageFolderName'); var cfgAccessKeyId = configStorage.get('accessKeyId'); var cfgSecretAccessKey = configStorage.get('secretAccessKey'); var cfgUseRequestToGetUrl = configStorage.get('useRequestToGetUrl'); var cfgUseSignedUrl = configStorage.get('useSignedUrl'); var cfgExternalHost = configStorage.get('externalHost'); /** * Don't hard-code your credentials! * Export the following environment variables instead: * * export AWS_ACCESS_KEY_ID='AKID' * export AWS_SECRET_ACCESS_KEY='SECRET' */ var configS3 = { region: cfgRegion, endpoint: cfgEndpoint, accessKeyId: cfgAccessKeyId, secretAccessKey: cfgSecretAccessKey }; if (configS3.endpoint) { configS3.sslEnabled = false; configS3.s3ForcePathStyle = true; } AWS.config.update(configS3); var s3Client = new AWS.S3(); if (configS3.endpoint) { s3Client.endpoint = new AWS.Endpoint(configS3.endpoint); } var cfgEndpointParsed = null; if (cfgEndpoint) { cfgEndpointParsed = url.parse(cfgEndpoint); } //s3 allow only 1000 per request var MAX_DELETE_OBJECTS = 1000; function getFilePath(strPath) { //todo return cfgStorageFolderName + '/' + strPath; } function joinListObjects(inputArray, outputArray) { var length = inputArray.length; for (var i = 0; i < length; i++) { outputArray.push(inputArray[i].Key.substring((cfgStorageFolderName + '/').length)); } } function listObjectsExec(output, params, resolve, reject) { s3Client.listObjects(params, function(err, data) { if (err) { reject(err); } else { joinListObjects(data.Contents, output); if (data.IsTruncated) { params.Marker = data.NextMarker; listObjectsExec(output, params, resolve, reject); } else { resolve(output); } } }); } function mapDeleteObjects(currentValue) { return {Key: currentValue}; } function deleteObjectsHelp(aKeys) { return new Promise(function(resolve, reject) { //todo Quiet var params = {Bucket: cfgBucketName, Delete: {Objects: aKeys}}; s3Client.deleteObjects(params, function(err, data) { if (err) { reject(err); } else { resolve(data); } }); }); } exports.getObject = function(strPath) { return new Promise(function(resolve, reject) { var params = {Bucket: cfgBucketName, Key: getFilePath(strPath)}; s3Client.getObject(params, function(err, data) { if (err) { reject(err); } else { resolve(data.Body); } }); }); }; exports.putObject = function(strPath, buffer, contentLength) { return new Promise(function(resolve, reject) { //todo рассмотреть Expires var params = {Bucket: cfgBucketName, Key: getFilePath(strPath), Body: buffer, ContentLength: contentLength, ContentType: mime.lookup(strPath), ContentDisposition : 'attachment;'}; s3Client.putObject(params, function(err, data) { if (err) { reject(err); } else { resolve(data); } }); }); }; exports.listObjects = function(strPath) { return new Promise(function(resolve, reject) { var params = {Bucket: cfgBucketName, Prefix: getFilePath(strPath)}; var output = []; listObjectsExec(output, params, resolve, reject); }); }; exports.deleteObject = function(strPath) { return new Promise(function(resolve, reject) { var params = {Bucket: cfgBucketName, Key: getFilePath(strPath)}; s3Client.deleteObject(params, function(err, data) { if (err) { reject(err); } else { resolve(data); } }); }); }; exports.deleteObjects = function(strPaths) { return new Promise(function(resolve) { resolve(strPaths.map(mapDeleteObjects)); }).then(function(aKeys) { var deletePromises = []; for (var i = 0; i < aKeys.length; i += MAX_DELETE_OBJECTS) { deletePromises.push(deleteObjectsHelp(aKeys.slice(i, i + MAX_DELETE_OBJECTS))); } return Promise.all(deletePromises); }); }; exports.getSignedUrl = function(baseUrl, strPath, optUrlExpires, optFilename) { return new Promise(function(resolve, reject) { var expires = optUrlExpires || 5; if (cfgUseRequestToGetUrl) { var contentDisposition = utils.getContentDisposition(optFilename || path.basename(strPath)); var params = { Bucket: cfgBucketName, Key: getFilePath(strPath), ResponseContentDisposition: contentDisposition, Expires: expires }; s3Client.getSignedUrl('getObject', params, function(err, data) { if (err) { reject(err); } else { resolve(utils.changeOnlyOfficeUrl(data, strPath, optFilename)); } }); } else { var host; if (cfgEndpointParsed && (cfgEndpointParsed.hostname == 'localhost' || cfgEndpointParsed.hostname == '127.0.0.1') && 80 == cfgEndpointParsed.port) { host = (cfgExternalHost ? cfgExternalHost : baseUrl) + cfgEndpointParsed.path; } else { host = cfgEndpoint; } if (host && host.length > 0 && '/' != host[host.length - 1]) { host += '/'; } var newUrl; if (cfgUseSignedUrl) { //todo уйти от parse var hostParsed = url.parse(host); var protocol = hostParsed.protocol.substring(0, hostParsed.protocol.length - 1); var signerOptions = { host: hostParsed.hostname, port: hostParsed.port, protocol: protocol, useSubdomain: false }; var awsUrlSigner = s3urlSigner.urlSigner(cfgAccessKeyId, cfgSecretAccessKey, signerOptions); newUrl = awsUrlSigner.getUrl('GET', getFilePath(strPath), cfgBucketName, expires); } else { newUrl = host + cfgBucketName + '/' + cfgStorageFolderName + '/' + strPath; } resolve(utils.changeOnlyOfficeUrl(newUrl, strPath, optFilename)); } }); }; \ No newline at end of file +'use strict'; var url = require('url'); var path = require('path'); var AWS = require('aws-sdk'); var mime = require('mime'); var s3urlSigner = require('amazon-s3-url-signer'); var utils = require('./utils'); var configStorage = require('config').get('storage'); var cfgRegion = configStorage.get('region'); var cfgEndpoint = configStorage.get('endpoint'); var cfgBucketName = configStorage.get('bucketName'); var cfgStorageFolderName = configStorage.get('storageFolderName'); var cfgAccessKeyId = configStorage.get('accessKeyId'); var cfgSecretAccessKey = configStorage.get('secretAccessKey'); var cfgUseRequestToGetUrl = configStorage.get('useRequestToGetUrl'); var cfgUseSignedUrl = configStorage.get('useSignedUrl'); var cfgExternalHost = configStorage.get('externalHost'); /** * Don't hard-code your credentials! * Export the following environment variables instead: * * export AWS_ACCESS_KEY_ID='AKID' * export AWS_SECRET_ACCESS_KEY='SECRET' */ var configS3 = { region: cfgRegion, endpoint: cfgEndpoint, accessKeyId: cfgAccessKeyId, secretAccessKey: cfgSecretAccessKey }; if (configS3.endpoint) { configS3.sslEnabled = false; configS3.s3ForcePathStyle = true; } AWS.config.update(configS3); var s3Client = new AWS.S3(); if (configS3.endpoint) { s3Client.endpoint = new AWS.Endpoint(configS3.endpoint); } var cfgEndpointParsed = null; if (cfgEndpoint) { cfgEndpointParsed = url.parse(cfgEndpoint); } //This operation enables you to delete multiple objects from a bucket using a single HTTP request. You may specify up to 1000 keys. var MAX_DELETE_OBJECTS = 1000; function getFilePath(strPath) { //todo return cfgStorageFolderName + '/' + strPath; } function joinListObjects(inputArray, outputArray) { var length = inputArray.length; for (var i = 0; i < length; i++) { outputArray.push(inputArray[i].Key.substring((cfgStorageFolderName + '/').length)); } } function listObjectsExec(output, params, resolve, reject) { s3Client.listObjects(params, function(err, data) { if (err) { reject(err); } else { joinListObjects(data.Contents, output); if (data.IsTruncated && (data.NextMarker || data.Contents.length > 0)) { params.Marker = data.NextMarker || data.Contents[data.Contents.length - 1].Key; listObjectsExec(output, params, resolve, reject); } else { resolve(output); } } }); } function mapDeleteObjects(currentValue) { return {Key: currentValue}; } function deleteObjectsHelp(aKeys) { return new Promise(function(resolve, reject) { //By default, the operation uses verbose mode in which the response includes the result of deletion of each key in your request. //In quiet mode the response includes only keys where the delete operation encountered an error. var params = {Bucket: cfgBucketName, Delete: {Objects: aKeys, Quiet: true}}; s3Client.deleteObjects(params, function(err, data) { if (err) { reject(err); } else { resolve(data); } }); }); } exports.getObject = function(strPath) { return new Promise(function(resolve, reject) { var params = {Bucket: cfgBucketName, Key: getFilePath(strPath)}; s3Client.getObject(params, function(err, data) { if (err) { reject(err); } else { resolve(data.Body); } }); }); }; exports.putObject = function(strPath, buffer, contentLength) { return new Promise(function(resolve, reject) { //todo рассмотреть Expires var params = {Bucket: cfgBucketName, Key: getFilePath(strPath), Body: buffer, ContentLength: contentLength, ContentType: mime.lookup(strPath)}; s3Client.putObject(params, function(err, data) { if (err) { reject(err); } else { resolve(data); } }); }); }; exports.listObjects = function(strPath) { return new Promise(function(resolve, reject) { var params = {Bucket: cfgBucketName, Prefix: getFilePath(strPath)}; var output = []; listObjectsExec(output, params, resolve, reject); }); }; exports.deleteObject = function(strPath) { return new Promise(function(resolve, reject) { var params = {Bucket: cfgBucketName, Key: getFilePath(strPath)}; s3Client.deleteObject(params, function(err, data) { if (err) { reject(err); } else { resolve(data); } }); }); }; exports.deleteObjects = function(strPaths) { var aKeys = strPaths.map(function (currentValue) { return {Key: getFilePath(currentValue)}; }); var deletePromises = []; for (var i = 0; i < aKeys.length; i += MAX_DELETE_OBJECTS) { deletePromises.push(deleteObjectsHelp(aKeys.slice(i, i + MAX_DELETE_OBJECTS))); } return Promise.all(deletePromises); }; exports.getSignedUrl = function(baseUrl, strPath, optUrlExpires, optFilename) { return new Promise(function(resolve, reject) { var expires = optUrlExpires || 604800; var contentDisposition = utils.getContentDispositionS3(optFilename || path.basename(strPath)); if (cfgUseRequestToGetUrl) { //default Expires 900 seconds var params = { Bucket: cfgBucketName, Key: getFilePath(strPath), ResponseContentDisposition: contentDisposition, Expires: expires }; s3Client.getSignedUrl('getObject', params, function(err, data) { if (err) { reject(err); } else { resolve(utils.changeOnlyOfficeUrl(data, strPath, optFilename)); } }); } else { var host; if (cfgRegion) { host = 'https://s3-'+cfgRegion+'.amazonaws.com'; } else if (cfgEndpointParsed && (cfgEndpointParsed.hostname == 'localhost' || cfgEndpointParsed.hostname == '127.0.0.1') && 80 == cfgEndpointParsed.port) { host = (cfgExternalHost ? cfgExternalHost : baseUrl) + cfgEndpointParsed.path; } else { host = cfgEndpoint; } if (host && host.length > 0 && '/' != host[host.length - 1]) { host += '/'; } var newUrl; if (cfgUseSignedUrl) { //todo уйти от parse var hostParsed = url.parse(host); var protocol = hostParsed.protocol.substring(0, hostParsed.protocol.length - 1); var signerOptions = { host: hostParsed.hostname, port: hostParsed.port, protocol: protocol, useSubdomain: false }; var awsUrlSigner = s3urlSigner.urlSigner(cfgAccessKeyId, cfgSecretAccessKey, signerOptions); newUrl = awsUrlSigner.getUrl('GET', getFilePath(strPath), cfgBucketName, expires, contentDisposition); } else { newUrl = host + cfgBucketName + '/' + cfgStorageFolderName + '/' + strPath; } resolve(utils.changeOnlyOfficeUrl(newUrl, strPath, optFilename)); } }); }; \ No newline at end of file diff --git a/Common/sources/utils.js b/Common/sources/utils.js index 4435a02d..a331e092 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -159,7 +159,21 @@ function getContentDisposition (filename, useragent) { } return contentDisposition; } +function getContentDispositionS3 (filename, useragent) { + var contentDisposition = 'attachment;'; + if (useragent != null && -1 != useragent.toLowerCase().indexOf('android')) { + contentDisposition += ' filename=' + makeAndroidSafeFileName(filename); + } else { + if (containsAllAsciiNP(filename)) { + contentDisposition += ' filename=' + filename; + } else { + contentDisposition += ' filename*=UTF-8\'\'' + encodeRFC5987ValueChars(filename); + } + } + return contentDisposition; +} exports.getContentDisposition = getContentDisposition; +exports.getContentDispositionS3 = getContentDispositionS3; function downloadUrlPromise(uri, optTimeout, optLimit) { return new Promise(function (resolve, reject) { //todo может стоит делать url.parse, а потом с каждой частью отдельно работать