[feature] add azure blob storage

Common/npm-shrinkwrap.json

@@ -3076,6 +3076,219 @@
         }
       }
     },
+    "@azure/abort-controller": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz",
+      "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==",
+      "requires": {
+        "tslib": "^2.6.2"
+      },
+      "dependencies": {
+        "tslib": {
+          "version": "2.8.1",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+          "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="
+        }
+      }
+    },
+    "@azure/core-auth": {
+      "version": "1.9.0",
+      "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.9.0.tgz",
+      "integrity": "sha512-FPwHpZywuyasDSLMqJ6fhbOK3TqUdviZNF8OqRGA4W5Ewib2lEEZ+pBsYcBa88B2NGO/SEnYPGhyBqNlE8ilSw==",
+      "requires": {
+        "@azure/abort-controller": "^2.0.0",
+        "@azure/core-util": "^1.11.0",
+        "tslib": "^2.6.2"
+      },
+      "dependencies": {
+        "tslib": {
+          "version": "2.8.1",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+          "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="
+        }
+      }
+    },
+    "@azure/core-client": {
+      "version": "1.9.3",
+      "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.9.3.tgz",
+      "integrity": "sha512-/wGw8fJ4mdpJ1Cum7s1S+VQyXt1ihwKLzfabS1O/RDADnmzVc01dHn44qD0BvGH6KlZNzOMW95tEpKqhkCChPA==",
+      "requires": {
+        "@azure/abort-controller": "^2.0.0",
+        "@azure/core-auth": "^1.4.0",
+        "@azure/core-rest-pipeline": "^1.9.1",
+        "@azure/core-tracing": "^1.0.0",
+        "@azure/core-util": "^1.6.1",
+        "@azure/logger": "^1.0.0",
+        "tslib": "^2.6.2"
+      },
+      "dependencies": {
+        "tslib": {
+          "version": "2.8.1",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+          "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="
+        }
+      }
+    },
+    "@azure/core-http-compat": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.2.0.tgz",
+      "integrity": "sha512-1kW8ZhN0CfbNOG6C688z5uh2yrzALE7dDXHiR9dY4vt+EbhGZQSbjDa5bQd2rf3X2pdWMsXbqbArxUyeNdvtmg==",
+      "requires": {
+        "@azure/abort-controller": "^2.0.0",
+        "@azure/core-client": "^1.3.0",
+        "@azure/core-rest-pipeline": "^1.19.0"
+      }
+    },
+    "@azure/core-lro": {
+      "version": "2.7.2",
+      "resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.7.2.tgz",
+      "integrity": "sha512-0YIpccoX8m/k00O7mDDMdJpbr6mf1yWo2dfmxt5A8XVZVVMz2SSKaEbMCeJRvgQ0IaSlqhjT47p4hVIRRy90xw==",
+      "requires": {
+        "@azure/abort-controller": "^2.0.0",
+        "@azure/core-util": "^1.2.0",
+        "@azure/logger": "^1.0.0",
+        "tslib": "^2.6.2"
+      },
+      "dependencies": {
+        "tslib": {
+          "version": "2.8.1",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+          "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="
+        }
+      }
+    },
+    "@azure/core-paging": {
+      "version": "1.6.2",
+      "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.6.2.tgz",
+      "integrity": "sha512-YKWi9YuCU04B55h25cnOYZHxXYtEvQEbKST5vqRga7hWY9ydd3FZHdeQF8pyh+acWZvppw13M/LMGx0LABUVMA==",
+      "requires": {
+        "tslib": "^2.6.2"
+      },
+      "dependencies": {
+        "tslib": {
+          "version": "2.8.1",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+          "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="
+        }
+      }
+    },
+    "@azure/core-rest-pipeline": {
+      "version": "1.19.1",
+      "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.19.1.tgz",
+      "integrity": "sha512-zHeoI3NCs53lLBbWNzQycjnYKsA1CVKlnzSNuSFcUDwBp8HHVObePxrM7HaX+Ha5Ks639H7chNC9HOaIhNS03w==",
+      "requires": {
+        "@azure/abort-controller": "^2.0.0",
+        "@azure/core-auth": "^1.8.0",
+        "@azure/core-tracing": "^1.0.1",
+        "@azure/core-util": "^1.11.0",
+        "@azure/logger": "^1.0.0",
+        "http-proxy-agent": "^7.0.0",
+        "https-proxy-agent": "^7.0.0",
+        "tslib": "^2.6.2"
+      },
+      "dependencies": {
+        "tslib": {
+          "version": "2.8.1",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+          "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="
+        }
+      }
+    },
+    "@azure/core-tracing": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.2.0.tgz",
+      "integrity": "sha512-UKTiEJPkWcESPYJz3X5uKRYyOcJD+4nYph+KpfdPRnQJVrZfk0KJgdnaAWKfhsBBtAf/D58Az4AvCJEmWgIBAg==",
+      "requires": {
+        "tslib": "^2.6.2"
+      },
+      "dependencies": {
+        "tslib": {
+          "version": "2.8.1",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+          "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="
+        }
+      }
+    },
+    "@azure/core-util": {
+      "version": "1.11.0",
+      "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.11.0.tgz",
+      "integrity": "sha512-DxOSLua+NdpWoSqULhjDyAZTXFdP/LKkqtYuxxz1SCN289zk3OG8UOpnCQAz/tygyACBtWp/BoO72ptK7msY8g==",
+      "requires": {
+        "@azure/abort-controller": "^2.0.0",
+        "tslib": "^2.6.2"
+      },
+      "dependencies": {
+        "tslib": {
+          "version": "2.8.1",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+          "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="
+        }
+      }
+    },
+    "@azure/core-xml": {
+      "version": "1.4.5",
+      "resolved": "https://registry.npmjs.org/@azure/core-xml/-/core-xml-1.4.5.tgz",
+      "integrity": "sha512-gT4H8mTaSXRz7eGTuQyq1aIJnJqeXzpOe9Ay7Z3FrCouer14CbV3VzjnJrNrQfbBpGBLO9oy8BmrY75A0p53cA==",
+      "requires": {
+        "fast-xml-parser": "^5.0.7",
+        "tslib": "^2.8.1"
+      },
+      "dependencies": {
+        "fast-xml-parser": {
+          "version": "5.0.9",
+          "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.0.9.tgz",
+          "integrity": "sha512-2mBwCiuW3ycKQQ6SOesSB8WeF+fIGb6I/GG5vU5/XEptwFFhp9PE8b9O7fbs2dpq9fXn4ULR3UsfydNUCntf5A==",
+          "requires": {
+            "strnum": "^2.0.5"
+          }
+        },
+        "strnum": {
+          "version": "2.0.5",
+          "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.0.5.tgz",
+          "integrity": "sha512-YAT3K/sgpCUxhxNMrrdhtod3jckkpYwH6JAuwmUdXZsmzH1wUyzTMrrK2wYCEEqlKwrWDd35NeuUkbBy/1iK+Q=="
+        },
+        "tslib": {
+          "version": "2.8.1",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+          "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="
+        }
+      }
+    },
+    "@azure/logger": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.1.4.tgz",
+      "integrity": "sha512-4IXXzcCdLdlXuCG+8UKEwLA1T1NHqUfanhXYHiQTn+6sfWCZXduqbtXDGceg3Ce5QxTGo7EqmbV6Bi+aqKuClQ==",
+      "requires": {
+        "tslib": "^2.6.2"
+      },
+      "dependencies": {
+        "tslib": {
+          "version": "2.8.1",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+          "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="
+        }
+      }
+    },
+    "@azure/storage-blob": {
+      "version": "12.27.0",
+      "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.27.0.tgz",
+      "integrity": "sha512-IQjj9RIzAKatmNca3D6bT0qJ+Pkox1WZGOg2esJF2YLHb45pQKOwGPIAV+w3rfgkj7zV3RMxpn/c6iftzSOZJQ==",
+      "requires": {
+        "@azure/abort-controller": "^2.1.2",
+        "@azure/core-auth": "^1.4.0",
+        "@azure/core-client": "^1.6.2",
+        "@azure/core-http-compat": "^2.0.0",
+        "@azure/core-lro": "^2.2.0",
+        "@azure/core-paging": "^1.1.1",
+        "@azure/core-rest-pipeline": "^1.10.1",
+        "@azure/core-tracing": "^1.1.2",
+        "@azure/core-util": "^1.6.1",
+        "@azure/core-xml": "^1.4.3",
+        "@azure/logger": "^1.0.0",
+        "events": "^3.0.0",
+        "tslib": "^2.2.0"
+      }
+    },
     "@smithy/abort-controller": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-1.0.2.tgz",
@@ -4959,6 +5172,11 @@
         }
       }
     },
+    "agent-base": {
+      "version": "7.1.3",
+      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz",
+      "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw=="
+    },
     "ajv": {
       "version": "5.5.2",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz",
@@ -4993,7 +5211,7 @@
     "asap": {
       "version": "2.0.6",
       "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
-      "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY="
+      "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA=="
     },
     "asn1": {
       "version": "0.2.4",
@@ -5084,7 +5302,7 @@
     "clone": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz",
-      "integrity": "sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18="
+      "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w=="
     },
     "co": {
       "version": "4.6.0",
@@ -5177,6 +5395,11 @@
       "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
       "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ="
     },
+    "events": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
+      "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q=="
+    },
     "extend": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
@@ -5252,6 +5475,30 @@
         "har-schema": "^2.0.0"
       }
     },
+    "http-proxy-agent": {
+      "version": "7.0.2",
+      "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz",
+      "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==",
+      "requires": {
+        "agent-base": "^7.1.0",
+        "debug": "^4.3.4"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "4.4.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
+          "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
+          "requires": {
+            "ms": "^2.1.3"
+          }
+        },
+        "ms": {
+          "version": "2.1.3",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+          "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
+        }
+      }
+    },
     "http-signature": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
@@ -5262,6 +5509,30 @@
         "sshpk": "^1.7.0"
       }
     },
+    "https-proxy-agent": {
+      "version": "7.0.6",
+      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz",
+      "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==",
+      "requires": {
+        "agent-base": "^7.1.2",
+        "debug": "4"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "4.4.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
+          "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
+          "requires": {
+            "ms": "^2.1.3"
+          }
+        },
+        "ms": {
+          "version": "2.1.3",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+          "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
+        }
+      }
+    },
     "inherits": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
@@ -5489,7 +5760,7 @@
     "pify": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz",
-      "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY="
+      "integrity": "sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg=="
     },
     "psl": {
       "version": "1.1.29",
@@ -5569,7 +5840,7 @@
     "requires-port": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
-      "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8="
+      "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ=="
     },
     "rfdc": {
       "version": "1.3.0",
@@ -5693,7 +5964,7 @@
     "through": {
       "version": "2.3.8",
       "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
-      "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
+      "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg=="
     },
     "tough-cookie": {
       "version": "2.4.3",

Common/package.json

@@ -7,6 +7,7 @@
     "@aws-sdk/client-s3": "3.637.0",
     "@aws-sdk/node-http-handler": "3.374.0",
     "@aws-sdk/s3-request-presigner": "3.370.0",
+    "@azure/storage-blob": "12.27.0",
     "amqplib": "0.8.0",
     "co": "4.6.0",
     "config": "2.0.1",

Common/sources/storage-az.js

@@ -0,0 +1,207 @@
+'use strict';
+const fs = require('fs');
+const url = require('url');
+const path = require('path');
+const { BlobServiceClient, StorageSharedKeyCredential, generateBlobSASQueryParameters, BlobSASPermissions } = require('@azure/storage-blob');
+const mime = require('mime');
+const config = require('config');
+const { Readable } = require('stream');
+const utils = require('./utils');
+const ms = require('ms');
+const commonDefines = require('./../../Common/sources/commondefines');
+
+const cfgExpSessionAbsolute = ms(config.get('services.CoAuthoring.expire.sessionabsolute'));
+const MAX_DELETE_OBJECTS = 1000;
+let blobServiceClients = {};
+
+function getBlobServiceClient(storageCfg) {
+    const configKey = `${storageCfg.accessKeyId}_${storageCfg.bucketName}`;
+    if (!blobServiceClients[configKey]) {
+        const credential = new StorageSharedKeyCredential(
+            storageCfg.accessKeyId,
+            storageCfg.secretAccessKey
+        );
+        blobServiceClients[configKey] = new BlobServiceClient(
+            `https://${storageCfg.accessKeyId}.blob.core.windows.net`,
+            credential
+        );
+    }
+    return blobServiceClients[configKey];
+}
+
+function getContainerClient(storageCfg) {
+    const blobServiceClient = getBlobServiceClient(storageCfg);
+    return blobServiceClient.getContainerClient(storageCfg.bucketName);
+}
+
+function getBlobClient(storageCfg, blobName) {
+  const containerClient = getContainerClient(storageCfg);
+  return containerClient.getBlockBlobClient(blobName);
+}
+
+function getFilePath(storageCfg, strPath) {
+    const storageFolderName = storageCfg.storageFolderName;
+    return `${storageFolderName}/${strPath}`
+}
+
+async function listObjectsExec(storageCfg, prefix, output = []) {
+    const containerClient = getContainerClient(storageCfg);
+    const storageFolderName = storageCfg.storageFolderName;
+    const prefixWithFolder = storageFolderName ? `${storageFolderName}/${prefix}` : prefix;
+
+    for await (const blob of containerClient.listBlobsFlat({ prefix: prefixWithFolder })) {
+        const relativePath = storageFolderName ?
+            blob.name.substring(storageFolderName.length + 1) : blob.name;
+        output.push(relativePath);
+    }
+    return output;
+}
+
+async function deleteObjectsHelp(storageCfg, aKeys) {
+    const containerClient = getContainerClient(storageCfg);
+    await Promise.all(
+        aKeys.map(key => containerClient.deleteBlob(key.Key))
+    );
+}
+
+async function headObject(storageCfg, strPath) {
+    const blobClient = getBlobClient(storageCfg, getFilePath(storageCfg, strPath));
+    const properties = await blobClient.getProperties();
+    return { ContentLength: properties.contentLength };
+}
+
+async function getObject(storageCfg, strPath) {
+    const blobClient = getBlobClient(storageCfg, getFilePath(storageCfg, strPath));
+    const response = await blobClient.download();
+    return await utils.stream2Buffer(response.readableStreamBody);
+}
+
+async function createReadStream(storageCfg, strPath) {
+    const blobClient = getBlobClient(storageCfg, getFilePath(storageCfg, strPath));
+    const response = await blobClient.download();
+    return {
+        contentLength: response.contentLength,
+        readStream: response.readableStreamBody
+    };
+}
+
+async function putObject(storageCfg, strPath, buffer, contentLength) {
+    const blobClient = getBlobClient(storageCfg, getFilePath(storageCfg, strPath));
+    
+    const uploadOptions = {
+        blobHTTPHeaders: {
+            contentType: mime.getType(strPath),
+            contentDisposition: utils.getContentDisposition(path.basename(strPath))
+        }
+    };
+    if (buffer instanceof Buffer) {
+        // Handle Buffer upload
+        await blobClient.uploadData(buffer, uploadOptions);
+    } else if (typeof buffer.pipe === 'function') {
+        // Handle Stream upload
+        await blobClient.uploadStream(buffer, undefined, undefined, uploadOptions);
+    } else {
+        throw new TypeError('Input must be Buffer or Readable stream');
+    }
+}
+
+async function uploadObject(storageCfg, strPath, filePath) {
+    const blockBlobClient = getBlobClient(storageCfg, getFilePath(storageCfg, strPath));
+    const input = fs.createReadStream(filePath);
+    const uploadStream = input instanceof Readable ? input : new Readable({
+        read() {
+            this.push(input);
+            this.push(null);
+        }
+    });
+
+    await new Promise((resolve, reject) => {
+        uploadStream.on('error', reject);
+        blockBlobClient.uploadStream(
+            uploadStream,
+            undefined,
+            undefined,
+            {
+                blobHTTPHeaders: {
+                    contentType: mime.getType(strPath),
+                    contentDisposition: utils.getContentDisposition(path.basename(strPath))
+                }
+            }
+        )
+        .then(resolve)
+        .catch(reject);
+    });
+}
+
+async function copyObject(storageCfgSrc, storageCfgDst, sourceKey, destinationKey) {
+    const sourceBlobClient = getBlobClient(storageCfgSrc, getFilePath(storageCfgSrc, sourceKey));
+    const destBlobClient = getBlobClient(storageCfgDst, getFilePath(storageCfgDst, destinationKey));
+    const sasToken = generateBlobSASQueryParameters({
+        containerName: storageCfgSrc.bucketName,
+        blobName: getFilePath(storageCfgSrc, sourceKey),
+        permissions: BlobSASPermissions.parse("r"),
+        startsOn: new Date(),
+        expiresOn: new Date(Date.now() + 3600 * 1000)
+    }, new StorageSharedKeyCredential(storageCfgSrc.accessKeyId, storageCfgSrc.secretAccessKey)).toString();
+
+    await destBlobClient.syncCopyFromURL(`${sourceBlobClient.url}?${sasToken}`);
+}
+
+async function listObjects(storageCfg, strPath) {
+    return await listObjectsExec(storageCfg, strPath);
+}
+
+async function deleteObject(storageCfg, strPath) {
+    const blobClient = getBlobClient(storageCfg, getFilePath(storageCfg, strPath));
+    await blobClient.delete();
+}
+
+async function deleteObjects(storageCfg, strPaths) {
+    let aKeys = strPaths.map(path => ({ Key: getFilePath(storageCfg, path) }));
+    for (let i = 0; i < aKeys.length; i += MAX_DELETE_OBJECTS) {
+        await deleteObjectsHelp(storageCfg, aKeys.slice(i, i + MAX_DELETE_OBJECTS));
+    }
+}
+
+async function deletePath(storageCfg, strPath) {
+    let list = await listObjects(storageCfg, strPath);
+    await deleteObjects(storageCfg, list);
+}
+
+async function getSignedUrlWrapper(ctx, storageCfg, baseUrl, strPath, urlType, optFilename, opt_creationDate) {
+    const storageUrlExpires = storageCfg.fs.urlExpires;
+    let expires = (commonDefines.c_oAscUrlTypes.Session === urlType ? cfgExpSessionAbsolute / 1000 : storageUrlExpires) || 31536000;
+    expires = Math.min(expires, 604800);
+   
+    const userFriendlyName = optFilename ? optFilename.replace(/\//g, "%2f") : path.basename(strPath);
+    const contentDisposition = utils.getContentDisposition(userFriendlyName, null, null);
+
+    const blobClient = getBlobClient(storageCfg, getFilePath(storageCfg, strPath));
+
+    const sasOptions = {
+        permissions: BlobSASPermissions.parse("r"),
+        expiresOn: new Date(Date.now() + expires * 1000),
+        contentDisposition,
+        contentType: mime.getType(strPath)
+    };
+
+    return await blobClient.generateSasUrl(sasOptions);
+}
+
+function needServeStatic() {
+    return false;
+}
+
+module.exports = {
+    headObject,
+    getObject,
+    createReadStream,
+    putObject,
+    uploadObject,
+    copyObject,
+    listObjects,
+    deleteObject,
+    deletePath,
+    getSignedUrl: getSignedUrlWrapper,
+    needServeStatic
+};

package.json

@@ -78,6 +78,7 @@
     "integration tests with server instance": "cd ./DocService && jest integration/withServerInstance --inject-globals=false --config=../tests/jest.config.js",
     "integration database tests": "cd ./DocService && jest integration/databaseTests --inject-globals=false --config=../tests/jest.config.js",
     "tests": "cd ./DocService && jest --inject-globals=false --config=../tests/jest.config.js",
+    "tests:dev": "cd ./DocService && jest --inject-globals=false --config=../tests/jest.config.js --watch",
     "install:Common": "npm ci --prefix ./Common",
     "install:DocService": "npm ci --prefix ./DocService",
     "install:FileConverter": "npm ci --prefix ./FileConverter",

tests/integration/withServerInstance/forgottenFilesCommnads.tests.js

@@ -50,6 +50,7 @@ const cfgTokenEnableRequestOutbox = config.get('services.CoAuthoring.token.enabl
 const cfgStorageName = config.get('storage.name');
 const cfgEndpoint = config.get('storage.endpoint');
 const cfgBucketName = config.get('storage.bucketName');
+const cfgAccessKeyId = config.get('storage.accessKeyId');
 const ctx = new operationContext.Context();
 
 const testFilesNames = {
@@ -184,12 +185,18 @@ describe('Command service', function () {
         let urlPattern;
         if ("storage-fs" === cfgStorageName) {
           urlPattern = 'http://localhost:8000/cache/files/forgotten/--key--/output.docx/output.docx';
-        } else {
+        } else if ("storage-s3" === cfgStorageName) {
           let host = cfgEndpoint.slice(0, "https://".length) + cfgBucketName + "." + cfgEndpoint.slice("https://".length);
           if (host[host.length - 1] === '/') {
             host = host.slice(0, -1);
           }
           urlPattern = host + '/files/forgotten/--key--/output.docx';
+        } else {
+          let host = cfgEndpoint.slice(0, "https://".length) + cfgAccessKeyId + "." + cfgEndpoint.slice("https://".length) + '/' + cfgBucketName;
+          if (host[host.length - 1] === '/') {
+            host = host.slice(0, -1);
+          }
+          urlPattern = host + '/files/forgotten/--key--/output.docx';
         }
 
         const expected = { key, error };