diff --git a/.github/workflows/azureStorageTests.yml b/.github/workflows/azureStorageTests.yml new file mode 100644 index 00000000..ad58f41b --- /dev/null +++ b/.github/workflows/azureStorageTests.yml @@ -0,0 +1,110 @@ +name: Azure Storage Tests +on: + push: + branches: + - '**' + paths: + - 'tests/integration/withServerInstance/storage.tests.js' + - 'Common/sources/storage/**' + - 'DocService/sources/routes/static.js' + +jobs: + azure-storage-tests: + name: Azure Storage Tests + runs-on: ubuntu-latest + + steps: + - name: Check out repository code + uses: actions/checkout@v3 + + - name: Run Azurite docker container + run: | + docker run --name azurite \ + -p 10000:10000 \ + -p 10001:10001 \ + -p 10002:10002 \ + -d mcr.microsoft.com/azure-storage/azurite \ + azurite-blob --blobHost 0.0.0.0 --loose + + - name: Caching dependencies + uses: actions/setup-node@v3 + with: + node-version: '20' + cache: 'npm' + cache-dependency-path: | + ./npm-shrinkwrap.json + ./Common/npm-shrinkwrap.json + ./DocService/npm-shrinkwrap.json + + - name: Install modules + run: | + npm ci + npm --prefix Common ci + npm --prefix DocService ci + + - name: Setup Azure storage test environment + run: | + # Wait for Azurite to be ready + sleep 15 + + # Create Azure storage configuration + cat > Common/config/local.json << 'EOF' + { + "storage": { + "name": "storage-az", + "region": "", + "endpoint": "http://127.0.0.1:10000/devstoreaccount1", + "bucketName": "test-container", + "storageFolderName": "files", + "cacheFolderName": "data", + "accessKeyId": "devstoreaccount1", + "secretAccessKey": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + }, + "persistentStorage": { + "storageFolderName": "files/persistent" + } + } + EOF + + echo "Azure storage configuration created" + + - name: Create Azure container using Node.js from Common directory + run: | + # Wait a bit more for Azurite to be fully ready + sleep 10 + + # Run Node.js script from Common directory where Azure dependencies are installed + cd Common + node -e " + const { BlobServiceClient, StorageSharedKeyCredential } = require('@azure/storage-blob'); + + async function setupContainer() { + try { + const accountName = 'devstoreaccount1'; + const accountKey = 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=='; + const endpoint = 'http://127.0.0.1:10000/devstoreaccount1'; + + const credential = new StorageSharedKeyCredential(accountName, accountKey); + const blobServiceClient = new BlobServiceClient(endpoint, credential); + const containerClient = blobServiceClient.getContainerClient('test-container'); + + console.log('Creating container...'); + await containerClient.createIfNotExists(); + console.log('Container created successfully'); + + // Upload a test file if needed + const blockBlobClient = containerClient.getBlockBlobClient('testfile.txt'); + await blockBlobClient.upload('Test content', Buffer.byteLength('Test content')); + console.log('Test file uploaded'); + + } catch (error) { + console.error('Error setting up Azure storage:', error); + process.exit(1); + } + } + + setupContainer(); + " + + - name: Run storage tests + run: npm run storage-tests \ No newline at end of file diff --git a/.github/workflows/fsStorageTests.yml b/.github/workflows/fsStorageTests.yml new file mode 100644 index 00000000..f6d2d054 --- /dev/null +++ b/.github/workflows/fsStorageTests.yml @@ -0,0 +1,64 @@ +name: fs Storage Tests +on: + push: + branches: + - '**' + paths: + - 'tests/integration/withServerInstance/storage.tests.js' + - 'Common/sources/storage/**' + - 'DocService/sources/routes/static.js' + +jobs: + fs-storage-tests: + name: File System Storage + runs-on: ubuntu-latest + + steps: + - name: Check out repository code + uses: actions/checkout@v3 + + - name: Caching dependencies + uses: actions/setup-node@v3 + with: + node-version: '20' + cache: 'npm' + cache-dependency-path: | + ./npm-shrinkwrap.json + ./Common/npm-shrinkwrap.json + ./DocService/npm-shrinkwrap.json + + - name: Install modules + run: | + npm ci + npm --prefix Common ci + npm --prefix DocService ci + + - name: Creating service configuration + run: | + mkdir -p /tmp/storage + mkdir -p Common/config + echo '{ + "storage": { + "name": "storage-fs", + "fs": { + "folderPath": "/tmp/storage", + "urlExpires": 900, + "secretString": "verysecretstring" + }, + "region": "", + "endpoint": "http://localhost/s3", + "bucketName": "cache", + "storageFolderName": "files", + "cacheFolderName": "data", + "urlExpires": 604800, + "accessKeyId": "", + "secretAccessKey": "", + "sslEnabled": false, + "s3ForcePathStyle": true, + "externalHost": "", + "useDirectStorageUrls": true + }, + }' > Common/config/local.json + + - name: Run storage tests + run: npm run storage-tests \ No newline at end of file diff --git a/.github/workflows/s3storageTests.yml b/.github/workflows/s3storageTests.yml new file mode 100644 index 00000000..cb3a6f45 --- /dev/null +++ b/.github/workflows/s3storageTests.yml @@ -0,0 +1,68 @@ +name: s3 Storage Tests +on: + push: + branches: + - '**' + paths: + - 'tests/integration/withServerInstance/storage.tests.js' + - 'Common/sources/storage/**' + - 'DocService/sources/routes/static.js' + +jobs: + storage-tests: + name: Storage Tests + runs-on: ubuntu-latest + + steps: + - name: Run MinIO docker container + run: | + docker run --name minio \ + -p 9000:9000 \ + -p 9001:9001 \ + -e "MINIO_ROOT_USER=minioadmin" \ + -e "MINIO_ROOT_PASSWORD=minioadmin" \ + -d minio/minio server /data --console-address ":9001" + + - name: Check out repository code + uses: actions/checkout@v3 + + - name: Caching dependencies + uses: actions/setup-node@v3 + with: + node-version: '20' + cache: 'npm' + cache-dependency-path: | + ./npm-shrinkwrap.json + ./Common/npm-shrinkwrap.json + ./DocService/npm-shrinkwrap.json + + - name: Install modules + run: | + npm ci + npm --prefix Common ci + npm --prefix DocService ci + + - name: Creating storage configuration + run: | + echo '{ + "storage": { + "name": "storage-s3", + "region": "us-east-1", + "endpoint": "http://localhost:9000", + "accessKeyId": "minioadmin", + "secretAccessKey": "minioadmin", + "bucket": "cache", + "forcePathStyle": true + }, + "persistentStorage": { + "storageFolderName": "files/persistent" + } + }' >> Common/config/local.json + + - name: Create MinIO buckets + run: | + docker exec minio mc alias set myminio http://localhost:9000 minioadmin minioadmin + docker exec minio mc mb myminio/cache + + - name: Run storage tests + run: npm run storage-tests \ No newline at end of file diff --git a/Common/sources/storage/storage-base.js b/Common/sources/storage/storage-base.js index 5364e452..f31a2c5a 100644 --- a/Common/sources/storage/storage-base.js +++ b/Common/sources/storage/storage-base.js @@ -141,12 +141,13 @@ async function deletePath(ctx, strPath, opt_specialDir) { let storageCfg = getStorageCfg(ctx, opt_specialDir); return await storage.deletePath(storageCfg, getStoragePath(ctx, strPath, opt_specialDir)); } -async function getSignedUrl(ctx, baseUrl, strPath, urlType, optFilename, opt_creationDate, opt_specialDir) { +async function getSignedUrl(ctx, baseUrl, strPath, urlType, optFilename, opt_creationDate, opt_specialDir, useDirectStorageUrls) { let storage = getStorage(opt_specialDir); let storageCfg = getStorageCfg(ctx, opt_specialDir); let storagePath = getStoragePath(ctx, strPath, opt_specialDir); + const directUrlsEnabled = useDirectStorageUrls ?? storageCfg.useDirectStorageUrls; - if (storageCfg.useDirectStorageUrls && storage.getDirectSignedUrl) { + if (directUrlsEnabled && storage.getDirectSignedUrl) { return await storage.getDirectSignedUrl(ctx, storageCfg, baseUrl, storagePath, urlType, optFilename, opt_creationDate); } else { const storageSecretString = storageCfg.fs.secretString; diff --git a/DocService/sources/routes/static.js b/DocService/sources/routes/static.js index 9f2f694b..5e1ecb82 100644 --- a/DocService/sources/routes/static.js +++ b/DocService/sources/routes/static.js @@ -36,6 +36,7 @@ const { pipeline } = require('node:stream/promises'); const express = require('express'); const config = require('config'); const operationContext = require('./../../../Common/sources/operationContext'); +const tenantManager = require('./../../../Common/sources/tenantManager'); const utils = require('./../../../Common/sources/utils'); const storage = require('./../../../Common/sources/storage/storage-base'); const urlModule = require("url"); @@ -108,7 +109,7 @@ function createCacheMiddleware(prefix, rootPath, cfgStorage, secret, rout) { } const filename = urlParsed.pathname && decodeURIComponent(path.basename(urlParsed.pathname)); - const filePath = decodeURI(req.url.substring(1, index)); + let filePath = decodeURI(req.url.substring(1, index)); if (cfgStorage.name === 'storage-fs') { const sendFileOptions = { root: rootPath, @@ -129,6 +130,9 @@ function createCacheMiddleware(prefix, rootPath, cfgStorage, secret, rout) { const ctx = new operationContext.Context(); ctx.initFromRequest(req); await ctx.initTenantCache(); + if (tenantManager.isMultitenantMode(ctx) && filePath.startsWith(ctx.tenant + '/')) { + filePath = filePath.substring(ctx.tenant.length + 1); + } const result = await storage.createReadStream(ctx, filePath, rout); res.setHeader('Content-Type', mime.getType(filename)); diff --git a/package.json b/package.json index 32618fff..0a1ed89e 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "perf-png": "cd ./DocService&& cross-env NODE_ENV=development-windows NODE_CONFIG_DIR=../Common/config node ../tests/perf/convertImageToPng.js", "unit tests": "cd ./DocService && jest unit --inject-globals=false --config=../tests/jest.config.js", "integration tests with server instance": "cd ./DocService && jest integration/withServerInstance --inject-globals=false --config=../tests/jest.config.js", + "storage-tests": "cd ./DocService && jest integration/withServerInstance/storage.tests.js --inject-globals=false --config=../tests/jest.config.js", "integration database tests": "cd ./DocService && jest integration/databaseTests --inject-globals=false --config=../tests/jest.config.js", "tests": "cd ./DocService && jest --inject-globals=false --config=../tests/jest.config.js", "tests:dev": "cd ./DocService && jest --inject-globals=false --config=../tests/jest.config.js --watch", diff --git a/tests/integration/withServerInstance/storage.tests.js b/tests/integration/withServerInstance/storage.tests.js index 94351381..04ba5dbc 100644 --- a/tests/integration/withServerInstance/storage.tests.js +++ b/tests/integration/withServerInstance/storage.tests.js @@ -30,7 +30,20 @@ * */ -const {jest, describe, test, expect} = require('@jest/globals'); +const {jest, describe, test, expect, beforeAll, afterAll} = require('@jest/globals'); +jest.mock("fs/promises", () => ({ + ...jest.requireActual('fs/promises'), + cp: jest.fn().mockImplementation((from, to) => fs.writeFileSync(to, testFileData3)) +})); +const mockNeedServeStatic = jest.fn().mockReturnValue(true); +jest.mock('../../../Common/sources/storage/storage-base', () => { + const originalModule = jest.requireActual('../../../Common/sources/storage/storage-base'); + return { + ...originalModule, + needServeStatic: mockNeedServeStatic + }; +}); +const { cp } = require('fs/promises'); const http = require('http'); const https = require('https'); const fs = require('fs'); @@ -41,26 +54,23 @@ let testFileData2 = "test22"; let testFileData3 = "test333"; let testFileData4 = testFileData3; -jest.mock("fs/promises", () => ({ - ...jest.requireActual('fs/promises'), - cp: jest.fn().mockImplementation((from, to) => fs.writeFileSync(to, testFileData3)) -})); -const { cp } = require('fs/promises'); - +const express = require('express'); const operationContext = require('../../../Common/sources/operationContext'); const tenantManager = require('../../../Common/sources/tenantManager'); const storage = require('../../../Common/sources/storage/storage-base'); const utils = require('../../../Common/sources/utils'); const commonDefines = require("../../../Common/sources/commondefines"); const config = require('../../../Common/node_modules/config'); +const staticRouter = require('../../../DocService/sources/routes/static'); const cfgCacheStorage = config.get('storage'); const cfgPersistentStorage = utils.deepMergeObjects({}, cfgCacheStorage, config.get('persistentStorage')); const ctx = operationContext.global; +const PORT = 3457; const rand = Math.floor(Math.random() * 1000000); const testDir = "DocService-DocsCoServer-storage-" + rand; -const baseUrl = "http://localhost:8000"; +const baseUrl = `http://localhost:${PORT}`; const urlType = commonDefines.c_oAscUrlTypes.Session; let testFile1 = testDir + "/test1.txt"; let testFile2 = testDir + "/test2.txt"; @@ -71,6 +81,23 @@ let specialDirForgotten = "forgotten"; console.debug(`testDir: ${testDir}`) +let server; + +beforeAll(async () => { + //start server to server static files generated by getSignedUrl + const app = express(); + app.use('/', staticRouter); + server = app.listen(PORT, () => { + console.debug('listening on ' + PORT); + }); +}); + +afterAll(async () => { + if (server) { + await new Promise((resolve) => server.close(resolve)); + } +}); + function getStorageCfg(specialDir) { return specialDir ? cfgPersistentStorage : cfgCacheStorage; } @@ -200,10 +227,8 @@ function runTestForDir(ctx, isMultitenantMode, specialDir) { let output, outputText; output = await storage.createReadStream(ctx, testFile1, specialDir); - await utils.sleep(100); expect(output.contentLength).toEqual(testFileData1.length); outputText = await utils.stream2Buffer(output.readStream); - await utils.sleep(100); expect(outputText.toString("utf8")).toEqual(testFileData1); output = await storage.createReadStream(ctx, testFile2, specialDir); @@ -261,6 +286,33 @@ function runTestForDir(ctx, isMultitenantMode, specialDir) { } expect(data.sort()).toEqual([testFileData3, testFileData4].sort()); }); + test("getSignedUrl with direct URLs enabled", async () => { + let buffer = Buffer.from(testFileData1); + let res = await storage.putObject(ctx, testFile1, buffer, buffer.length, specialDirCache); + expect(res).toEqual(undefined); + + let url = await storage.getSignedUrl(ctx, baseUrl, testFile1, urlType, undefined, undefined, specialDirCache, true); + let data = await request(url); + expect(data).toEqual(testFileData1); + + if (cfgCacheStorage.name !== 'storage-fs') { + expect(url).toContain(cfgCacheStorage.endpoint); + expect(url).toContain(cfgCacheStorage.bucketName); + } + }); + test("getSignedUrl with direct URLs disabled", async () => { + let buffer = Buffer.from(testFileData1); + let res = await storage.putObject(ctx, testFile1, buffer, buffer.length, specialDirCache); + expect(res).toEqual(undefined); + + let url = await storage.getSignedUrl(ctx, baseUrl, testFile1, urlType, undefined, undefined, specialDirCache, false); + let data = await request(url); + expect(data).toEqual(testFileData1); + + expect(url).toContain('md5'); + expect(url).toContain('expires'); + expect(url).toContain(cfgCacheStorage.storageFolderName); + }); test("deleteObject", async () => { let list; list = await storage.listObjects(ctx, testDir, specialDir);