Forgotten files commands (#415)

* [ds] Add forgotten files comands

* Feature fixes 1

* Feature fixes 2

* [feature] Refactor validateCommand and deleteForgotten

* Feature fixes 3

* Feature fixes 4

* [test] Move jest module

* Feature fixes 5

* [test] Move jest.config.js

---------

Co-authored-by: Sergey Konovalov <Sergey.Konovalov@onlyoffice.com>

tests/env-setup.js

@@ -0,0 +1,15 @@
+const platforms = {
+  'win32': 'windows',
+  'darwin': 'mac',
+  'linux': 'linux'
+};
+const platform = platforms[process.platform];
+
+process.env.NODE_ENV = `development-${platform}`;
+process.env.NODE_CONFIG_DIR = '../Common/config';
+
+if (platform === 'mac') {
+  process.env.DYLD_LIBRARY_PATH = '../FileConverter/bin/';
+} else if (platform === 'linux') {
+  process.env.LD_LIBRARY_PATH = '../FileConverter/bin/';
+}

tests/integration/forgottenFilesCommnads.tests.js

@@ -0,0 +1,246 @@
+const { describe, test, expect, afterAll, beforeAll } = require('@jest/globals');
+const http = require('http');
+
+const { signToken } = require('../../DocService/sources/DocsCoServer');
+const storage = require('../../Common/sources/storage-base');
+const constants = require('../../Common/sources/commondefines');
+const operationContext = require('../../Common/sources/operationContext');
+const utils = require("../../Common/sources/utils");
+const config = require('../../Common/node_modules/config');
+
+const cfgForgottenFiles = config.get('services.CoAuthoring.server.forgottenfiles');
+const cfgForgottenFilesName = config.get('services.CoAuthoring.server.forgottenfilesname');
+const cfgTokenAlgorithm = config.get('services.CoAuthoring.token.session.algorithm');
+const cfgSecretOutbox = config.get('services.CoAuthoring.secret.outbox');
+const cfgTokenOutboxExpires = config.get('services.CoAuthoring.token.outbox.expires');
+const cfgTokenEnableRequestOutbox = config.get('services.CoAuthoring.token.enable.request.outbox');
+const ctx = new operationContext.Context();
+const testFilesNames = {
+  get: 'DocService-DocsCoServer-forgottenFilesCommands-getForgotten-integration-test',
+  delete1: 'DocService-DocsCoServer-forgottenFilesCommands-deleteForgotten-integration-test',
+  // delete2: 'DocService-DocsCoServer-forgottenFilesCommands-deleteForgotten-2-integration-test',
+  // delete3: 'DocService-DocsCoServer-forgottenFilesCommands-deleteForgotten-3-integration-test',
+  getList: 'DocService-DocsCoServer-forgottenFilesCommands-getForgottenList-integration-test'
+};
+
+function makeRequest(requestBody, timeout = 5000) {
+  return new Promise(async (resolve, reject) => {
+    const timer = setTimeout(() => reject('Request timeout'), timeout);
+
+    let body = '';
+    if (cfgTokenEnableRequestOutbox) {
+      const secret = utils.getSecretByElem(cfgSecretOutbox);
+      const token = await signToken(ctx, requestBody, cfgTokenAlgorithm, cfgTokenOutboxExpires, constants.c_oAscSecretType.Inbox, secret);
+      body = JSON.stringify({ token });
+    } else {
+      body = JSON.stringify(requestBody);
+    }
+
+    const options = {
+      port: '8000',
+      path: '/coauthoring/CommandService.ashx',
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(body)
+      }
+    };
+    const request = http.request(options, (response) => {
+      response.setEncoding('utf8');
+
+      let data = '';
+      response.on('data', (chunk) => {
+        data += chunk
+      });
+      response.on('end', () => {
+        resolve(data);
+        clearTimeout(timer);
+      });
+    });
+
+    request.on('error', (error) => {
+      reject(error);
+      clearTimeout(timer);
+    });
+
+    request.write(body);
+    request.end();
+  });
+}
+
+function getKeysDirectories(keys) {
+  return keys.map(value => value.split('/')[0]);
+}
+
+beforeAll(async function () {
+  const buffer = Buffer.from('Forgotten commands test file');
+  for (const index in testFilesNames) {
+    await storage.putObject(ctx, `${testFilesNames[index]}/${cfgForgottenFilesName}.docx`, buffer, buffer.length, cfgForgottenFiles);
+  }
+});
+
+afterAll(async function () {
+  const keys = await storage.listObjects(ctx, '', cfgForgottenFiles);
+  const deletePromises = keys.filter(key => key.includes('DocService-DocsCoServer-forgottenFilesCommands'))
+    .map(filteredKey => storage.deleteObject(ctx, filteredKey, cfgForgottenFiles));
+
+  return Promise.allSettled(deletePromises);
+});
+
+// Assumed, that server is already up.
+describe('Command service', function () {
+  describe('Forgotten files commands parameters validation', function () {
+    describe('Invalid key format', function () {
+      const tests = ['getForgotten', 'deleteForgotten'];
+      const addSpecialCases = (invalidRequests, expected, testSubject) => {
+        invalidRequests.push({
+          c: testSubject
+        });
+        expected.push({ error: 1});
+
+        invalidRequests.push({
+          c: testSubject,
+          key: null
+        });
+        expected.push({
+          key: null,
+          error: 1
+        });
+      };
+
+      for (const testSubject of tests) {
+        test(testSubject, async function () {
+          const invalidKeys = [true, [], {}, 1, 1.1];
+          const invalidRequests = invalidKeys.map(key => {
+            return {
+              c: testSubject,
+              key
+            }
+          });
+
+          const expected = invalidKeys.map(key => {
+            return {
+              key,
+              error: 1,
+            };
+          });
+
+          addSpecialCases(invalidRequests, expected, testSubject);
+
+          for (const index in invalidRequests) {
+            const actualResponse = await makeRequest(invalidRequests[index]);
+            const actual = JSON.parse(actualResponse);
+
+            expect(actual).toEqual(expected[index]);
+          }
+        });
+      }
+    });
+  });
+  
+  describe('Forgotten files commands verification', function () {
+    describe('getForgotten', function () {
+      const createExpected = ({ key, error }) => {
+        const validKey = typeof key === 'string' && error === 0
+        const urlPattern = 'http://localhost:8000/cache/files/forgotten/--key--/output.docx/output.docx';
+
+        const expected = { key, error };
+
+        if (validKey) {
+          expected.url = urlPattern.replace('--key--', key);
+        }
+
+        return expected;
+      };
+
+      const testCases = {
+        'Single key': { key: testFilesNames.get, error: 0 },
+        'Not existed key': { key: '--not-existed--', error: 1 },
+      };
+
+      for (const testCase in testCases) {
+        test(testCase, async () => {
+          const requestBody = {
+            c: 'getForgotten',
+            key: testCases[testCase].key
+          };
+
+          const actualResponse = await makeRequest(requestBody);
+
+          const expected = createExpected(testCases[testCase]);
+          const actual = JSON.parse(actualResponse);
+
+          if (actual.url) {
+            actual.url = actual.url.split('?')[0];
+          }
+
+          expect(actual).toEqual(expected);
+        });
+      }
+    });
+
+    describe('deleteForgotten', function () {
+      const createExpected = ({ key, error }) => {
+        return {
+          key,
+          error
+        };
+      };
+
+      const testCases = {
+        'Single key': { key: testFilesNames.delete1, error: 0 },
+        'Not existed key': { key: '--not-existed--', error: 1 },
+      };
+
+      for (const testCase in testCases) {
+        test(testCase, async () => {
+          const requestBody = {
+            c: 'deleteForgotten',
+            key: testCases[testCase].key
+          };
+
+          const alreadyExistedDirectories = getKeysDirectories(await storage.listObjects(ctx, '', cfgForgottenFiles));
+          const directoryToBeDeleted = testCases[testCase].error !== 0 ? '--not-existed--' : testCases[testCase].key;
+          const shouldExist = alreadyExistedDirectories.filter(directory => directoryToBeDeleted !== directory);
+
+          const actualResponse = await makeRequest(requestBody);
+
+          const expected = createExpected(testCases[testCase]);
+          const actual = JSON.parse(actualResponse);
+
+          const directoriesExistedAfterDeletion = getKeysDirectories(await storage.listObjects(ctx, '', cfgForgottenFiles));
+          expect(actual).toEqual(expected);
+          // Checking that files not existing on disk/cloud.
+          expect(shouldExist).toEqual(directoriesExistedAfterDeletion);
+        });
+      }
+    });
+    
+    describe('getForgottenList', function () {
+      test('Main case', async () => {
+        const requestBody = {
+          c: 'getForgottenList'
+        };
+
+        const stateBeforeChanging = await makeRequest(requestBody);
+        const alreadyExistedDirectories = JSON.parse(stateBeforeChanging);
+
+        const docId = 'DocService-DocsCoServer-forgottenFilesCommands-getForgottenList-2-integration-test';
+        const buffer = Buffer.from('getForgottenList test file');
+        await storage.putObject(ctx, `${docId}/${cfgForgottenFilesName}.docx`, buffer, buffer.length, cfgForgottenFiles);
+        alreadyExistedDirectories.keys.push(docId);
+
+        const actualResponse = await makeRequest(requestBody);
+        const actual = JSON.parse(actualResponse);
+        const expected = {
+          error: 0,
+          keys: alreadyExistedDirectories.keys
+        }
+
+        actual.keys?.sort();
+        expected.keys.sort();
+        expect(actual).toEqual(expected);
+      });
+    });
+  });
+});

tests/jest.config.js

@@ -0,0 +1,192 @@
+/*
+ * For a detailed explanation regarding each configuration property, visit:
+ * https://jestjs.io/docs/configuration
+ */
+
+module.exports = {
+  // All imported modules in your tests should be mocked automatically
+  // automock: false,
+
+  // Stop running tests after `n` failures
+  // bail: 0,
+
+  // The directory where Jest should store its cached dependency information
+  // cacheDirectory: "",
+
+  // Automatically clear mock calls, instances, contexts and results before every test
+  clearMocks: true,
+
+  // Indicates whether the coverage information should be collected while executing the test
+  // collectCoverage: false,
+
+  // An array of glob patterns indicating a set of files for which coverage information should be collected
+  // collectCoverageFrom: undefined,
+
+  // The directory where Jest should output its coverage files
+  // coverageDirectory: undefined,
+
+  // An array of regexp pattern strings used to skip coverage collection
+  // coveragePathIgnorePatterns: [
+  //   "\\\\node_modules\\\\"
+  // ],
+
+  // Indicates which provider should be used to instrument code for coverage
+  coverageProvider: "v8",
+
+  // A list of reporter names that Jest uses when writing coverage reports
+  // coverageReporters: [
+  //   "json",
+  //   "text",
+  //   "lcov",
+  //   "clover"
+  // ],
+
+  // An object that configures minimum threshold enforcement for coverage results
+  // coverageThreshold: undefined,
+
+  // A path to a custom dependency extractor
+  // dependencyExtractor: undefined,
+
+  // Make calling deprecated APIs throw helpful error messages
+  // errorOnDeprecated: false,
+
+  // The default configuration for fake timers
+  // fakeTimers: {
+  //   "enableGlobally": false
+  // },
+
+  // Force coverage collection from ignored files using an array of glob patterns
+  // forceCoverageMatch: [],
+
+  // A path to a module which exports an async function that is triggered once before all test suites
+  // globalSetup: undefined,
+
+  // A path to a module which exports an async function that is triggered once after all test suites
+  // globalTeardown: undefined,
+
+  // A set of global variables that need to be available in all test environments
+  // globals: {},
+
+  // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
+  // maxWorkers: "50%",
+
+  // An array of directory names to be searched recursively up from the requiring module's location
+  // moduleDirectories: [
+  //   "node_modules"
+  // ],
+
+  // An array of file extensions your modules use
+  // moduleFileExtensions: [
+  //   "js",
+  //   "mjs",
+  //   "cjs",
+  //   "jsx",
+  //   "ts",
+  //   "tsx",
+  //   "json",
+  //   "node"
+  // ],
+
+  // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
+  // moduleNameMapper: {},
+
+  // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
+  // modulePathIgnorePatterns: [],
+
+  // Activates notifications for test results
+  // notify: false,
+
+  // An enum that specifies notification mode. Requires { notify: true }
+  // notifyMode: "failure-change",
+
+  // A preset that is used as a base for Jest's configuration
+  // preset: undefined,
+
+  // Run tests from one or more projects
+  // projects: undefined,
+
+  // Use this configuration option to add custom reporters to Jest
+  // reporters: undefined,
+
+  // Automatically reset mock state before every test
+  // resetMocks: false,
+
+  // Reset the module registry before running each individual test
+  // resetModules: false,
+
+  // A path to a custom resolver
+  // resolver: undefined,
+
+  // Automatically restore mock state and implementation before every test
+  // restoreMocks: false,
+
+  // The root directory that Jest should scan for tests and modules within
+  // rootDir: undefined,
+
+  // A list of paths to directories that Jest should use to search for files in
+  // roots: ["<rootDir>"],
+
+  // Allows you to use a custom runner instead of Jest's default test runner
+  // runner: "jest-runner",
+
+  // The paths to modules that run some code to configure or set up the testing environment before each test
+  setupFiles: ['./env-setup.js'],
+
+  // A list of paths to modules that run some code to configure or set up the testing framework before each test
+  // setupFilesAfterEnv: [],
+
+  // The number of seconds after which a test is considered as slow and reported as such in the results.
+  // slowTestThreshold: 5,
+
+  // A list of paths to snapshot serializer modules Jest should use for snapshot testing
+  // snapshotSerializers: [],
+
+  // The test environment that will be used for testing
+  // testEnvironment: "jest-environment-node",
+
+  // Options that will be passed to the testEnvironment
+  // testEnvironmentOptions: {},
+
+  // Adds a location field to test results
+  // testLocationInResults: false,
+
+  // The glob patterns Jest uses to detect test files
+  testMatch: [
+    "**/?(*.)+(spec|tests).[tj]s?(x)"
+  ],
+
+  // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
+  // testPathIgnorePatterns: [
+  //   "\\\\node_modules\\\\"
+  // ],
+
+  // The regexp pattern or array of patterns that Jest uses to detect test files
+  // testRegex: [],
+
+  // This option allows the use of a custom results processor
+  // testResultsProcessor: undefined,
+
+  // This option allows use of a custom test runner
+  // testRunner: "jest-circus/runner",
+
+  // A map from regular expressions to paths to transformers
+  // transform: undefined,
+
+  // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
+  // transformIgnorePatterns: [
+  //   "\\\\node_modules\\\\",
+  //   "\\.pnp\\.[^\\\\]+$"
+  // ],
+
+  // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
+  // unmockedModulePathPatterns: undefined,
+
+  // Indicates whether each individual test should be reported during the run
+  // verbose: undefined,
+
+  // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
+  // watchPathIgnorePatterns: [],
+
+  // Whether to use watchman for file crawling
+  // watchman: true,
+};