diff --git a/.github/workflows/damengDatabaseTests.yml b/.github/workflows/damengDatabaseTests.yml index 90418829..05514918 100644 --- a/.github/workflows/damengDatabaseTests.yml +++ b/.github/workflows/damengDatabaseTests.yml @@ -1,6 +1,8 @@ name: Dameng database tests on: push: + branches: + - '**' paths: - 'tests/integration/databaseTests/**' - 'DocService/sources/databaseConnectors/baseConnector.js' @@ -46,4 +48,4 @@ jobs: docker exec dameng bash -c "cat /createdb.sql | /opt/dmdbms/bin/disql SYSDBA/SYSDBA001:5236" - name: Run Jest - run: npm run "integration database tests" \ No newline at end of file + run: npm run "integration database tests" diff --git a/.github/workflows/mssqlDatabaseTests.yml b/.github/workflows/mssqlDatabaseTests.yml index 81cd7e59..e9f7dab5 100644 --- a/.github/workflows/mssqlDatabaseTests.yml +++ b/.github/workflows/mssqlDatabaseTests.yml @@ -1,6 +1,8 @@ name: MSSQL database tests on: push: + branches: + - '**' paths: - 'tests/integration/databaseTests/**' - 'DocService/sources/databaseConnectors/baseConnector.js' @@ -46,4 +48,4 @@ jobs: docker exec mssql /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P onlYoff1ce -i /createdb.sql - name: Run Jest - run: npm run "integration database tests" \ No newline at end of file + run: npm run "integration database tests" diff --git a/.github/workflows/mysqlDatabaseTests.yml b/.github/workflows/mysqlDatabaseTests.yml index b1557ba4..4edb8da5 100644 --- a/.github/workflows/mysqlDatabaseTests.yml +++ b/.github/workflows/mysqlDatabaseTests.yml @@ -1,6 +1,8 @@ name: MYSQL database tests on: push: + branches: + - '**' paths: - 'tests/integration/databaseTests/**' - 'DocService/sources/databaseConnectors/baseConnector.js' @@ -20,7 +22,7 @@ jobs: - name: Caching dependencies uses: actions/setup-node@v3 with: - node-version: '14' + node-version: '16' cache: 'npm' cache-dependency-path: | ./npm-shrinkwrap.json @@ -43,4 +45,4 @@ jobs: docker exec mysql mysql -h 127.0.0.1 -u root --password=onlyoffice -D onlyoffice -e 'source /createdb.sql' - name: Run Jest - run: npm run "integration database tests" \ No newline at end of file + run: npm run "integration database tests" diff --git a/.github/workflows/oracleDatabaseTests.yml b/.github/workflows/oracleDatabaseTests.yml index 76123a6f..3911a252 100644 --- a/.github/workflows/oracleDatabaseTests.yml +++ b/.github/workflows/oracleDatabaseTests.yml @@ -1,6 +1,8 @@ name: Oracle database tests on: push: + branches: + - '**' paths: - 'tests/integration/databaseTests/**' - 'DocService/sources/databaseConnectors/baseConnector.js' @@ -46,4 +48,4 @@ jobs: docker exec oracle sqlplus -s onlyoffice/onlyoffice@//localhost/xepdb1 @/createdb.sql - name: Run Jest - run: npm run "integration database tests" \ No newline at end of file + run: npm run "integration database tests" diff --git a/.github/workflows/postgreDatabaseTests.yml b/.github/workflows/postgreDatabaseTests.yml index a9484fef..9d5f20be 100644 --- a/.github/workflows/postgreDatabaseTests.yml +++ b/.github/workflows/postgreDatabaseTests.yml @@ -1,6 +1,8 @@ name: Postgre database tests on: push: + branches: + - '**' paths: - 'tests/integration/databaseTests/**' - 'DocService/sources/databaseConnectors/baseConnector.js' @@ -43,4 +45,4 @@ jobs: docker exec postgres psql -d onlyoffice -U onlyoffice -a -f /createdb.sql - name: Run Jest - run: npm run "integration database tests" \ No newline at end of file + run: npm run "integration database tests" diff --git a/.github/workflows/unitTests.yml b/.github/workflows/unitTests.yml index bdef3f3c..e1d0d2b6 100644 --- a/.github/workflows/unitTests.yml +++ b/.github/workflows/unitTests.yml @@ -1,6 +1,8 @@ name: Service unit tests on: push: + branches: + - '**' paths: - '**.js' - '!tests/integration/**' @@ -31,4 +33,4 @@ jobs: npm --prefix DocService ci - name: Run Jest - run: npm run "unit tests" \ No newline at end of file + run: npm run "unit tests" diff --git a/Common/config/default.json b/Common/config/default.json index 73254ad5..170d3868 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -35,6 +35,8 @@ "s3ForcePathStyle": true, "externalHost": "" }, + "persistentStorage": { + }, "rabbitmq": { "url": "amqp://guest:guest@localhost:5672", "socketOptions": {}, @@ -93,25 +95,28 @@ "favIconUrlWord" : "/web-apps/apps/documenteditor/main/resources/img/favicon.ico", "favIconUrlCell" : "/web-apps/apps/spreadsheeteditor/main/resources/img/favicon.ico", "favIconUrlSlide" : "/web-apps/apps/presentationeditor/main/resources/img/favicon.ico", + "favIconUrlPdf" : "/web-apps/apps/pdfeditor/main/resources/img/favicon.ico", "fileInfoBlockList" : ["FileUrl"], - "pdfView": ["pdf", "djvu", "xps", "oxps"], - "wordView": ["doc", "dotx", "dotm", "dot", "fodt", "ott", "rtf", "mht", "mhtml", "html", "htm", "xml", "epub", "fb2", "sxw", "stw", "wps", "wpt"], - "wordEdit": ["docx", "docm", "docxf", "oform", "odt", "txt"], + "pdfView": ["djvu", "xps", "oxps"], + "pdfEdit": ["pdf"], + "forms": ["pdf"], + "wordView": ["doc", "dotx", "dotm", "dot", "fodt", "ott", "rtf", "mht", "mhtml", "html", "htm", "xml", "epub", "fb2", "sxw", "stw", "wps", "wpt", "docxf", "oform"], + "wordEdit": ["docx", "docm", "odt", "txt"], "cellView": ["xls", "xlsb", "xltx", "xltm", "xlt", "fods", "ots", "sxc", "xml", "et", "ett"], "cellEdit": ["xlsx", "xlsm", "ods", "csv"], "slideView": ["ppt", "ppsx", "ppsm", "pps", "potx", "potm", "pot", "fodp", "otp", "sxi", "dps", "dpt"], "slideEdit": ["pptx", "pptm", "odp"], - "publicKey": "BgIAAACkAABSU0ExAAgAAAEAAQD/NVqekFNi8X3p6Bvdlaxm0GGuggW5kKfVEQzPGuOkGVrz6DrOMNR+k7Pq8tONY+1NHgS6Z+v3959em78qclVDuQX77Tkml0xMHAQHN4sAHF9iQJS8gOBUKSVKaHD7Z8YXch6F212YSUSc8QphpDSHWVShU7rcUeLQsd/0pkflh5+um4YKEZhm4Mou3vstp5p12NeffyK1WFZF7q4jB7jclAslYKQsP82YY3DcRwu5Tl/+W0ifVcXze0mI7v1reJ12pKn8ifRiq+0q5oJST3TRSrvmjLg9Gt3ozhVIt2HUi3La7Qh40YOAUXm0g/hUq2BepeOp1C7WSvaOFHXe6Hqq", - "modulus": "qnro3nUUjvZK1i7UqeOlXmCrVPiDtHlRgIPReAjt2nKL1GG3SBXO6N0aPbiM5rtK0XRPUoLmKu2rYvSJ/Kmkdp14a/3uiEl788VVn0hb/l9OuQtH3HBjmM0/LKRgJQuU3LgHI67uRVZYtSJ/n9fYdZqnLfveLsrgZpgRCoabrp+H5Uem9N+x0OJR3LpToVRZhzSkYQrxnERJmF3bhR5yF8Zn+3BoSiUpVOCAvJRAYl8cAIs3BwQcTEyXJjnt+wW5Q1VyKr+bXp/39+tnugQeTe1jjdPy6rOTftQwzjro81oZpOMazwwR1aeQuQWCrmHQZqyV3Rvo6X3xYlOQnlo1/w==", - "exponent": "AQAB", - "privateKey": "MIIEowIBAAKCAQEAqnro3nUUjvZK1i7UqeOlXmCrVPiDtHlRgIPReAjt2nKL1GG3SBXO6N0aPbiM5rtK0XRPUoLmKu2rYvSJ/Kmkdp14a/3uiEl788VVn0hb/l9OuQtH3HBjmM0/LKRgJQuU3LgHI67uRVZYtSJ/n9fYdZqnLfveLsrgZpgRCoabrp+H5Uem9N+x0OJR3LpToVRZhzSkYQrxnERJmF3bhR5yF8Zn+3BoSiUpVOCAvJRAYl8cAIs3BwQcTEyXJjnt+wW5Q1VyKr+bXp/39+tnugQeTe1jjdPy6rOTftQwzjro81oZpOMazwwR1aeQuQWCrmHQZqyV3Rvo6X3xYlOQnlo1/wIDAQABAoIBAQCKtUSBs8tNYrGTQTlBHXrwpkDg+u7WSZt5sEcfnkxA39BLtlHU8gGO0E9Ihr8GAL+oWjUsEltJ9GTtN8CJ9lFdPVS8sTiCZR/YQOggmFRZTJyVzMrkXgF7Uwwiu3+KxLiTOZx9eRhfDBlTD8W9fXaegX2i2Xp2ohUhBHthEBLdaZTWFi5Sid/Y0dDzBeP6UIJorZ5D+1ybaeIVHjndpwNsIEUGUxPFLrkeiU8Rm4MJ9ahxfywcP7DjQoPGY9Ge5cBhpxfzERWf732wUD6o3+L9tvOBU00CLVjULbGZKTVE2FJMyXK9jr6Zor9Mkhomp6/8Agkr9rp+TPyelFGYEz8hAoGBAOEc09CrL3eYBkhNEcaMQzxBLvOGpg8kaDX5SaArHfl9+U9yzRqss4ARECanp9HuHfjMQo7iejao0ngDjL7BNMSaH74QlSsPOY2iOm8Qvx8/zb7g4h9r1zLjFZb3mpSA4snRZvvdiZ9ugbuVPmhXnDzRRMg45MibJeeOTJNylofRAoGBAMHfF/WutqKDoX25qZo9m74W4bttOj6oIDk1N4/c6M1Z1v/aptYSE06bkWngj9P46kqjaay4hgMtzyGruc5aojPx5MHHf5bo14+Jv4NzYtR2llrUxO+UJX7aCfUYXI7RC93GUmhpeQ414j7SNAXec58d7e+ETw+6cHiAWO4uOSTPAoGATPq5qDLR4Zi4FUNdn8LZPyKfNqHF6YmupT5hIgd8kZO1jKiaYNPL8jBjkIRmjBBcaXcYD5p85nImvumf2J9jNxPpZOpwyC/Fo5xlVROp97qu1eY7DTmodntXJ6/2SXAlnZQhHmHsrPtyG752f+HtyJJbbgiem8cKWDu+DfHybfECgYBbSLo1WiBwgN4nHqZ3E48jgA6le5azLeKOTTpuKKwNFMIhEkj//t7MYn+jhLL0Mf3PSwZU50Vidc1To1IHkbFSGBGIFHFFEzl8QnXEZS4hr/y3o/teezj0c6HAn8nlDRUzRVBEDXWMdV6kCcGpCccTIrqHzpqTY0vV0UkOTQFnDQKBgAxSEhm/gtCYJIMCBe+KBJT9uECV5xDQopTTjsGOkd4306EN2dyPOIlAfwM6K/0qWisa0Ei5i8TbRRuBeTTdLEYLqXCJ7fj5tdD1begBdSVtHQ2WHqzPJSuImTkFi9NXxd1XUyZFM3y6YQvlssSuL7QSxUIEtZHnrJTt3QDd10dj", - "publicKeyOld": "BgIAAACkAABSU0ExAAgAAAEAAQD/NVqekFNi8X3p6Bvdlaxm0GGuggW5kKfVEQzPGuOkGVrz6DrOMNR+k7Pq8tONY+1NHgS6Z+v3959em78qclVDuQX77Tkml0xMHAQHN4sAHF9iQJS8gOBUKSVKaHD7Z8YXch6F212YSUSc8QphpDSHWVShU7rcUeLQsd/0pkflh5+um4YKEZhm4Mou3vstp5p12NeffyK1WFZF7q4jB7jclAslYKQsP82YY3DcRwu5Tl/+W0ifVcXze0mI7v1reJ12pKn8ifRiq+0q5oJST3TRSrvmjLg9Gt3ozhVIt2HUi3La7Qh40YOAUXm0g/hUq2BepeOp1C7WSvaOFHXe6Hqq", - "modulusOld": "qnro3nUUjvZK1i7UqeOlXmCrVPiDtHlRgIPReAjt2nKL1GG3SBXO6N0aPbiM5rtK0XRPUoLmKu2rYvSJ/Kmkdp14a/3uiEl788VVn0hb/l9OuQtH3HBjmM0/LKRgJQuU3LgHI67uRVZYtSJ/n9fYdZqnLfveLsrgZpgRCoabrp+H5Uem9N+x0OJR3LpToVRZhzSkYQrxnERJmF3bhR5yF8Zn+3BoSiUpVOCAvJRAYl8cAIs3BwQcTEyXJjnt+wW5Q1VyKr+bXp/39+tnugQeTe1jjdPy6rOTftQwzjro81oZpOMazwwR1aeQuQWCrmHQZqyV3Rvo6X3xYlOQnlo1/w==", - "exponentOld": "AQAB", - "privateKeyOld": "MIIEowIBAAKCAQEAqnro3nUUjvZK1i7UqeOlXmCrVPiDtHlRgIPReAjt2nKL1GG3SBXO6N0aPbiM5rtK0XRPUoLmKu2rYvSJ/Kmkdp14a/3uiEl788VVn0hb/l9OuQtH3HBjmM0/LKRgJQuU3LgHI67uRVZYtSJ/n9fYdZqnLfveLsrgZpgRCoabrp+H5Uem9N+x0OJR3LpToVRZhzSkYQrxnERJmF3bhR5yF8Zn+3BoSiUpVOCAvJRAYl8cAIs3BwQcTEyXJjnt+wW5Q1VyKr+bXp/39+tnugQeTe1jjdPy6rOTftQwzjro81oZpOMazwwR1aeQuQWCrmHQZqyV3Rvo6X3xYlOQnlo1/wIDAQABAoIBAQCKtUSBs8tNYrGTQTlBHXrwpkDg+u7WSZt5sEcfnkxA39BLtlHU8gGO0E9Ihr8GAL+oWjUsEltJ9GTtN8CJ9lFdPVS8sTiCZR/YQOggmFRZTJyVzMrkXgF7Uwwiu3+KxLiTOZx9eRhfDBlTD8W9fXaegX2i2Xp2ohUhBHthEBLdaZTWFi5Sid/Y0dDzBeP6UIJorZ5D+1ybaeIVHjndpwNsIEUGUxPFLrkeiU8Rm4MJ9ahxfywcP7DjQoPGY9Ge5cBhpxfzERWf732wUD6o3+L9tvOBU00CLVjULbGZKTVE2FJMyXK9jr6Zor9Mkhomp6/8Agkr9rp+TPyelFGYEz8hAoGBAOEc09CrL3eYBkhNEcaMQzxBLvOGpg8kaDX5SaArHfl9+U9yzRqss4ARECanp9HuHfjMQo7iejao0ngDjL7BNMSaH74QlSsPOY2iOm8Qvx8/zb7g4h9r1zLjFZb3mpSA4snRZvvdiZ9ugbuVPmhXnDzRRMg45MibJeeOTJNylofRAoGBAMHfF/WutqKDoX25qZo9m74W4bttOj6oIDk1N4/c6M1Z1v/aptYSE06bkWngj9P46kqjaay4hgMtzyGruc5aojPx5MHHf5bo14+Jv4NzYtR2llrUxO+UJX7aCfUYXI7RC93GUmhpeQ414j7SNAXec58d7e+ETw+6cHiAWO4uOSTPAoGATPq5qDLR4Zi4FUNdn8LZPyKfNqHF6YmupT5hIgd8kZO1jKiaYNPL8jBjkIRmjBBcaXcYD5p85nImvumf2J9jNxPpZOpwyC/Fo5xlVROp97qu1eY7DTmodntXJ6/2SXAlnZQhHmHsrPtyG752f+HtyJJbbgiem8cKWDu+DfHybfECgYBbSLo1WiBwgN4nHqZ3E48jgA6le5azLeKOTTpuKKwNFMIhEkj//t7MYn+jhLL0Mf3PSwZU50Vidc1To1IHkbFSGBGIFHFFEzl8QnXEZS4hr/y3o/teezj0c6HAn8nlDRUzRVBEDXWMdV6kCcGpCccTIrqHzpqTY0vV0UkOTQFnDQKBgAxSEhm/gtCYJIMCBe+KBJT9uECV5xDQopTTjsGOkd4306EN2dyPOIlAfwM6K/0qWisa0Ei5i8TbRRuBeTTdLEYLqXCJ7fj5tdD1begBdSVtHQ2WHqzPJSuImTkFi9NXxd1XUyZFM3y6YQvlssSuL7QSxUIEtZHnrJTt3QDd10dj", + "publicKey": "BgIAAACkAABSU0ExAAgAAAEAAQBpTpiJQ2hD8plpGTfEEmcq4IKyr31HikXpuVSBraMfqyodn2PGXBJ3daNSmdPOc0Nz4HO9Auljn8YYXDPBdpiABptSKvEDPF23Q+Qytg0+vCRyondyBcW91w7KLzXce3fnk8ZfJ8QtbZPL9m11wJIWZueQF+l0HKYx4lty+nccbCanytFTADkGQ3SnmExGEF3rBz6I9+OcrDDK9NKPJgEmCiuyei/d4XbPgKls3EIG0h38X5mVF2VytfWm2Yu850B6z3N4MYhj4b4vsYT62zEC4pMRUeb8dIBy4Jsmr3avtmeO00MUH6DVyPC8nirixj2YIOPKk13CdVqGDSXA3cvl", + "modulus": "E5CBDDC0250D865A75C25D93CAE320983DC6E22A9EBCF0C8D5A01F1443D38E67B6AF76AF269BE0728074FCE6511193E20231DBFA84B12FBEE16388317873CF7A40E7BC8BD9A6F5B572651795995FFC1DD20642DC6CA980CF76E1DD2F7AB22B0A2601268FD2F4CA30AC9CE3F7883E07EB5D10464C98A7744306390053D1CAA7266C1C77FA725BE231A61C74E91790E7661692C0756DF6CB936D2DC4275FC693E7777BDC352FCA0ED7BDC5057277A27224BC3E0DB632E443B75D3C03F12A529B06809876C1335C18C69F63E902BD73E0734373CED39952A37577125CC6639F1D2AAB1FA3AD8154B9E9458A477DAFB282E02A6712C437196999F243684389984E69", + "exponent": "65537", + "privateKey": "LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tDQpNSUlFdndJQkFEQU5CZ2txaGtpRzl3MEJBUUVGQUFTQ0JLa3dnZ1NsQWdFQUFvSUJBUURseTkzQUpRMkdXblhDDQpYWlBLNHlDWVBjYmlLcDY4OE1qVm9COFVROU9PWjdhdmRxOG1tK0J5Z0hUODVsRVJrK0lDTWR2NmhMRXZ2dUZqDQppREY0Yzg5NlFPZThpOW1tOWJWeVpSZVZtVi84SGRJR1F0eHNxWURQZHVIZEwzcXlLd29tQVNhUDB2VEtNS3ljDQo0L2VJUGdmclhSQkdUSmluZEVNR09RQlQwY3FuSm13Y2QvcHlXK0l4cGh4MDZSZVE1MllXa3NCMWJmYkxrMjB0DQp4Q2RmeHBQbmQzdmNOUy9LRHRlOXhRVnlkNkp5Skx3K0RiWXk1RU8zWFR3RDhTcFNtd2FBbUhiQk0xd1l4cDlqDQo2UUs5YytCelEzUE8wNWxTbzNWM0VsekdZNThkS3FzZm82MkJWTG5wUllwSGZhK3lndUFxWnhMRU54bHBtZkpEDQphRU9KbUU1cEFnTUJBQUVDZ2dFQUxpTCtSS09yMFh1OEJPZ1EwajFEd0EwM0x4VnJoWGU2ZXRtSkkrSnlTVGNkDQpnS0VOald6aVpWclJJaTJEdlVtNXFNTWw3V2hTd3NsS0sxZWV4eFpKWTd4QVNxU3hjRW9Jd2d6MTdUMDcvanhtDQpmSWRVQmlVS0RaMUt2OFBXbUlyM29LVytma1hXaS9tMXpsSWUwcVhScFRtc0dORXNIUUxFcWkwcm1haVhUWE9SDQovMkxkd2k2a1pSM3NXRng5N1lTNE14L3B1ZUdKVFhFYWk2QVZFWnpONUdvZzZ4RDhIWFIxUnZxK2hoZCtNb2NHDQpmblU0SGdpbEtSZm9KbFdkOUZPc2NnU3VmS0cwTDNWaU80ZlNLVTQ2bDVhdWxsRFlVazVFQ01XaXd1S1NxU0U3DQpxRDQ1akkzbWJPcmU3UzR1M1MzVFdkRDNsendpWEw0OUxkd0tsRUM0bVFLQmdRRDBzTHIwR0g0V3IrUVgyeEpFDQp1QS9DYjhRVzQxbDhpU0NCVFJaWlIvc0pPZCtvM3JiY1ZpZGx6Ty9FYlpibFhHNFpQRG1SamdCQ0dLSVA1RVppDQowRHNMK1d2MzJXT280NExweEpHaHFFeGJtMEgxaVoxelo5N2wwUDhmdkloSEU0MmdtYUxUb09JR0RoUFNYR3Z2DQp6bHFPSGJHYnE0anNFUmMxanAxYmVqNXE2d0tCZ1FEd2F1ZUljNHBSY2hIOThRWWlkY3lyOFZ3ZzlLaGJuZllYDQp5M1c0UlBsWnRCZEYzNGlKYWlvK0FTenVnby96eTFSVGNWcnNDc2tZV1h5S0RVUXoxeXUwaUNuZytmRENVblRtDQpYR21Fb0VHTmhrNHZUSk90N2hCYXYxL0phL2RVaXBHZjZtWFV1YW53SjBlKzEvRXQvQjBhaDVYMVVtNUF5TlpJDQpNK1N5UnozdSt3S0JnUUNqdnRVTlhvcWFnaENCQ21CNlRqWjFwcmV4bldrWUZ1Z0N2MlNTVU1JazFXN2dJbEo2DQp0c2pjcmoxUjFRaWk2cXpmQkZkK0dXb0EwVjA2aDBlMi9xUlZDZy8vcDZHeXRyVzMzSXljZ3ZTK1pQTEo3dExJDQpGUjJyNjZXZlJscG9QaVNMOGVSdC9QN2trRzBoWENuN0s3dWIyVEV1L0thL1cxeU53YWQ2UFI4aUN3S0JnUUM4DQpYY1pTcnRRc3hBYzh3OTllbUpWb0VvOXdjc0NHSjlsdEEwaVV1OVh5WnB2bGJ5SjNKK3M0OFlyV3hRMHNvcDdMDQpVZ0UrOTZSZm81MWtQTWkzSlZ0azgxcDhudGY0S01yV3dva2FGTVhIc1BjSk1DSjFJQlZJUkxFMEM1ZVpjWWh2DQpseU41N0k0dFQxbHpPWllKeFlLNENvdC96cm43b0YvajZtVEJHZmg0aVFLQmdRQ2lKTVV4UnowMS9jekgvWFNYDQpnbzNkVmJIUTRGRU91ZlduRTNFYjkzUzhyMC9lcTFSTTExOHJiMFRxenVpYWRXMnhZRFU0bnVjV1FscmxtcTBkDQpGWS9tK0h5OTdwcXlrNmptb1U1SS9EK3NzQklvWUhXTG5IOS94ZnZERWsySkdTSlNIdHp1MEQ0RURDL3JnUTQ5DQpNYllzTzVvVXJGOHRQbGhqNXZ6YmYzR0tMQT09DQotLS0tLUVORCBQUklWQVRFIEtFWS0tLS0tDQo=", + "publicKeyOld": "BgIAAACkAABSU0ExAAgAAAEAAQBpTpiJQ2hD8plpGTfEEmcq4IKyr31HikXpuVSBraMfqyodn2PGXBJ3daNSmdPOc0Nz4HO9Auljn8YYXDPBdpiABptSKvEDPF23Q+Qytg0+vCRyondyBcW91w7KLzXce3fnk8ZfJ8QtbZPL9m11wJIWZueQF+l0HKYx4lty+nccbCanytFTADkGQ3SnmExGEF3rBz6I9+OcrDDK9NKPJgEmCiuyei/d4XbPgKls3EIG0h38X5mVF2VytfWm2Yu850B6z3N4MYhj4b4vsYT62zEC4pMRUeb8dIBy4Jsmr3avtmeO00MUH6DVyPC8nirixj2YIOPKk13CdVqGDSXA3cvl", + "modulusOld": "E5CBDDC0250D865A75C25D93CAE320983DC6E22A9EBCF0C8D5A01F1443D38E67B6AF76AF269BE0728074FCE6511193E20231DBFA84B12FBEE16388317873CF7A40E7BC8BD9A6F5B572651795995FFC1DD20642DC6CA980CF76E1DD2F7AB22B0A2601268FD2F4CA30AC9CE3F7883E07EB5D10464C98A7744306390053D1CAA7266C1C77FA725BE231A61C74E91790E7661692C0756DF6CB936D2DC4275FC693E7777BDC352FCA0ED7BDC5057277A27224BC3E0DB632E443B75D3C03F12A529B06809876C1335C18C69F63E902BD73E0734373CED39952A37577125CC6639F1D2AAB1FA3AD8154B9E9458A477DAFB282E02A6712C437196999F243684389984E69", + "exponentOld": "65537", + "privateKeyOld": "LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tDQpNSUlFdndJQkFEQU5CZ2txaGtpRzl3MEJBUUVGQUFTQ0JLa3dnZ1NsQWdFQUFvSUJBUURseTkzQUpRMkdXblhDDQpYWlBLNHlDWVBjYmlLcDY4OE1qVm9COFVROU9PWjdhdmRxOG1tK0J5Z0hUODVsRVJrK0lDTWR2NmhMRXZ2dUZqDQppREY0Yzg5NlFPZThpOW1tOWJWeVpSZVZtVi84SGRJR1F0eHNxWURQZHVIZEwzcXlLd29tQVNhUDB2VEtNS3ljDQo0L2VJUGdmclhSQkdUSmluZEVNR09RQlQwY3FuSm13Y2QvcHlXK0l4cGh4MDZSZVE1MllXa3NCMWJmYkxrMjB0DQp4Q2RmeHBQbmQzdmNOUy9LRHRlOXhRVnlkNkp5Skx3K0RiWXk1RU8zWFR3RDhTcFNtd2FBbUhiQk0xd1l4cDlqDQo2UUs5YytCelEzUE8wNWxTbzNWM0VsekdZNThkS3FzZm82MkJWTG5wUllwSGZhK3lndUFxWnhMRU54bHBtZkpEDQphRU9KbUU1cEFnTUJBQUVDZ2dFQUxpTCtSS09yMFh1OEJPZ1EwajFEd0EwM0x4VnJoWGU2ZXRtSkkrSnlTVGNkDQpnS0VOald6aVpWclJJaTJEdlVtNXFNTWw3V2hTd3NsS0sxZWV4eFpKWTd4QVNxU3hjRW9Jd2d6MTdUMDcvanhtDQpmSWRVQmlVS0RaMUt2OFBXbUlyM29LVytma1hXaS9tMXpsSWUwcVhScFRtc0dORXNIUUxFcWkwcm1haVhUWE9SDQovMkxkd2k2a1pSM3NXRng5N1lTNE14L3B1ZUdKVFhFYWk2QVZFWnpONUdvZzZ4RDhIWFIxUnZxK2hoZCtNb2NHDQpmblU0SGdpbEtSZm9KbFdkOUZPc2NnU3VmS0cwTDNWaU80ZlNLVTQ2bDVhdWxsRFlVazVFQ01XaXd1S1NxU0U3DQpxRDQ1akkzbWJPcmU3UzR1M1MzVFdkRDNsendpWEw0OUxkd0tsRUM0bVFLQmdRRDBzTHIwR0g0V3IrUVgyeEpFDQp1QS9DYjhRVzQxbDhpU0NCVFJaWlIvc0pPZCtvM3JiY1ZpZGx6Ty9FYlpibFhHNFpQRG1SamdCQ0dLSVA1RVppDQowRHNMK1d2MzJXT280NExweEpHaHFFeGJtMEgxaVoxelo5N2wwUDhmdkloSEU0MmdtYUxUb09JR0RoUFNYR3Z2DQp6bHFPSGJHYnE0anNFUmMxanAxYmVqNXE2d0tCZ1FEd2F1ZUljNHBSY2hIOThRWWlkY3lyOFZ3ZzlLaGJuZllYDQp5M1c0UlBsWnRCZEYzNGlKYWlvK0FTenVnby96eTFSVGNWcnNDc2tZV1h5S0RVUXoxeXUwaUNuZytmRENVblRtDQpYR21Fb0VHTmhrNHZUSk90N2hCYXYxL0phL2RVaXBHZjZtWFV1YW53SjBlKzEvRXQvQjBhaDVYMVVtNUF5TlpJDQpNK1N5UnozdSt3S0JnUUNqdnRVTlhvcWFnaENCQ21CNlRqWjFwcmV4bldrWUZ1Z0N2MlNTVU1JazFXN2dJbEo2DQp0c2pjcmoxUjFRaWk2cXpmQkZkK0dXb0EwVjA2aDBlMi9xUlZDZy8vcDZHeXRyVzMzSXljZ3ZTK1pQTEo3dExJDQpGUjJyNjZXZlJscG9QaVNMOGVSdC9QN2trRzBoWENuN0s3dWIyVEV1L0thL1cxeU53YWQ2UFI4aUN3S0JnUUM4DQpYY1pTcnRRc3hBYzh3OTllbUpWb0VvOXdjc0NHSjlsdEEwaVV1OVh5WnB2bGJ5SjNKK3M0OFlyV3hRMHNvcDdMDQpVZ0UrOTZSZm81MWtQTWkzSlZ0azgxcDhudGY0S01yV3dva2FGTVhIc1BjSk1DSjFJQlZJUkxFMEM1ZVpjWWh2DQpseU41N0k0dFQxbHpPWllKeFlLNENvdC96cm43b0YvajZtVEJHZmg0aVFLQmdRQ2lKTVV4UnowMS9jekgvWFNYDQpnbzNkVmJIUTRGRU91ZlduRTNFYjkzUzhyMC9lcTFSTTExOHJiMFRxenVpYWRXMnhZRFU0bnVjV1FscmxtcTBkDQpGWS9tK0h5OTdwcXlrNmptb1U1SS9EK3NzQklvWUhXTG5IOS94ZnZERWsySkdTSlNIdHp1MEQ0RURDL3JnUTQ5DQpNYllzTzVvVXJGOHRQbGhqNXZ6YmYzR0tMQT09DQotLS0tLUVORCBQUklWQVRFIEtFWS0tLS0tDQo=", "refreshLockInterval": "10m", "dummy" : { - "enable": true, + "enable": false, "sampleFilePath": "" } }, @@ -137,7 +142,7 @@ "allow": true, "blockPrivateIP": true, "proxyUrl": "", - "proxyAuth": { + "proxyUser": { "username": "", "password": "" }, @@ -170,11 +175,11 @@ "openProtectedFile": true, "isAnonymousSupport": true, "editorDataStorage": "editorDataMemory", + "editorStatStorage": "", "assemblyFormatAsOrigin": true, "newFileTemplate" : "../../document-templates/new", "downloadFileAllowExt": ["pdf", "xlsx"], "tokenRequiredParams": true, - "allowPrivateIPAddressForSignedRequests": true, "forceSaveUsingButtonWithoutChanges": false }, "requestDefaults": { @@ -235,7 +240,8 @@ "pool": { "idleTimeoutMillis": 30000 } - } + }, + "mysqlExtraOptions": {} }, "redis": { "name": "redis", @@ -244,10 +250,14 @@ "port": 6379, "options": {}, "optionsCluster": {}, - "iooptions": {}, + "iooptions": { + "lazyConnect": true + }, "iooptionsClusterNodes": [ ], - "iooptionsClusterOptions": {} + "iooptionsClusterOptions": { + "lazyConnect": true + } }, "pubsub": { "maxChanges": 1000 diff --git a/Common/sources/commondefines.js b/Common/sources/commondefines.js index 947d7217..1eb80be8 100644 --- a/Common/sources/commondefines.js +++ b/Common/sources/commondefines.js @@ -32,6 +32,7 @@ 'use strict'; +const config = require("config"); const constants = require('./constants'); function InputCommand(data, copyExplicit) { @@ -108,6 +109,7 @@ function InputCommand(data, copyExplicit) { this['status_info_in'] = data['status_info_in']; this['attempt'] = data['attempt']; this['convertToOrigin'] = data['convertToOrigin']; + this['isSaveAs'] = data['isSaveAs']; if (copyExplicit) { this['withAuthorization'] = data['withAuthorization']; this['externalChangeInfo'] = data['externalChangeInfo']; @@ -171,6 +173,7 @@ function InputCommand(data, copyExplicit) { this['attempt'] = undefined; this['convertToOrigin'] = undefined; this['originformat'] = undefined; + this['isSaveAs'] = undefined; } } InputCommand.prototype = { @@ -363,8 +366,12 @@ InputCommand.prototype = { getJsonParams: function() { return this['jsonparams']; }, - setJsonParams: function(data) { - this['jsonparams'] = data; + appendJsonParams: function (data) { + if (this['jsonparams']) { + config.util.extendDeep(this['jsonparams'], data); + } else { + this['jsonparams'] = data; + } }, getLCID: function() { return this['lcid']; @@ -503,6 +510,12 @@ InputCommand.prototype = { }, setConvertToOrigin: function(data) { this['convertToOrigin'] = data; + }, + getIsSaveAs: function() { + return this['isSaveAs']; + }, + setIsSaveAs: function(data) { + this['isSaveAs'] = data; } }; diff --git a/Common/sources/constants.js b/Common/sources/constants.js index d370a625..c3e2ecc1 100644 --- a/Common/sources/constants.js +++ b/Common/sources/constants.js @@ -50,7 +50,8 @@ exports.VIEWER_ONLY = /^(?:(pdf|djvu|xps|oxps))$/; exports.DEFAULT_DOC_ID = 'docId'; exports.DEFAULT_USER_ID = 'userId'; exports.ALLOWED_PROTO = /^https?$/i; -exports.SHARED_KEY_NAME = 'WOPISrc'; +exports.SHARD_KEY_WOPI_NAME = 'WOPISrc'; +exports.SHARD_KEY_API_NAME = 'shardkey'; exports.RIGHTS = { None : 0, @@ -282,6 +283,7 @@ exports.FILE_STATUS_UPDATE_VERSION = 'updateversion'; exports.ACTIVEMQ_QUEUE_PREFIX = 'queue://'; exports.ACTIVEMQ_TOPIC_PREFIX = 'topic://'; +exports.TEMPLATES_DEFAULT_LOCALE = 'en-US'; exports.TEMPLATES_FOLDER_LOCALE_COLLISON_MAP = { 'en': 'en-US', 'pt': 'pt-BR', @@ -289,11 +291,6 @@ exports.TEMPLATES_FOLDER_LOCALE_COLLISON_MAP = { 'pt-PT': 'pt-PT', 'zh-TW': 'zh-TW' }; -exports.SUPPORTED_TEMPLATES_EXTENSIONS = { - 'Word': ['docx', 'docxf'], - 'Excel': ['xlsx'], - 'PowerPoint': ['pptx'] -}; exports.TABLE_RESULT_SCHEMA = [ 'tenant', 'id', diff --git a/Common/sources/operationContext.js b/Common/sources/operationContext.js index 585d0b81..78fe402f 100644 --- a/Common/sources/operationContext.js +++ b/Common/sources/operationContext.js @@ -41,11 +41,12 @@ function Context(){ this.logger = logger.getLogger('nodeJS'); this.initDefault(); } -Context.prototype.init = function(tenant, docId, userId, opt_shardKey) { +Context.prototype.init = function(tenant, docId, userId, opt_shardKey, opt_WopiSrc) { this.setTenant(tenant); this.setDocId(docId); this.setUserId(userId); this.setShardKey(opt_shardKey); + this.setWopiSrc(opt_WopiSrc); this.config = null; this.secret = null; @@ -65,21 +66,23 @@ Context.prototype.initFromConnection = function(conn) { } } let userId = conn.user?.id; - let shardKey = utils.getShardByConnection(this, conn); - this.init(tenant, docId || this.docId, userId || this.userId, shardKey); + let shardKey = utils.getShardKeyByConnection(this, conn); + let wopiSrc = utils.getWopiSrcByConnection(this, conn); + this.init(tenant, docId || this.docId, userId || this.userId, shardKey, wopiSrc); }; Context.prototype.initFromRequest = function(req) { let tenant = tenantManager.getTenantByRequest(this, req); let shardKey = utils.getShardKeyByRequest(this, req); - this.init(tenant, this.docId, this.userId, shardKey); + let wopiSrc = utils.getWopiSrcByRequest(this, req); + this.init(tenant, this.docId, this.userId, shardKey, wopiSrc); }; Context.prototype.initFromTaskQueueData = function(task) { let ctx = task.getCtx(); - this.init(ctx.tenant, ctx.docId, ctx.userId, ctx.shardKey); + this.init(ctx.tenant, ctx.docId, ctx.userId, ctx.shardKey, ctx.wopiSrc); }; Context.prototype.initFromPubSub = function(data) { let ctx = data.ctx; - this.init(ctx.tenant, ctx.docId, ctx.userId, ctx.shardKey); + this.init(ctx.tenant, ctx.docId, ctx.userId, ctx.shardKey, ctx.wopiSrc); }; Context.prototype.initTenantCache = async function() { this.config = await tenantManager.getTenantConfig(this); @@ -101,12 +104,16 @@ Context.prototype.setUserId = function(userId) { Context.prototype.setShardKey = function(shardKey) { this.shardKey = shardKey; }; +Context.prototype.setWopiSrc = function(wopiSrc) { + this.wopiSrc = wopiSrc; +}; Context.prototype.toJSON = function() { return { tenant: this.tenant, docId: this.docId, userId: this.userId, - shardKey: this.shardKey + shardKey: this.shardKey, + wopiSrc: this.wopiSrc } }; Context.prototype.getCfg = function(property, defaultValue) { diff --git a/Common/sources/storage-base.js b/Common/sources/storage-base.js index cf25061f..0d7869e3 100644 --- a/Common/sources/storage-base.js +++ b/Common/sources/storage-base.js @@ -31,102 +31,185 @@ */ 'use strict'; +const os = require('os'); +const cluster = require('cluster'); var config = require('config'); var utils = require('./utils'); -var storage = require('./' + config.get('storage.name')); -var tenantManager = require('./tenantManager'); +const cfgCacheStorage = config.get('storage'); +const cfgPersistentStorage = utils.deepMergeObjects({}, cfgCacheStorage, config.get('persistentStorage')); -const cfgCacheFolderName = config.get('storage.cacheFolderName'); +const cacheStorage = require('./' + cfgCacheStorage.name); +const persistentStorage = require('./' + cfgPersistentStorage.name); +const tenantManager = require('./tenantManager'); + +const HEALTH_CHECK_KEY_MAX = 10000; function getStoragePath(ctx, strPath, opt_specialDir) { - opt_specialDir = opt_specialDir || cfgCacheFolderName; - return opt_specialDir + '/' + tenantManager.getTenantPathPrefix(ctx) + strPath.replace(/\\/g, '/') + opt_specialDir = opt_specialDir || cfgCacheStorage.cacheFolderName; + return opt_specialDir + '/' + tenantManager.getTenantPathPrefix(ctx) + strPath.replace(/\\/g, '/'); +} +function getStorage(opt_specialDir) { + return opt_specialDir ? persistentStorage : cacheStorage; +} +function getStorageCfg(ctx, opt_specialDir) { + return opt_specialDir ? cfgPersistentStorage : cfgCacheStorage; +} +function canCopyBetweenStorage(storageCfgSrc, storageCfgDst) { + return storageCfgSrc.name === storageCfgDst.name && storageCfgSrc.endpoint === storageCfgDst.endpoint; +} +function isDiffrentPersistentStorage() { + return !canCopyBetweenStorage(cacheStorage, cfgPersistentStorage); } -exports.headObject = function(ctx, strPath, opt_specialDir) { - return storage.headObject(getStoragePath(ctx, strPath, opt_specialDir)); -}; -exports.getObject = function(ctx, strPath, opt_specialDir) { - return storage.getObject(getStoragePath(ctx, strPath, opt_specialDir)); -}; -exports.createReadStream = function(ctx, strPath, opt_specialDir) { - return storage.createReadStream(getStoragePath(ctx, strPath, opt_specialDir)); -}; -exports.putObject = function(ctx, strPath, buffer, contentLength, opt_specialDir) { - return storage.putObject(getStoragePath(ctx, strPath, opt_specialDir), buffer, contentLength); -}; -exports.uploadObject = function(ctx, strPath, filePath, opt_specialDir) { - return storage.uploadObject(getStoragePath(ctx, strPath, opt_specialDir), filePath); -}; -exports.copyObject = function(ctx, sourceKey, destinationKey, opt_specialDirSrc, opt_specialDirDst) { - let storageSrc = getStoragePath(ctx, sourceKey, opt_specialDirSrc); - let storageDst = getStoragePath(ctx, destinationKey, opt_specialDirDst); - return storage.copyObject(storageSrc, storageDst); -}; -exports.copyPath = function(ctx, sourcePath, destinationPath, opt_specialDirSrc, opt_specialDirDst) { - let storageSrc = getStoragePath(ctx, sourcePath, opt_specialDirSrc); - let storageDst = getStoragePath(ctx, destinationPath, opt_specialDirDst); - return storage.listObjects(storageSrc).then(function(list) { - return Promise.all(list.map(function(curValue) { - return storage.copyObject(curValue, storageDst + '/' + exports.getRelativePath(storageSrc, curValue)); - })); - }); -}; -exports.listObjects = function(ctx, strPath, opt_specialDir) { +async function headObject(ctx, strPath, opt_specialDir) { + let storage = getStorage(opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.headObject(storageCfg, getStoragePath(storageCfg, strPath, opt_specialDir)); +} +async function getObject(ctx, strPath, opt_specialDir) { + let storage = getStorage(opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.getObject(storageCfg, getStoragePath(storageCfg, strPath, opt_specialDir)); +} +async function createReadStream(ctx, strPath, opt_specialDir) { + let storage = getStorage(opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.createReadStream(storageCfg, getStoragePath(storageCfg, strPath, opt_specialDir)); +} +async function putObject(ctx, strPath, buffer, contentLength, opt_specialDir) { + let storage = getStorage(opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.putObject(storageCfg, getStoragePath(ctx, strPath, opt_specialDir), buffer, contentLength); +} +async function uploadObject(ctx, strPath, filePath, opt_specialDir) { + let storage = getStorage(opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.uploadObject(storageCfg, getStoragePath(ctx, strPath, opt_specialDir), filePath); +} +async function copyObject(ctx, sourceKey, destinationKey, opt_specialDirSrc, opt_specialDirDst) { + let storageSrc = getStorage(opt_specialDirSrc); + let storagePathSrc = getStoragePath(ctx, sourceKey, opt_specialDirSrc); + let storagePathDst = getStoragePath(ctx, destinationKey, opt_specialDirDst); + let storageCfgSrc = getStorageCfg(ctx, opt_specialDirSrc); + let storageCfgDst = getStorageCfg(ctx, opt_specialDirDst); + if (canCopyBetweenStorage(storageCfgSrc, storageCfgDst)){ + return await storageSrc.copyObject(storageCfgSrc, storageCfgDst, storagePathSrc, storagePathDst); + } else { + let storageDst = getStorage(opt_specialDirDst); + //todo stream + let buffer = await storageSrc.getObject(storageCfgSrc, storagePathSrc); + return await storageDst.putObject(storageCfgDst, storagePathDst, buffer, buffer.length); + } +} +async function copyPath(ctx, sourcePath, destinationPath, opt_specialDirSrc, opt_specialDirDst) { + let list = await listObjects(ctx, sourcePath, opt_specialDirSrc); + await Promise.all(list.map(function(curValue) { + return copyObject(ctx, curValue, destinationPath + '/' + getRelativePath(sourcePath, curValue), opt_specialDirSrc, opt_specialDirDst); + })); +} +async function listObjects(ctx, strPath, opt_specialDir) { + let storage = getStorage(opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); let prefix = getStoragePath(ctx, "", opt_specialDir); - return storage.listObjects(getStoragePath(ctx, strPath, opt_specialDir)).then(function(list) { + try { + let list = await storage.listObjects(storageCfg, getStoragePath(ctx, strPath, opt_specialDir)); return list.map((currentValue) => { return currentValue.substring(prefix.length); }); - }).catch(function(e) { + } catch (e) { ctx.logger.error('storage.listObjects: %s', e.stack); return []; - }); -}; -exports.deleteObject = function(ctx, strPath, opt_specialDir) { - return storage.deleteObject(getStoragePath(ctx, strPath, opt_specialDir)); -}; -exports.deletePath = function(ctx, strPath, opt_specialDir) { - return storage.deletePath(getStoragePath(ctx, strPath, opt_specialDir)); -}; -exports.getSignedUrl = function(ctx, baseUrl, strPath, urlType, optFilename, opt_creationDate, opt_specialDir) { - return storage.getSignedUrl(ctx, baseUrl, getStoragePath(ctx, strPath, opt_specialDir), urlType, optFilename, opt_creationDate); -}; -exports.getSignedUrls = function(ctx, baseUrl, strPath, urlType, opt_creationDate, opt_specialDir) { - let storageSrc = getStoragePath(ctx, strPath, opt_specialDir); - return storage.listObjects(storageSrc).then(function(list) { - return Promise.all(list.map(function(curValue) { - return storage.getSignedUrl(ctx, baseUrl, curValue, urlType, undefined, opt_creationDate); - })).then(function(urls) { - var outputMap = {}; - for (var i = 0; i < list.length && i < urls.length; ++i) { - outputMap[exports.getRelativePath(storageSrc, list[i])] = urls[i]; - } - return outputMap; - }); - }); -}; -exports.getSignedUrlsArrayByArray = function(ctx, baseUrl, list, urlType, opt_specialDir) { - return Promise.all(list.map(function(curValue) { - let storageSrc = getStoragePath(ctx, curValue, opt_specialDir); - return storage.getSignedUrl(ctx, baseUrl, storageSrc, urlType, undefined); + } +} +async function deleteObject(ctx, strPath, opt_specialDir) { + let storage = getStorage(opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.deleteObject(storageCfg, getStoragePath(ctx, strPath, opt_specialDir)); +} +async function deletePath(ctx, strPath, opt_specialDir) { + let storage = getStorage(opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.deletePath(storageCfg, getStoragePath(ctx, strPath, opt_specialDir)); +} +async function getSignedUrl(ctx, baseUrl, strPath, urlType, optFilename, opt_creationDate, opt_specialDir) { + let storage = getStorage(opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.getSignedUrl(ctx, storageCfg, baseUrl, getStoragePath(ctx, strPath, opt_specialDir), urlType, optFilename, opt_creationDate); +} +async function getSignedUrls(ctx, baseUrl, strPath, urlType, opt_creationDate, opt_specialDir) { + let storagePathSrc = getStoragePath(ctx, strPath, opt_specialDir); + let storage = getStorage(opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + let list = await storage.listObjects(storageCfg, storagePathSrc, storageCfg); + let urls = await Promise.all(list.map(function(curValue) { + return storage.getSignedUrl(ctx, storageCfg, baseUrl, curValue, urlType, undefined, opt_creationDate); })); -}; -exports.getSignedUrlsByArray = function(ctx, baseUrl, list, optPath, urlType, opt_specialDir) { - return exports.getSignedUrlsArrayByArray(ctx, baseUrl, list, urlType, opt_specialDir).then(function(urls) { - var outputMap = {}; - for (var i = 0; i < list.length && i < urls.length; ++i) { - if (optPath) { - let storageSrc = getStoragePath(ctx, optPath, opt_specialDir); - outputMap[exports.getRelativePath(storageSrc, list[i])] = urls[i]; - } else { - outputMap[list[i]] = urls[i]; - } + let outputMap = {}; + for (let i = 0; i < list.length && i < urls.length; ++i) { + outputMap[getRelativePath(storagePathSrc, list[i])] = urls[i]; + } + return outputMap; +} +async function getSignedUrlsArrayByArray(ctx, baseUrl, list, urlType, opt_specialDir) { + return await Promise.all(list.map(function (curValue) { + let storage = getStorage(opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + let storagePathSrc = getStoragePath(ctx, curValue, opt_specialDir); + return storage.getSignedUrl(ctx, storageCfg, baseUrl, storagePathSrc, urlType, undefined); + })); +} +async function getSignedUrlsByArray(ctx, baseUrl, list, optPath, urlType, opt_specialDir) { + let urls = await getSignedUrlsArrayByArray(ctx, baseUrl, list, urlType, opt_specialDir); + var outputMap = {}; + for (var i = 0; i < list.length && i < urls.length; ++i) { + if (optPath) { + let storagePathSrc = getStoragePath(ctx, optPath, opt_specialDir); + outputMap[getRelativePath(storagePathSrc, list[i])] = urls[i]; + } else { + outputMap[list[i]] = urls[i]; } - return outputMap; - }); -}; -exports.getRelativePath = function(strBase, strPath) { + } + return outputMap; +} +function getRelativePath(strBase, strPath) { return strPath.substring(strBase.length + 1); +} +async function healthCheck(ctx, opt_specialDir) { + const clusterId = cluster.isWorker ? cluster.worker.id : ''; + const tempName = 'hc_' + os.hostname() + '_' + clusterId + '_' + Math.round(Math.random() * HEALTH_CHECK_KEY_MAX); + const tempBuffer = Buffer.from([1, 2, 3, 4, 5]); + try { + //It's proper to putObject one tempName + await putObject(ctx, tempName, tempBuffer, tempBuffer.length, opt_specialDir); + //try to prevent case, when another process can remove same tempName + await deleteObject(ctx, tempName, opt_specialDir); + } catch (err) { + ctx.logger.warn('healthCheck storage(%s) error %s', opt_specialDir, err.stack); + } +} +function needServeStatic(opt_specialDir) { + let storage = getStorage(opt_specialDir); + return storage.needServeStatic(); +} + +module.exports = { + headObject, + getObject, + createReadStream, + putObject, + uploadObject, + copyObject, + copyPath, + listObjects, + deleteObject, + deletePath, + getSignedUrl, + getSignedUrls, + getSignedUrlsArrayByArray, + getSignedUrlsByArray, + getRelativePath, + isDiffrentPersistentStorage, + healthCheck, + needServeStatic }; diff --git a/Common/sources/storage-fs.js b/Common/sources/storage-fs.js index 267456af..e7f8c373 100644 --- a/Common/sources/storage-fs.js +++ b/Common/sources/storage-fs.js @@ -38,39 +38,33 @@ var path = require('path'); var utils = require("./utils"); var crypto = require('crypto'); const ms = require('ms'); +const config = require('config'); const commonDefines = require('./../../Common/sources/commondefines'); const constants = require('./../../Common/sources/constants'); -var config = require('config'); -var configStorage = config.get('storage'); -var cfgBucketName = configStorage.get('bucketName'); -var cfgStorageFolderName = configStorage.get('storageFolderName'); -var configFs = configStorage.get('fs'); -var cfgStorageFolderPath = configFs.get('folderPath'); -var cfgStorageSecretString = configFs.get('secretString'); -var cfgStorageUrlExpires = configFs.get('urlExpires'); const cfgExpSessionAbsolute = ms(config.get('services.CoAuthoring.expire.sessionabsolute')); -function getFilePath(strPath) { - return path.join(cfgStorageFolderPath, strPath); +function getFilePath(storageCfg, strPath) { + const storageFolderPath = storageCfg.fs.folderPath; + return path.join(storageFolderPath, strPath); } function getOutputPath(strPath) { return strPath.replace(/\\/g, '/'); } -async function headObject(strPath) { - let fsPath = getFilePath(strPath); +async function headObject(storageCfg, strPath) { + let fsPath = getFilePath(storageCfg, strPath); let stats = await stat(fsPath); return {ContentLength: stats.size}; } -async function getObject(strPath) { - let fsPath = getFilePath(strPath); +async function getObject(storageCfg, strPath) { + let fsPath = getFilePath(storageCfg, strPath); return await readFile(fsPath); } -async function createReadStream(strPath) { - let fsPath = getFilePath(strPath); +async function createReadStream(storageCfg, strPath) { + let fsPath = getFilePath(storageCfg, strPath); let stats = await stat(fsPath); let contentLength = stats.size; let readStream = await utils.promiseCreateReadStream(fsPath); @@ -80,8 +74,8 @@ async function createReadStream(strPath) { }; } -async function putObject(strPath, buffer, contentLength) { - var fsPath = getFilePath(strPath); +async function putObject(storageCfg, strPath, buffer, contentLength) { + var fsPath = getFilePath(storageCfg, strPath); await mkdir(path.dirname(fsPath), {recursive: true}); if (Buffer.isBuffer(buffer)) { @@ -92,63 +86,75 @@ async function putObject(strPath, buffer, contentLength) { } } -async function uploadObject(strPath, filePath) { - let fsPath = getFilePath(strPath); +async function uploadObject(storageCfg, strPath, filePath) { + let fsPath = getFilePath(storageCfg, strPath); await cp(filePath, fsPath, {force: true, recursive: true}); } -async function copyObject(sourceKey, destinationKey) { - let fsPathSource = getFilePath(sourceKey); - let fsPathDestination = getFilePath(destinationKey); +async function copyObject(storageCfgSrc, storageCfgDst, sourceKey, destinationKey) { + let fsPathSource = getFilePath(storageCfgSrc, sourceKey); + let fsPathDestination = getFilePath(storageCfgDst, destinationKey); await cp(fsPathSource, fsPathDestination, {force: true, recursive: true}); } -async function listObjects(strPath) { - let fsPath = getFilePath(strPath); +async function listObjects(storageCfg, strPath) { + const storageFolderPath = storageCfg.fs.folderPath; + let fsPath = getFilePath(storageCfg, strPath); let values = await utils.listObjects(fsPath); return values.map(function(curvalue) { - return getOutputPath(curvalue.substring(cfgStorageFolderPath.length + 1)); + return getOutputPath(curvalue.substring(storageFolderPath.length + 1)); }); } -async function deleteObject(strPath) { - const fsPath = getFilePath(strPath); +async function deleteObject(storageCfg, strPath) { + const fsPath = getFilePath(storageCfg, strPath); return rm(fsPath, {force: true, recursive: true}); } -async function deletePath(strPath) { - const fsPath = getFilePath(strPath); +async function deletePath(storageCfg, strPath) { + const fsPath = getFilePath(storageCfg, strPath); return rm(fsPath, {force: true, recursive: true}); } -async function getSignedUrl(ctx, baseUrl, strPath, urlType, optFilename, opt_creationDate) { +async function getSignedUrl(ctx, storageCfg, baseUrl, strPath, urlType, optFilename, opt_creationDate) { + const storageSecretString = storageCfg.fs.secretString; + const storageUrlExpires = storageCfg.fs.urlExpires; + const bucketName = storageCfg.bucketName; + const storageFolderName = storageCfg.storageFolderName; //replace '/' with %2f before encodeURIComponent becase nginx determine %2f as '/' and get wrong system path - var userFriendlyName = optFilename ? encodeURIComponent(optFilename.replace(/\//g, "%2f")) : path.basename(strPath); - var uri = '/' + cfgBucketName + '/' + cfgStorageFolderName + '/' + strPath + '/' + userFriendlyName; + const userFriendlyName = optFilename ? encodeURIComponent(optFilename.replace(/\//g, "%2f")) : path.basename(strPath); + var uri = '/' + bucketName + '/' + storageFolderName + '/' + strPath + '/' + userFriendlyName; //RFC 1123 does not allow underscores https://stackoverflow.com/questions/2180465/can-domain-name-subdomains-have-an-underscore-in-it - var url = utils.checkBaseUrl(ctx, baseUrl).replace(/_/g, "%5f"); + var url = utils.checkBaseUrl(ctx, baseUrl, storageCfg).replace(/_/g, "%5f"); url += uri; var date = Date.now(); let creationDate = opt_creationDate || date; - let expiredAfter = (commonDefines.c_oAscUrlTypes.Session === urlType ? (cfgExpSessionAbsolute / 1000) : cfgStorageUrlExpires) || 31536000; + let expiredAfter = (commonDefines.c_oAscUrlTypes.Session === urlType ? (cfgExpSessionAbsolute / 1000) : storageUrlExpires) || 31536000; //todo creationDate can be greater because mysql CURRENT_TIMESTAMP uses local time, not UTC var expires = creationDate + Math.ceil(Math.abs(date - creationDate) / expiredAfter) * expiredAfter; expires = Math.ceil(expires / 1000); expires += expiredAfter; - var md5 = crypto.createHash('md5').update(expires + decodeURIComponent(uri) + cfgStorageSecretString).digest("base64"); + var md5 = crypto.createHash('md5').update(expires + decodeURIComponent(uri) + storageSecretString).digest("base64"); md5 = md5.replace(/\+/g, "-").replace(/\//g, "_").replace(/=/g, ""); url += '?md5=' + encodeURIComponent(md5); url += '&expires=' + encodeURIComponent(expires); if (ctx.shardKey) { - url += `&${constants.SHARED_KEY_NAME}=${encodeURIComponent(ctx.shardKey)}`; + url += `&${constants.SHARD_KEY_API_NAME}=${encodeURIComponent(ctx.shardKey)}`; + } + if (ctx.wopiSrc) { + url += `&${constants.SHARD_KEY_WOPI_NAME}=${encodeURIComponent(ctx.wopiSrc)}`; } url += '&filename=' + userFriendlyName; return url; } +function needServeStatic() { + return true; +} + module.exports = { headObject, getObject, @@ -159,5 +165,6 @@ module.exports = { listObjects, deleteObject, deletePath, - getSignedUrl + getSignedUrl, + needServeStatic }; diff --git a/Common/sources/storage-s3.js b/Common/sources/storage-s3.js index 97c89ce0..603cb2b6 100644 --- a/Common/sources/storage-s3.js +++ b/Common/sources/storage-s3.js @@ -39,190 +39,190 @@ const { GetObjectCommand, PutObjectCommand, CopyObjectCommand} = require("@aws-s const { DeleteObjectsCommand, DeleteObjectCommand } = require("@aws-sdk/client-s3"); const { getSignedUrl } = require("@aws-sdk/s3-request-presigner"); const mime = require('mime'); +const config = require('config'); const utils = require('./utils'); const ms = require('ms'); const commonDefines = require('./../../Common/sources/commondefines'); -const config = require('config'); -const configStorage = require('config').get('storage'); -const cfgRegion = configStorage.get('region'); -const cfgEndpoint = configStorage.get('endpoint'); -const cfgBucketName = configStorage.get('bucketName'); -const cfgStorageFolderName = configStorage.get('storageFolderName'); -const cfgAccessKeyId = configStorage.get('accessKeyId'); -const cfgSecretAccessKey = configStorage.get('secretAccessKey'); -const cfgSslEnabled = configStorage.get('sslEnabled'); -const cfgS3ForcePathStyle = configStorage.get('s3ForcePathStyle'); -const configFs = configStorage.get('fs'); -const cfgStorageUrlExpires = configFs.get('urlExpires'); const cfgExpSessionAbsolute = ms(config.get('services.CoAuthoring.expire.sessionabsolute')); -/** - * Don't hard-code your credentials! - * Export the following environment variables instead: - * - * export AWS_ACCESS_KEY_ID='AKID' - * export AWS_SECRET_ACCESS_KEY='SECRET' - */ -let configS3 = { - region: cfgRegion, - endpoint: cfgEndpoint, - credentials : { - accessKeyId: cfgAccessKeyId, - secretAccessKey: cfgSecretAccessKey - } -}; - -if (configS3.endpoint) { - configS3.tls = cfgSslEnabled; - configS3.forcePathStyle = cfgS3ForcePathStyle; -} -const client = new S3Client(configS3); - //This operation enables you to delete multiple objects from a bucket using a single HTTP request. You may specify up to 1000 keys. const MAX_DELETE_OBJECTS = 1000; +let clients = {}; -function getFilePath(strPath) { - //todo - return cfgStorageFolderName + '/' + strPath; +function getS3Client(storageCfg) { + /** + * Don't hard-code your credentials! + * Export the following environment variables instead: + * + * export AWS_ACCESS_KEY_ID='AKID' + * export AWS_SECRET_ACCESS_KEY='SECRET' + */ + let configS3 = { + region: storageCfg.region, + endpoint: storageCfg.endpoint, + credentials : { + accessKeyId: storageCfg.accessKeyId, + secretAccessKey: storageCfg.secretAccessKey + } + }; + + if (configS3.endpoint) { + configS3.tls = storageCfg.sslEnabled; + configS3.forcePathStyle = storageCfg.s3ForcePathStyle; + } + let configJson = JSON.stringify(configS3); + let client = clients[configJson]; + if (!client) { + client = new S3Client(configS3); + clients[configJson] = client; + } + return client; } -function joinListObjects(inputArray, outputArray) { + +function getFilePath(storageCfg, strPath) { + const storageFolderName = storageCfg.storageFolderName; + return storageFolderName + '/' + strPath; +} +function joinListObjects(storageCfg, inputArray, outputArray) { if (!inputArray) { return; } + const storageFolderName = storageCfg.storageFolderName; let length = inputArray.length; for (let i = 0; i < length; i++) { - outputArray.push(inputArray[i].Key.substring((cfgStorageFolderName + '/').length)); + outputArray.push(inputArray[i].Key.substring((storageFolderName + '/').length)); } } -async function listObjectsExec(output, params) { - const data = await client.send(new ListObjectsCommand(params)); - joinListObjects(data.Contents, output); +async function listObjectsExec(storageCfg, output, params) { + const data = await getS3Client(storageCfg).send(new ListObjectsCommand(params)); + joinListObjects(storageCfg, data.Contents, output); if (data.IsTruncated && (data.NextMarker || (data.Contents && data.Contents.length > 0))) { params.Marker = data.NextMarker || data.Contents[data.Contents.length - 1].Key; - return await listObjectsExec(output, params); + return await listObjectsExec(storageCfg, output, params); } else { return output; } } -async function deleteObjectsHelp(aKeys) { +async function deleteObjectsHelp(storageCfg, aKeys) { //By default, the operation uses verbose mode in which the response includes the result of deletion of each key in your request. //In quiet mode the response includes only keys where the delete operation encountered an error. const input = { - Bucket: cfgBucketName, + Bucket: storageCfg.bucketName, Delete: { Objects: aKeys, Quiet: true } }; const command = new DeleteObjectsCommand(input); - await client.send(command); + await getS3Client(storageCfg).send(command); } -async function headObject(strPath) { +async function headObject(storageCfg, strPath) { const input = { - Bucket: cfgBucketName, - Key: getFilePath(strPath) + Bucket: storageCfg.bucketName, + Key: getFilePath(storageCfg, strPath) }; const command = new HeadObjectCommand(input); - let output = await client.send(command); + let output = await getS3Client(storageCfg).send(command); return {ContentLength: output.ContentLength}; } -async function getObject(strPath) { +async function getObject(storageCfg, strPath) { const input = { - Bucket: cfgBucketName, - Key: getFilePath(strPath) + Bucket: storageCfg.bucketName, + Key: getFilePath(storageCfg, strPath) }; const command = new GetObjectCommand(input); - const output = await client.send(command); + const output = await getS3Client(storageCfg).send(command); return await utils.stream2Buffer(output.Body); } -async function createReadStream(strPath) { +async function createReadStream(storageCfg, strPath) { const input = { - Bucket: cfgBucketName, - Key: getFilePath(strPath) + Bucket: storageCfg.bucketName, + Key: getFilePath(storageCfg, strPath) }; const command = new GetObjectCommand(input); - const output = await client.send(command); + const output = await getS3Client(storageCfg).send(command); return { contentLength: output.ContentLength, readStream: output.Body }; } -async function putObject(strPath, buffer, contentLength) { +async function putObject(storageCfg, strPath, buffer, contentLength) { //todo consider Expires const input = { - Bucket: cfgBucketName, - Key: getFilePath(strPath), + Bucket: storageCfg.bucketName, + Key: getFilePath(storageCfg, strPath), Body: buffer, ContentLength: contentLength, ContentType: mime.getType(strPath) }; const command = new PutObjectCommand(input); - await client.send(command); + await getS3Client(storageCfg).send(command); } -async function uploadObject(strPath, filePath) { +async function uploadObject(storageCfg, strPath, filePath) { const file = fs.createReadStream(filePath); //todo рассмотреть Expires const input = { - Bucket: cfgBucketName, - Key: getFilePath(strPath), + Bucket: storageCfg.bucketName, + Key: getFilePath(storageCfg, strPath), Body: file, ContentType: mime.getType(strPath) }; const command = new PutObjectCommand(input); - await client.send(command); + await getS3Client(storageCfg).send(command); } -async function copyObject(sourceKey, destinationKey) { +async function copyObject(storageCfgSrc, storageCfgDst, sourceKey, destinationKey) { //todo source bucket const input = { - Bucket: cfgBucketName, - Key: getFilePath(destinationKey), - CopySource: `/${cfgBucketName}/${getFilePath(sourceKey)}` + Bucket: storageCfgDst.bucketName, + Key: getFilePath(storageCfgDst, destinationKey), + CopySource: `/${storageCfgSrc.bucketName}/${getFilePath(storageCfgSrc, sourceKey)}` }; const command = new CopyObjectCommand(input); - await client.send(command); + await getS3Client(storageCfgDst).send(command); } -async function listObjects(strPath) { +async function listObjects(storageCfg, strPath) { let params = { - Bucket: cfgBucketName, - Prefix: getFilePath(strPath) + Bucket: storageCfg.bucketName, + Prefix: getFilePath(storageCfg, strPath) }; let output = []; - await listObjectsExec(output, params); + await listObjectsExec(storageCfg, output, params); return output; } -async function deleteObject(strPath) { +async function deleteObject(storageCfg, strPath) { const input = { - Bucket: cfgBucketName, - Key: getFilePath(strPath) + Bucket: storageCfg.bucketName, + Key: getFilePath(storageCfg, strPath) }; const command = new DeleteObjectCommand(input); - await client.send(command); + await getS3Client(storageCfg).send(command); }; -async function deleteObjects(strPaths) { +async function deleteObjects(storageCfg, strPaths) { let aKeys = strPaths.map(function (currentValue) { - return {Key: getFilePath(currentValue)}; + return {Key: getFilePath(storageCfg, currentValue)}; }); for (let i = 0; i < aKeys.length; i += MAX_DELETE_OBJECTS) { - await deleteObjectsHelp(aKeys.slice(i, i + MAX_DELETE_OBJECTS)); + await deleteObjectsHelp(storageCfg, aKeys.slice(i, i + MAX_DELETE_OBJECTS)); } } -async function deletePath(strPath) { - let list = await listObjects(strPath); - await deleteObjects(list); +async function deletePath(storageCfg, strPath) { + let list = await listObjects(storageCfg, strPath); + await deleteObjects(storageCfg, list); } -async function getSignedUrlWrapper(ctx, baseUrl, strPath, urlType, optFilename, opt_creationDate) { - let expires = (commonDefines.c_oAscUrlTypes.Session === urlType ? cfgExpSessionAbsolute / 1000 : cfgStorageUrlExpires) || 31536000; +async function getSignedUrlWrapper(ctx, storageCfg, baseUrl, strPath, urlType, optFilename, opt_creationDate) { + const storageUrlExpires = storageCfg.fs.urlExpires; + let expires = (commonDefines.c_oAscUrlTypes.Session === urlType ? cfgExpSessionAbsolute / 1000 : storageUrlExpires) || 31536000; // Signature version 4 presigned URLs must have an expiration date less than one week in the future expires = Math.min(expires, 604800); let userFriendlyName = optFilename ? optFilename.replace(/\//g, "%2f") : path.basename(strPath); let contentDisposition = utils.getContentDisposition(userFriendlyName, null, null); const input = { - Bucket: cfgBucketName, - Key: getFilePath(strPath), + Bucket: storageCfg.bucketName, + Key: getFilePath(storageCfg, strPath), ResponseContentDisposition: contentDisposition }; const command = new GetObjectCommand(input); @@ -230,12 +230,16 @@ async function getSignedUrlWrapper(ctx, baseUrl, strPath, urlType, optFilename, let options = { expiresIn: expires }; - return await getSignedUrl(client, command, options); + return await getSignedUrl(getS3Client(storageCfg), command, options); //extra query params cause SignatureDoesNotMatch //https://stackoverflow.com/questions/55503009/amazon-s3-signature-does-not-match-when-extra-query-params-ga-added-in-url // return utils.changeOnlyOfficeUrl(url, strPath, optFilename); } +function needServeStatic() { + return false; +} + module.exports = { headObject, getObject, @@ -246,5 +250,6 @@ module.exports = { listObjects, deleteObject, deletePath, - getSignedUrl: getSignedUrlWrapper + getSignedUrl: getSignedUrlWrapper, + needServeStatic }; diff --git a/Common/sources/utils.js b/Common/sources/utils.js index ab0b9a32..def7c5d9 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -35,6 +35,7 @@ //Fix EPROTO error in node 8.x at some web sites(https://github.com/nodejs/node/issues/21513) require("tls").DEFAULT_ECDH_CURVE = "auto"; +const { pipeline } = require('node:stream/promises'); var config = require('config'); var fs = require('fs'); var path = require('path'); @@ -51,7 +52,6 @@ const NodeCache = require( "node-cache" ); const ms = require('ms'); const constants = require('./constants'); const commonDefines = require('./commondefines'); -const logger = require('./logger'); const forwarded = require('forwarded'); const { RequestFilteringHttpAgent, RequestFilteringHttpsAgent } = require("request-filtering-agent"); const https = require('https'); @@ -83,7 +83,6 @@ const cfgTokenOutboxUrlExclusionRegex = config.get('services.CoAuthoring.token.o const cfgSecret = config.get('aesEncrypt.secret'); const cfgAESConfig = config.get('aesEncrypt.config'); const cfgRequesFilteringAgent = config.get('services.CoAuthoring.request-filtering-agent'); -const cfgAllowPrivateIPAddressForSignedRequests = config.get('services.CoAuthoring.server.allowPrivateIPAddressForSignedRequests'); const cfgStorageExternalHost = config.get('storage.externalHost'); const cfgExternalRequestDirectIfIn = config.get('externalRequest.directIfIn'); const cfgExternalRequestAction = config.get('externalRequest.action'); @@ -273,7 +272,6 @@ function isRedirectResponse(response) { function isAllowDirectRequest(ctx, uri, isInJwtToken) { let res = false; const tenExternalRequestDirectIfIn = ctx.getCfg('externalRequest.directIfIn', cfgExternalRequestDirectIfIn); - const tenAllowPrivateIPAddressForSignedRequests = ctx.getCfg('services.CoAuthoring.server.allowPrivateIPAddressForSignedRequests', cfgAllowPrivateIPAddressForSignedRequests); let allowList = tenExternalRequestDirectIfIn.allowList; if (allowList.length > 0) { let allowIndex = allowList.findIndex((allowPrefix) => { @@ -281,7 +279,7 @@ function isAllowDirectRequest(ctx, uri, isInJwtToken) { }, uri); res = -1 !== allowIndex; ctx.logger.debug("isAllowDirectRequest check allow list res=%s", res); - } else if (tenExternalRequestDirectIfIn.jwtToken && tenAllowPrivateIPAddressForSignedRequests) { + } else if (tenExternalRequestDirectIfIn.jwtToken) { res = isInJwtToken; ctx.logger.debug("isAllowDirectRequest url in jwt token res=%s", res); } @@ -353,7 +351,7 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A uri = URI.serialize(URI.parse(uri)); var urlParsed = url.parse(uri); let sizeLimit = optLimit || Number.MAX_VALUE; - let bufferLength = 0; + let bufferLength = 0, timeoutId; let hash = crypto.createHash('sha256'); //if you expect binary data, you should set encoding: null let connectionAndInactivity = optTimeout && optTimeout.connectionAndInactivity && ms(optTimeout.connectionAndInactivity); @@ -378,6 +376,7 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A Object.assign(options.headers, opt_headers); } let fError = function(err) { + clearTimeout(timeoutId); reject(err); } if (!opt_streamWriter) { @@ -389,6 +388,7 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A } executed = true; if (err) { + clearTimeout(timeoutId); reject(err); } else { var contentLength = response.caseless.get('content-length'); @@ -396,6 +396,7 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A ctx.logger.warn('downloadUrlPromise body size mismatch: uri=%s; content-length=%s; body.length=%d', uri, contentLength, body.length); } let sha256 = hash.digest('hex'); + clearTimeout(timeoutId); resolve({response: response, body: body, sha256: sha256}); } }; @@ -417,13 +418,21 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A error.response = response; if (opt_streamWriter && !isRedirectResponse(response)) { this.off('error', fError); - resolve(pipeStreams(this, opt_streamWriter, true)); + pipeline(this, opt_streamWriter) + .then(resolve, reject) + .finally(() => { + clearTimeout(timeoutId); + }); } else { raiseErrorObj(this, error); } } else if (opt_streamWriter) { this.off('error', fError); - resolve(pipeStreams(this, opt_streamWriter, true)); + pipeline(this, opt_streamWriter) + .then(resolve, reject) + .finally(() => { + clearTimeout(timeoutId); + }); } }; let fData = function(chunk) { @@ -439,13 +448,13 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A .on('data', fData) .on('error', fError); if (optTimeout && optTimeout.wholeCycle) { - setTimeout(function() { + timeoutId = setTimeout(function() { raiseError(ro, 'ETIMEDOUT', 'Error: whole request cycle timeout'); }, ms(optTimeout.wholeCycle)); } }); } -function postRequestPromise(ctx, uri, postData, postDataStream, postDataSize, optTimeout, opt_Authorization, opt_header) { +function postRequestPromise(ctx, uri, postData, postDataStream, postDataSize, optTimeout, opt_Authorization, opt_headers) { return new Promise(function(resolve, reject) { const tenTenantRequestDefaults = ctx.getCfg('services.CoAuthoring.requestDefaults', cfgRequestDefaults); const tenTokenOutboxHeader = ctx.getCfg('services.CoAuthoring.token.outbox.header', cfgTokenOutboxHeader); @@ -453,29 +462,32 @@ function postRequestPromise(ctx, uri, postData, postDataStream, postDataSize, op //IRI to URI uri = URI.serialize(URI.parse(uri)); var urlParsed = url.parse(uri); - var headers = {'Content-Type': 'application/json'}; + let connectionAndInactivity = optTimeout && optTimeout.connectionAndInactivity && ms(optTimeout.connectionAndInactivity); + let options = config.util.extendDeep({}, tenTenantRequestDefaults); + Object.assign(options, {uri: urlParsed, encoding: 'utf8', timeout: connectionAndInactivity}); + //baseRequest creates new agent(win-ca injects in globalAgent) + options.agentOptions = https.globalAgent.options; + if (postData) { + options.body = postData; + } + if (!options.headers) { + options.headers = {}; + } if (opt_Authorization) { //todo ctx.getCfg - headers[tenTokenOutboxHeader] = tenTokenOutboxPrefix + opt_Authorization; + options.headers[tenTokenOutboxHeader] = tenTokenOutboxPrefix + opt_Authorization; + } + if (opt_headers) { + Object.assign(options.headers, opt_headers); } - headers = opt_header || headers; if (undefined !== postDataSize) { //If no Content-Length is set, data will automatically be encoded in HTTP Chunked transfer encoding, //so that server knows when the data ends. The Transfer-Encoding: chunked header is added. //https://nodejs.org/api/http.html#requestwritechunk-encoding-callback //issue with Transfer-Encoding: chunked wopi and sharepoint 2019 //https://community.alteryx.com/t5/Dev-Space/Download-Tool-amp-Microsoft-SharePoint-Chunked-Request-Error/td-p/735824 - headers['Content-Length'] = postDataSize; + options.headers['Content-Length'] = postDataSize; } - let connectionAndInactivity = optTimeout && optTimeout.connectionAndInactivity && ms(optTimeout.connectionAndInactivity); - let options = config.util.extendDeep({}, tenTenantRequestDefaults); - Object.assign(options, {uri: urlParsed, encoding: 'utf8', headers: headers, timeout: connectionAndInactivity}); - //baseRequest creates new agent(win-ca injects in globalAgent) - options.agentOptions = https.globalAgent.options; - if (postData) { - options.body = postData; - } - let executed = false; let ro = request.post(options, function(err, response, body) { if (executed) { @@ -535,6 +547,8 @@ exports.mapAscServerErrorToOldError = function(error) { res = -7; break; case constants.CONVERT_LIMITS : + res = -10; + break; case constants.CONVERT_NEED_PARAMS : case constants.CONVERT_LIBREOFFICE : case constants.CONVERT_CORRUPTED : @@ -783,14 +797,22 @@ function getDomainByRequest(ctx, req) { } exports.getDomainByConnection = getDomainByConnection; exports.getDomainByRequest = getDomainByRequest; -function getShardByConnection(ctx, conn) { - return conn?.handshake?.query?.[constants.SHARED_KEY_NAME]; +function getShardKeyByConnection(ctx, conn) { + return conn?.handshake?.query?.[constants.SHARD_KEY_API_NAME]; +} +function getWopiSrcByConnection(ctx, conn) { + return conn?.handshake?.query?.[constants.SHARD_KEY_WOPI_NAME]; } function getShardKeyByRequest(ctx, req) { - return req.query[constants.SHARED_KEY_NAME]; + return req.query?.[constants.SHARD_KEY_API_NAME]; } -exports.getShardByConnection = getShardByConnection; +function getWopiSrcByRequest(ctx, req) { + return req.query?.[constants.SHARD_KEY_WOPI_NAME]; +} +exports.getShardKeyByConnection = getShardKeyByConnection; +exports.getWopiSrcByConnection = getWopiSrcByConnection; exports.getShardKeyByRequest = getShardKeyByRequest; +exports.getWopiSrcByRequest = getWopiSrcByRequest; function stream2Buffer(stream) { return new Promise(function(resolve, reject) { if (!stream.readable) { @@ -1142,8 +1164,9 @@ exports.convertLicenseInfoToServerParams = function(licenseInfo) { license.buildNumber = commonDefines.buildNumber; return license; }; -exports.checkBaseUrl = function(ctx, baseUrl) { - const tenStorageExternalHost = ctx.getCfg('storage.externalHost', cfgStorageExternalHost); +exports.checkBaseUrl = function(ctx, baseUrl, opt_storageCfg) { + let storageExternalHost = opt_storageCfg ? opt_storageCfg.externalHost : cfgStorageExternalHost + const tenStorageExternalHost = ctx.getCfg('storage.externalHost', storageExternalHost); return tenStorageExternalHost ? tenStorageExternalHost : baseUrl; }; exports.resolvePath = function(object, path, defaultValue) { diff --git a/DocService/npm-shrinkwrap.json b/DocService/npm-shrinkwrap.json index d384c95c..34ee9088 100644 --- a/DocService/npm-shrinkwrap.json +++ b/DocService/npm-shrinkwrap.json @@ -843,9 +843,9 @@ } }, "async": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", - "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz", + "integrity": "sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==" }, "asynckit": { "version": "0.4.0", @@ -1148,9 +1148,9 @@ "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==" }, "cookie": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", - "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==" + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==" }, "cookie-signature": { "version": "1.0.6", @@ -1243,9 +1243,9 @@ "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" }, "denque": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/denque/-/denque-2.0.1.tgz", - "integrity": "sha512-tfiWc6BQLXNLpNiR5iGd0Ocu3P3VpxfzFiqubLgMfhfOw9WyvgJBd46CClNn9k3qfbjvT//0cf7AlYRX/OslMQ==" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" }, "depd": { "version": "1.1.2", @@ -1332,9 +1332,9 @@ "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, "ejs": { - "version": "3.1.8", - "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.8.tgz", - "integrity": "sha512-/sXZeMlhS0ArkfX2Aw780gJzXSMPnKjtspYZv+f3NiKLlubezAHDU5+9xz6gd3/NhG3txQCo6xlglmTS+oTGEQ==", + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz", + "integrity": "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==", "requires": { "jake": "^10.8.5" } @@ -1519,16 +1519,16 @@ "integrity": "sha512-c2bQfLNbMzLPmzQuOr8fy0csy84WmwnER81W88DzTp9CYNPJ6yzOj2EZAh9pywYpqHnshVLHQJ8WzldAyfY+Iw==" }, "express": { - "version": "4.18.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", - "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", + "version": "4.19.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", + "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", "requires": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.1", + "body-parser": "1.20.2", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.5.0", + "cookie": "0.6.0", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", @@ -1556,15 +1556,30 @@ "vary": "~1.1.2" }, "dependencies": { - "accepts": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", - "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "body-parser": { + "version": "1.20.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", + "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", "requires": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" + "bytes": "3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.11.0", + "raw-body": "2.5.2", + "type-is": "~1.6.18", + "unpipe": "1.0.0" } }, + "bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" + }, "depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -1583,11 +1598,6 @@ "mime-db": "1.52.0" } }, - "negotiator": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", - "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" - }, "on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -1596,6 +1606,17 @@ "ee-first": "1.1.1" } }, + "raw-body": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "requires": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + } + }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -2117,7 +2138,7 @@ "is-property": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", - "integrity": "sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ=" + "integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==" }, "is-regex": { "version": "1.1.4", @@ -2191,14 +2212,14 @@ } }, "jake": { - "version": "10.8.5", - "resolved": "https://registry.npmjs.org/jake/-/jake-10.8.5.tgz", - "integrity": "sha512-sVpxYeuAhWt0OTWITwT98oyV0GsXyMlXCF+3L1SuafBVUIr/uILGRB+NqwkzhgXKvoJpDIpQvqkUALgdmQsQxw==", + "version": "10.8.7", + "resolved": "https://registry.npmjs.org/jake/-/jake-10.8.7.tgz", + "integrity": "sha512-ZDi3aP+fG/LchyBzUM804VjddnwfSfsdeYkwt8NcbKRvo4rFkjhs456iLFn3k2ZUWvNe4i48WACDbza8fhq2+w==", "requires": { "async": "^3.2.3", "chalk": "^4.0.2", - "filelist": "^1.0.1", - "minimatch": "^3.0.4" + "filelist": "^1.0.4", + "minimatch": "^3.1.2" } }, "jimp": { @@ -2347,9 +2368,9 @@ "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" }, "long": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", - "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" }, "lru-cache": { "version": "6.0.0", @@ -2531,16 +2552,16 @@ } }, "mysql2": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-2.3.3.tgz", - "integrity": "sha512-wxJUev6LgMSgACDkb/InIFxDprRa6T95+VEoR+xPvtngtccNH2dGjEB/fVZ8yg1gWv1510c9CvXuJHi5zUm0ZA==", + "version": "3.9.8", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.9.8.tgz", + "integrity": "sha512-+5JKNjPuks1FNMoy9TYpl77f+5frbTklz7eb3XDwbpsERRLEeXiW2PDEkakYF50UuKU2qwfGnyXpKYvukv8mGA==", "requires": { - "denque": "^2.0.1", + "denque": "^2.1.0", "generate-function": "^2.3.1", "iconv-lite": "^0.6.3", - "long": "^4.0.0", - "lru-cache": "^6.0.0", - "named-placeholders": "^1.1.2", + "long": "^5.2.1", + "lru-cache": "^8.0.0", + "named-placeholders": "^1.1.3", "seq-queue": "^0.0.5", "sqlstring": "^2.3.2" }, @@ -2553,34 +2574,25 @@ "safer-buffer": ">= 2.1.2 < 3.0.0" } }, - "sqlstring": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.3.tgz", - "integrity": "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==" + "lru-cache": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-8.0.5.tgz", + "integrity": "sha512-MhWWlVnuab1RG5/zMRRcVGXZLCXrZTgfwMikgzCegsPnG62yDQo5JnqKkrK4jO5iKqDAZGItAqN5CtKBCBWRUA==" } } }, "named-placeholders": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.2.tgz", - "integrity": "sha512-wiFWqxoLL3PGVReSZpjLVxyJ1bRqe+KKJVbr4hGs1KWfTZTQyezHFBbuKj9hsizHyGV2ne7EMjHdxEGAybD5SA==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.3.tgz", + "integrity": "sha512-eLoBxg6wE/rZkJPhU/xRX1WTpkFEwDJEN96oxFrTsqBdbT5ec295Q+CoHrL9IT0DipqKhmGcaZmwOt8OON5x1w==", "requires": { - "lru-cache": "^4.1.3" + "lru-cache": "^7.14.1" }, "dependencies": { "lru-cache": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", - "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", - "requires": { - "pseudomap": "^1.0.2", - "yallist": "^2.1.2" - } - }, - "yallist": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", - "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=" + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==" } } }, @@ -2836,11 +2848,6 @@ "ipaddr.js": "1.9.1" } }, - "pseudomap": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", - "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" - }, "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", @@ -2884,7 +2891,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "requires": { "core-util-is": "~1.0.0", @@ -3086,7 +3093,7 @@ "seq-queue": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/seq-queue/-/seq-queue-0.0.5.tgz", - "integrity": "sha1-1WgS4cAXpuTnw+Ojeh2m143TyT4=" + "integrity": "sha512-hr3Wtp/GZIc/6DAGPDcV4/9WoZhjrkXsi5B/07QgX8tsdc6ilr7BFM6PM6rbdAX1kFSDYeZGLipIZZKyQP0O5Q==" }, "serve-static": { "version": "1.15.0", @@ -3215,6 +3222,11 @@ "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" }, + "sqlstring": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.3.tgz", + "integrity": "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==" + }, "standard-as-callback": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", diff --git a/DocService/package.json b/DocService/package.json index 5dc7d96a..f42a1f6e 100644 --- a/DocService/package.json +++ b/DocService/package.json @@ -20,9 +20,9 @@ "cron": "1.5.0", "deep-equal": "1.0.1", "dmdb": "1.0.14280", - "ejs": "3.1.8", + "ejs": "3.1.10", "exif-parser": "0.1.12", - "express": "4.18.2", + "express": "4.19.2", "fakeredis": "2.0.0", "ioredis": "5.3.1", "jimp": "0.22.10", @@ -35,7 +35,7 @@ "multer": "1.4.3", "multi-integer-range": "4.0.7", "multiparty": "4.2.1", - "mysql2": "2.3.3", + "mysql2": "3.9.8", "oracledb": "6.3.0", "pg": "8.11.3", "redis": "4.6.11", diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 5e4a5343..4f2fa708 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -102,7 +102,11 @@ const queueService = require('./../../Common/sources/taskqueueRabbitMQ'); const operationContext = require('./../../Common/sources/operationContext'); const tenantManager = require('./../../Common/sources/tenantManager'); -const editorDataStorage = require('./' + config.get('services.CoAuthoring.server.editorDataStorage')); +const cfgEditorDataStorage = config.get('services.CoAuthoring.server.editorDataStorage'); +const cfgEditorStatStorage = config.get('services.CoAuthoring.server.editorStatStorage'); +const editorDataStorage = require('./' + cfgEditorDataStorage); +const editorStatStorage = require('./' + (cfgEditorStatStorage || cfgEditorDataStorage)); +const utilsDocService = require("./utilsDocService"); const cfgEditSingleton = config.get('services.CoAuthoring.server.edit_singleton'); const cfgEditor = config.get('services.CoAuthoring.editor'); @@ -153,7 +157,9 @@ const EditorTypes = { }; const defaultHttpPort = 80, defaultHttpsPort = 443; // Default ports (for http and https) -const editorData = new editorDataStorage(); +//todo remove editorDataStorage constructor usage after 8.1 +const editorData = editorDataStorage.EditorData ? new editorDataStorage.EditorData() : new editorDataStorage(); +const editorStat = editorStatStorage.EditorStat ? new editorStatStorage.EditorStat() : new editorDataStorage(); const clientStatsD = statsDClient.getClient(); let connections = []; // Active connections let lockDocumentsTimerId = {};//to drop connection that can't unlockDocument @@ -436,30 +442,30 @@ function updatePresenceCounters(ctx, conn, val) { //yield ctx.initTenantCache(); //no need.only global config } if (utils.isLiveViewer(conn)) { - yield editorData.incrLiveViewerConnectionsCountByShard(ctx, SHARD_ID, val); + yield editorStat.incrLiveViewerConnectionsCountByShard(ctx, SHARD_ID, val); if (aggregationCtx) { - yield editorData.incrLiveViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, val); + yield editorStat.incrLiveViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, val); } if (clientStatsD) { - let countLiveView = yield editorData.getLiveViewerConnectionsCount(ctx, connections); + let countLiveView = yield editorStat.getLiveViewerConnectionsCount(ctx, connections); clientStatsD.gauge('expireDoc.connections.liveview', countLiveView); } } else if (conn.isCloseCoAuthoring || (conn.user && conn.user.view)) { - yield editorData.incrViewerConnectionsCountByShard(ctx, SHARD_ID, val); + yield editorStat.incrViewerConnectionsCountByShard(ctx, SHARD_ID, val); if (aggregationCtx) { - yield editorData.incrViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, val); + yield editorStat.incrViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, val); } if (clientStatsD) { - let countView = yield editorData.getViewerConnectionsCount(ctx, connections); + let countView = yield editorStat.getViewerConnectionsCount(ctx, connections); clientStatsD.gauge('expireDoc.connections.view', countView); } } else { - yield editorData.incrEditorConnectionsCountByShard(ctx, SHARD_ID, val); + yield editorStat.incrEditorConnectionsCountByShard(ctx, SHARD_ID, val); if (aggregationCtx) { - yield editorData.incrEditorConnectionsCountByShard(aggregationCtx, SHARD_ID, val); + yield editorStat.incrEditorConnectionsCountByShard(aggregationCtx, SHARD_ID, val); } if (clientStatsD) { - let countEditors = yield editorData.getEditorConnectionsCount(ctx, connections); + let countEditors = yield editorStat.getEditorConnectionsCount(ctx, connections); clientStatsD.gauge('expireDoc.connections.edit', countEditors); } } @@ -490,7 +496,7 @@ function removePresence(ctx, conn) { let changeConnectionInfo = co.wrap(function*(ctx, conn, cmd) { if (!conn.denyChangeName && conn.user) { - yield* publish(ctx, {type: commonDefines.c_oPublishType.changeConnecitonInfo, ctx: ctx, docId: conn.docId, useridoriginal: conn.user.idOriginal, cmd: cmd}); + yield publish(ctx, {type: commonDefines.c_oPublishType.changeConnecitonInfo, ctx: ctx, docId: conn.docId, useridoriginal: conn.user.idOriginal, cmd: cmd}); return true; } return false; @@ -626,11 +632,11 @@ function* updateEditUsers(ctx, licenseInfo, userId, anonym, isLiveViewer) { licenseInfo.usersExpire - 1; let period = utils.getLicensePeriod(licenseInfo.startDate, now); if (isLiveViewer) { - yield editorData.addPresenceUniqueViewUser(ctx, userId, expireAt, {anonym: anonym}); - yield editorData.addPresenceUniqueViewUsersOfMonth(ctx, userId, period, {anonym: anonym, firstOpenDate: now.toISOString()}); + yield editorStat.addPresenceUniqueViewUser(ctx, userId, expireAt, {anonym: anonym}); + yield editorStat.addPresenceUniqueViewUsersOfMonth(ctx, userId, period, {anonym: anonym, firstOpenDate: now.toISOString()}); } else { - yield editorData.addPresenceUniqueUser(ctx, userId, expireAt, {anonym: anonym}); - yield editorData.addPresenceUniqueUsersOfMonth(ctx, userId, period, {anonym: anonym, firstOpenDate: now.toISOString()}); + yield editorStat.addPresenceUniqueUser(ctx, userId, expireAt, {anonym: anonym}); + yield editorStat.addPresenceUniqueUsersOfMonth(ctx, userId, period, {anonym: anonym, firstOpenDate: now.toISOString()}); } } function* getEditorsCount(ctx, docId, opt_hvals) { @@ -667,12 +673,12 @@ function* isUserReconnect(ctx, docId, userId, connectionId) { } let pubsubOnMessage = null;//todo move function -function* publish(ctx, data, optDocId, optUserId, opt_pubsub) { +async function publish(ctx, data, optDocId, optUserId, opt_pubsub) { var needPublish = true; let hvals; if (optDocId && optUserId) { needPublish = false; - hvals = yield editorData.getPresence(ctx, optDocId, connections); + hvals = await editorData.getPresence(ctx, optDocId, connections); for (var i = 0; i < hvals.length; ++i) { var elem = JSON.parse(hvals[i]); if (optUserId != elem.id) { @@ -690,7 +696,7 @@ function* publish(ctx, data, optDocId, optUserId, opt_pubsub) { //todo send connections from getLocalConnectionCount to pubsubOnMessage pubsubOnMessage(msg); } else if(realPubsub) { - yield realPubsub.publish(msg); + await realPubsub.publish(msg); } } return needPublish; @@ -742,7 +748,8 @@ async function sendServerRequest(ctx, uri, dataObject, opt_checkAndFixAuthorizat } dataObject.setToken(bodyToken); } - let postRes = await utils.postRequestPromise(ctx, uri, JSON.stringify(dataObject), undefined, undefined, tenCallbackRequestTimeout, auth); + let headers = {'Content-Type': 'application/json'}; + let postRes = await utils.postRequestPromise(ctx, uri, JSON.stringify(dataObject), undefined, undefined, tenCallbackRequestTimeout, auth, headers); ctx.logger.debug('postData response: data = %s', postRes.body); return postRes.body; } @@ -839,15 +846,16 @@ function* setForceSave(ctx, docId, forceSave, cmd, success, url) { if (commonDefines.c_oAscForceSaveTypes.Command !== forceSaveType) { let data = {type: forceSaveType, time: forceSave.getTime(), success: success}; if(commonDefines.c_oAscForceSaveTypes.Form === forceSaveType || commonDefines.c_oAscForceSaveTypes.Internal === forceSaveType) { - data = {code: commonDefines.c_oAscServerCommandErrors.NoError, time: null, inProgress: false}; + let code = success ? commonDefines.c_oAscServerCommandErrors.NoError : commonDefines.c_oAscServerCommandErrors.UnknownError; + data = {code: code, time: null, inProgress: false}; if (commonDefines.c_oAscForceSaveTypes.Internal === forceSaveType) { data.url = url; } let userId = cmd.getUserConnectionId(); docId = cmd.getUserConnectionDocId() || docId; - yield* publish(ctx, {type: commonDefines.c_oPublishType.rpc, ctx, docId, userId, data, responseKey: cmd.getResponseKey()}); + yield publish(ctx, {type: commonDefines.c_oPublishType.rpc, ctx, docId, userId, data, responseKey: cmd.getResponseKey()}); } else { - yield* publish(ctx, {type: commonDefines.c_oPublishType.forceSave, ctx: ctx, docId: docId, data: data}, cmd.getUserConnectionId()); + yield publish(ctx, {type: commonDefines.c_oPublishType.forceSave, ctx: ctx, docId: docId, data: data}, cmd.getUserConnectionId()); } } } @@ -911,7 +919,9 @@ async function applyForceSaveCache(ctx, docId, forceSave, type, opt_userConnecti } return res; } -async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_userId, opt_userConnectionId, opt_userConnectionDocId, opt_userIndex, opt_responseKey, opt_baseUrl, opt_queue, opt_pubsub, opt_conn, opt_initShardKey) { +async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_userId, opt_userConnectionId, + opt_userConnectionDocId, opt_userIndex, opt_responseKey, opt_baseUrl, + opt_queue, opt_pubsub, opt_conn, opt_initShardKey, opt_jsonParams) { const tenForceSaveUsingButtonWithoutChanges = ctx.getCfg('services.CoAuthoring.server.forceSaveUsingButtonWithoutChanges', cfgForceSaveUsingButtonWithoutChanges); ctx.logger.debug('startForceSave start'); let res = {code: commonDefines.c_oAscServerCommandErrors.NoError, time: null, inProgress: false}; @@ -932,7 +942,7 @@ async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_ newChangesLastDate.setMilliseconds(0);//remove milliseconds avoid issues with MySQL datetime rounding let newChangesLastTime = newChangesLastDate.getTime(); let baseUrl = utils.getBaseUrlByConnection(ctx, opt_conn); - let changeInfo = getExternalChangeInfo(opt_conn.user, newChangesLastTime); + let changeInfo = getExternalChangeInfo(opt_conn.user, newChangesLastTime, opt_conn.lang); await editorData.setForceSave(ctx, docId, newChangesLastTime, 0, baseUrl, changeInfo, null); forceSave = await editorData.getForceSave(ctx, docId); } @@ -961,13 +971,6 @@ async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_ forceSave.setAuthorUserId(opt_userId); forceSave.setAuthorUserIndex(opt_userIndex); - if (commonDefines.c_oAscForceSaveTypes.Timeout === type) { - await co(publish(ctx, { - type: commonDefines.c_oPublishType.forceSave, ctx: ctx, docId: docId, - data: {type: type, time: forceSave.getTime(), start: true} - }, undefined, undefined, opt_pubsub)); - } - let priority; let expiration; if (commonDefines.c_oAscForceSaveTypes.Timeout === type) { @@ -979,9 +982,15 @@ async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_ //start new convert let status = await converterService.convertFromChanges(ctx, docId, baseUrl, forceSave, startedForceSave.changeInfo, opt_userdata, opt_formdata, opt_userConnectionId, opt_userConnectionDocId, opt_responseKey, priority, expiration, - opt_queue, undefined, opt_initShardKey); + opt_queue, undefined, opt_initShardKey, opt_jsonParams); if (constants.NO_ERROR === status.err) { res.time = forceSave.getTime(); + if (commonDefines.c_oAscForceSaveTypes.Timeout === type) { + await publish(ctx, { + type: commonDefines.c_oPublishType.forceSave, ctx: ctx, docId: docId, + data: {type: type, time: forceSave.getTime(), start: true} + }, undefined, undefined, opt_pubsub); + } } else { res.code = commonDefines.c_oAscServerCommandErrors.UnknownError; } @@ -990,8 +999,8 @@ async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_ ctx.logger.debug('startForceSave end'); return res; } -function getExternalChangeInfo(user, date) { - return {user_id: user.id, user_id_original: user.idOriginal, user_name: user.username, change_date: date}; +function getExternalChangeInfo(user, date, lang) { + return {user_id: user.id, user_id_original: user.idOriginal, user_name: user.username, lang, change_date: date}; } let resetForceSaveAfterChanges = co.wrap(function*(ctx, docId, newChangesLastTime, puckerIndex, baseUrl, changeInfo) { const tenForceSaveEnable = ctx.getCfg('services.CoAuthoring.autoAssembly.enable', cfgForceSaveEnable); @@ -1035,9 +1044,11 @@ function* startRPC(ctx, conn, responseKey, data) { case 'sendForm': { let forceSaveRes; if (conn.user) { + //isPrint - to remove forms + let jsonParams = {'documentLayout': {'isPrint': true}}; forceSaveRes = yield startForceSave(ctx, docId, commonDefines.c_oAscForceSaveTypes.Form, undefined, data.formdata, conn.user.idOriginal, conn.user.id, undefined, conn.user.indexUser, - responseKey, undefined, undefined, undefined, conn); + responseKey, undefined, undefined, undefined, conn, undefined, jsonParams); } if (!forceSaveRes || commonDefines.c_oAscServerCommandErrors.NoError !== forceSaveRes.code || forceSaveRes.inProgress) { sendDataRpc(ctx, conn, responseKey, forceSaveRes); @@ -1203,7 +1214,7 @@ let onReplySendStatusDocument = co.wrap(function*(ctx, docId, replyData) { var oData = parseReplyData(ctx, replyData); if (!(oData && commonDefines.c_oAscServerCommandErrors.NoError == oData.error)) { // Error subscribing to callback, send warning - yield* publish(ctx, {type: commonDefines.c_oPublishType.warning, ctx: ctx, docId: docId, description: 'Error on save server subscription!'}); + yield publish(ctx, {type: commonDefines.c_oPublishType.warning, ctx: ctx, docId: docId, description: 'Error on save server subscription!'}); } }); function* publishCloseUsersConnection(ctx, docId, users, isOriginalId, code, description) { @@ -1212,7 +1223,7 @@ function* publishCloseUsersConnection(ctx, docId, users, isOriginalId, code, des map[val] = 1; return map; }, {}); - yield* publish(ctx, { + yield publish(ctx, { type: commonDefines.c_oPublishType.closeConnection, ctx: ctx, docId: docId, usersMap: usersMap, isOriginalId: isOriginalId, code: code, description: description }); @@ -1233,7 +1244,7 @@ function closeUsersConnection(ctx, docId, usersMap, isOriginalId, code, descript } function* dropUsersFromDocument(ctx, docId, users) { if (Array.isArray(users)) { - yield* publish(ctx, {type: commonDefines.c_oPublishType.drop, ctx: ctx, docId: docId, users: users, description: ''}); + yield publish(ctx, {type: commonDefines.c_oPublishType.drop, ctx: ctx, docId: docId, users: users, description: ''}); } } @@ -1323,7 +1334,7 @@ function* cleanDocumentOnExitNoChanges(ctx, docId, opt_userId, opt_userIndex, op yield* cleanDocumentOnExit(ctx, docId, false, opt_userIndex); } -function createSaveTimer(ctx, docId, opt_userId, opt_userIndex, opt_queue, opt_noDelay, opt_initShardKey) { +function createSaveTimer(ctx, docId, opt_userId, opt_userIndex, opt_userLcid, opt_queue, opt_noDelay, opt_initShardKey) { return co(function*(){ const tenAscSaveTimeOutDelay = ctx.getCfg('services.CoAuthoring.server.savetimeoutdelay', cfgAscSaveTimeOutDelay); @@ -1341,7 +1352,7 @@ function createSaveTimer(ctx, docId, opt_userId, opt_userIndex, opt_queue, opt_n } while (true) { if (!sqlBase.isLockCriticalSection(docId)) { - canvasService.saveFromChanges(ctx, docId, updateTask.statusInfo, null, opt_userId, opt_userIndex, opt_queue, opt_initShardKey); + yield canvasService.saveFromChanges(ctx, docId, updateTask.statusInfo, null, opt_userId, opt_userIndex, opt_userLcid, opt_queue, opt_initShardKey); break; } yield utils.sleep(c_oAscLockTimeOutDelay); @@ -1498,6 +1509,7 @@ async function isSchemaCompatible([tableName, tableSchema]) { exports.c_oAscServerStatus = c_oAscServerStatus; exports.editorData = editorData; +exports.editorStat = editorStat; exports.sendData = sendData; exports.modifyConnectionForPassword = modifyConnectionForPassword; exports.parseUrl = parseUrl; @@ -1590,175 +1602,175 @@ exports.install = function(server, callbackFunction) { }); }); - io.on('connection', function(conn) { - if (!conn) { - operationContext.global.logger.error("null == conn"); - return; - } + io.on('connection', async function(conn) { let ctx = new operationContext.Context(); - ctx.initFromConnection(conn); - //todo - //yield ctx.initTenantCache(); - if (constants.DEFAULT_DOC_ID === ctx.docId) { - ctx.logger.error('io.on connection unexpected key use key pattern = "%s" url = %s', constants.DOC_ID_PATTERN, conn.handshake?.url); - sendDataDisconnectReason(ctx, conn, constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); - conn.disconnect(true); - return; - } - if (getIsShutdown()) { - sendFileError(ctx, conn, 'Server shutdow'); - return; - } - conn.baseUrl = utils.getBaseUrlByConnection(ctx, conn); - conn.sessionIsSendWarning = false; - conn.sessionTimeConnect = conn.sessionTimeLastAction = new Date().getTime(); + try { + if (!conn) { + operationContext.global.logger.error("null == conn"); + return; + } + ctx.initFromConnection(conn); + await ctx.initTenantCache(); + if (constants.DEFAULT_DOC_ID === ctx.docId) { + ctx.logger.error('io.on connection unexpected key use key pattern = "%s" url = %s', constants.DOC_ID_PATTERN, conn.handshake?.url); + sendDataDisconnectReason(ctx, conn, constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); + conn.disconnect(true); + return; + } + if (getIsShutdown()) { + sendFileError(ctx, conn, 'Server shutdow'); + return; + } + conn.baseUrl = utils.getBaseUrlByConnection(ctx, conn); + conn.sessionIsSendWarning = false; + conn.sessionTimeConnect = conn.sessionTimeLastAction = new Date().getTime(); - conn.on('message', function(data) { - return co(function* () { - var docId = 'null'; - let ctx = new operationContext.Context(); - try { - ctx.initFromConnection(conn); - yield ctx.initTenantCache(); - const tenErrorFiles = ctx.getCfg('FileConverter.converter.errorfiles', cfgErrorFiles); + conn.on('message', function(data) { + return co(function* () { + var docId = 'null'; + let ctx = new operationContext.Context(); + try { + ctx.initFromConnection(conn); + yield ctx.initTenantCache(); + const tenErrorFiles = ctx.getCfg('FileConverter.converter.errorfiles', cfgErrorFiles); - var startDate = null; - if(clientStatsD) { - startDate = new Date(); - } + var startDate = null; + if(clientStatsD) { + startDate = new Date(); + } - docId = conn.docId; - ctx.logger.info('data.type = %s', data.type); - if(getIsShutdown()) - { - ctx.logger.debug('Server shutdown receive data'); - return; - } - if (conn.isCiriticalError && ('message' == data.type || 'getLock' == data.type || 'saveChanges' == data.type || - 'isSaveLock' == data.type)) { - ctx.logger.warn("conn.isCiriticalError send command: type = %s", data.type); - sendDataDisconnectReason(ctx, conn, constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); - conn.disconnect(true); - return; - } - if ((conn.isCloseCoAuthoring || (conn.user && conn.user.view)) && - ('getLock' == data.type || 'saveChanges' == data.type || 'isSaveLock' == data.type)) { - ctx.logger.warn("conn.user.view||isCloseCoAuthoring access deny: type = %s", data.type); - sendDataDisconnectReason(ctx, conn, constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); - conn.disconnect(true); - return; - } - yield encryptPasswordParams(ctx, data); - switch (data.type) { - case 'auth' : - try { - yield* auth(ctx, conn, data); - } catch(err){ - ctx.logger.error('auth error: %s', err.stack); + docId = conn.docId; + ctx.logger.info('data.type = %s', data.type); + if(getIsShutdown()) + { + ctx.logger.debug('Server shutdown receive data'); + return; + } + if (conn.isCiriticalError && ('message' == data.type || 'getLock' == data.type || 'saveChanges' == data.type || + 'isSaveLock' == data.type)) { + ctx.logger.warn("conn.isCiriticalError send command: type = %s", data.type); sendDataDisconnectReason(ctx, conn, constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); conn.disconnect(true); return; } - break; - case 'message' : - yield* onMessage(ctx, conn, data); - break; - case 'cursor' : - yield* onCursor(ctx, conn, data); - break; - case 'getLock' : - yield* getLock(ctx, conn, data, false); - break; - case 'saveChanges' : - yield* saveChanges(ctx, conn, data); - break; - case 'isSaveLock' : - yield* isSaveLock(ctx, conn, data); - break; - case 'unSaveLock' : - yield* unSaveLock(ctx, conn, -1, -1, -1); - break; // The index is sent -1, because this is an emergency withdrawal without saving - case 'getMessages' : - yield* getMessages(ctx, conn, data); - break; - case 'unLockDocument' : - yield* checkEndAuthLock(ctx, data.unlock, data.isSave, docId, conn.user.id, data.releaseLocks, data.deleteIndex, conn); - break; - case 'close': - yield* closeDocument(ctx, conn); - break; - case 'versionHistory' : { - let cmd = new commonDefines.InputCommand(data.cmd); - yield* versionHistory(ctx, conn, cmd); - break; - } - case 'openDocument' : { - var cmd = new commonDefines.InputCommand(data.message); - cmd.fillFromConnection(conn); - yield canvasService.openDocument(ctx, conn, cmd); - break; - } - case 'clientLog': - let level = data.level?.toLowerCase(); - if("trace" === level || "debug" === level || "info" === level || "warn" === level || "error" === level || "fatal" === level) { - ctx.logger[level]("clientLog: %s", data.msg); + if ((conn.isCloseCoAuthoring || (conn.user && conn.user.view)) && + ('getLock' == data.type || 'saveChanges' == data.type || 'isSaveLock' == data.type)) { + ctx.logger.warn("conn.user.view||isCloseCoAuthoring access deny: type = %s", data.type); + sendDataDisconnectReason(ctx, conn, constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); + conn.disconnect(true); + return; } - if ("error" === level && tenErrorFiles && docId) { - let destDir = 'browser/' + docId; - yield storage.copyPath(ctx, docId, destDir, undefined, tenErrorFiles); - yield* saveErrorChanges(ctx, docId, destDir); + yield encryptPasswordParams(ctx, data); + switch (data.type) { + case 'auth' : + try { + yield* auth(ctx, conn, data); + } catch(err){ + ctx.logger.error('auth error: %s', err.stack); + sendDataDisconnectReason(ctx, conn, constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); + conn.disconnect(true); + return; + } + break; + case 'message' : + yield* onMessage(ctx, conn, data); + break; + case 'cursor' : + yield* onCursor(ctx, conn, data); + break; + case 'getLock' : + yield getLock(ctx, conn, data, false); + break; + case 'saveChanges' : + yield* saveChanges(ctx, conn, data); + break; + case 'isSaveLock' : + yield* isSaveLock(ctx, conn, data); + break; + case 'unSaveLock' : + yield* unSaveLock(ctx, conn, -1, -1, -1); + break; // The index is sent -1, because this is an emergency withdrawal without saving + case 'getMessages' : + yield* getMessages(ctx, conn, data); + break; + case 'unLockDocument' : + yield* checkEndAuthLock(ctx, data.unlock, data.isSave, docId, conn.user.id, data.releaseLocks, data.deleteIndex, conn); + break; + case 'close': + yield* closeDocument(ctx, conn); + break; + case 'versionHistory' : { + let cmd = new commonDefines.InputCommand(data.cmd); + yield* versionHistory(ctx, conn, cmd); + break; + } + case 'openDocument' : { + var cmd = new commonDefines.InputCommand(data.message); + cmd.fillFromConnection(conn); + yield canvasService.openDocument(ctx, conn, cmd); + break; + } + case 'clientLog': + let level = data.level?.toLowerCase(); + if("trace" === level || "debug" === level || "info" === level || "warn" === level || "error" === level || "fatal" === level) { + ctx.logger[level]("clientLog: %s", data.msg); + } + if ("error" === level && tenErrorFiles && docId) { + let destDir = 'browser/' + docId; + yield storage.copyPath(ctx, docId, destDir, undefined, tenErrorFiles); + yield* saveErrorChanges(ctx, docId, destDir); + } + break; + case 'extendSession' : + ctx.logger.debug("extendSession idletime: %d", data.idletime); + conn.sessionIsSendWarning = false; + conn.sessionTimeLastAction = new Date().getTime() - data.idletime; + break; + case 'forceSaveStart' : + var forceSaveRes; + if (conn.user) { + forceSaveRes = yield startForceSave(ctx, docId, commonDefines.c_oAscForceSaveTypes.Button, + undefined, undefined, conn.user.idOriginal, conn.user.id, + undefined, conn.user.indexUser, undefined, undefined, undefined, undefined, conn); + } else { + forceSaveRes = {code: commonDefines.c_oAscServerCommandErrors.UnknownError, time: null}; + } + sendData(ctx, conn, {type: "forceSaveStart", messages: forceSaveRes}); + break; + case 'rpc' : + yield* startRPC(ctx, conn, data.responseKey, data.data); + break; + case 'authChangesAck' : + delete conn.authChangesAck; + break; + default: + ctx.logger.debug("unknown command %j", data); + break; } - break; - case 'extendSession' : - ctx.logger.debug("extendSession idletime: %d", data.idletime); - conn.sessionIsSendWarning = false; - conn.sessionTimeLastAction = new Date().getTime() - data.idletime; - break; - case 'forceSaveStart' : - var forceSaveRes; - if (conn.user) { - forceSaveRes = yield startForceSave(ctx, docId, commonDefines.c_oAscForceSaveTypes.Button, - undefined, undefined, conn.user.idOriginal, conn.user.id, - undefined, conn.user.indexUser, undefined, undefined, undefined, undefined, conn); - } else { - forceSaveRes = {code: commonDefines.c_oAscServerCommandErrors.UnknownError, time: null}; - } - sendData(ctx, conn, {type: "forceSaveStart", messages: forceSaveRes}); - break; - case 'rpc' : - yield* startRPC(ctx, conn, data.responseKey, data.data); - break; - case 'authChangesAck' : - delete conn.authChangesAck; - break; - default: - ctx.logger.debug("unknown command %j", data); - break; - } - if(clientStatsD) { - if('openDocument' != data.type) { - clientStatsD.timing('coauth.data.' + data.type, new Date() - startDate); + } catch (e) { + ctx.logger.error("error receiving response: type = %s %s", (data && data.type) ? data.type : 'null', e.stack); } - } - } catch (e) { - ctx.logger.error("error receiving response: type = %s %s", (data && data.type) ? data.type : 'null', e.stack); - } + }); }); - }); - conn.on("disconnect", function(reason) { - return co(function* () { - let ctx = new operationContext.Context(); - try { - ctx.initFromConnection(conn); - yield ctx.initTenantCache(); - yield* closeDocument(ctx, conn, reason); - } catch (err) { - ctx.logger.error('Error conn close: %s', err.stack); - } + conn.on("disconnect", function(reason) { + return co(function* () { + let ctx = new operationContext.Context(); + try { + ctx.initFromConnection(conn); + yield ctx.initTenantCache(); + yield* closeDocument(ctx, conn, reason); + } catch (err) { + ctx.logger.error('Error conn close: %s', err.stack); + } + }); }); - }); - _checkLicense(ctx, conn); + _checkLicense(ctx, conn); + } catch(err){ + ctx.logger.error('connection error: %s', err.stack); + sendDataDisconnectReason(ctx, conn, constants.DROP_CODE, constants.DROP_REASON); + conn.disconnect(true); + } }); io.engine.on("connection_error", (err) => { operationContext.global.logger.warn('io.connection_error code=%s, message=%s', err.code, err.message); @@ -1827,7 +1839,7 @@ exports.install = function(server, callbackFunction) { if (!participantsTimestamp) { participantsTimestamp = Date.now(); } - yield* publish(ctx, {type: commonDefines.c_oPublishType.participantsState, ctx: ctx, docId: docId, userId: tmpUser.id, participantsTimestamp: participantsTimestamp, participants: participants}, docId, tmpUser.id); + yield publish(ctx, {type: commonDefines.c_oPublishType.participantsState, ctx: ctx, docId: docId, userId: tmpUser.id, participantsTimestamp: participantsTimestamp, participants: participants}, docId, tmpUser.id); tmpUser.view = tmpView; // For this user, we remove the lock from saving @@ -1849,11 +1861,11 @@ exports.install = function(server, callbackFunction) { } } //Release locks - userLocks = yield* removeUserLocks(ctx, docId, conn.user.id); + userLocks = yield removeUserLocks(ctx, docId, conn.user.id); if (0 < userLocks.length) { //todo send nothing in case of close document //sendReleaseLock(conn, userLocks); - yield* publish(ctx, {type: commonDefines.c_oPublishType.releaseLock, ctx: ctx, docId: docId, userId: conn.user.id, locks: userLocks}, docId, conn.user.id); + yield publish(ctx, {type: commonDefines.c_oPublishType.releaseLock, ctx: ctx, docId: docId, userId: conn.user.id, locks: userLocks}, docId, conn.user.id); } // For this user, remove the Lock from the document @@ -1875,7 +1887,8 @@ exports.install = function(server, callbackFunction) { } if (needSaveChanges && !conn.encrypted) { // Send changes to save server - yield createSaveTimer(ctx, docId, tmpUser.idOriginal, userIndex); + let user_lcid = utilsDocService.localeToLCID(conn.lang); + yield createSaveTimer(ctx, docId, tmpUser.idOriginal, userIndex, user_lcid); } else if (needSendStatus) { yield* cleanDocumentOnExitNoChanges(ctx, docId, tmpUser.idOriginal, userIndex); } else { @@ -1941,31 +1954,19 @@ exports.install = function(server, callbackFunction) { return objChangesDocument; } - function* getAllLocks(ctx, docId) { - var docLockRes = []; - var docLock = yield editorData.getLocks(ctx, docId); - for (var i = 0; i < docLock.length; ++i) { - docLockRes.push(docLock[i]); - } - return docLockRes; - } - function* removeUserLocks(ctx, docId, userId) { - var userLocks = [], i; - var toCache = []; - var docLock = yield* getAllLocks(ctx, docId); - for (i = 0; i < docLock.length; ++i) { - var elem = docLock[i]; - if (elem.user === userId) { - userLocks.push(elem); - } else { - toCache.push(elem); + async function removeUserLocks(ctx, docId, userId) { + let locks = await editorData.getLocks(ctx, docId); + let res = []; + let toRemove = {}; + for (let lockId in locks) { + let lock = locks[lockId]; + if (lock.user === userId) { + toRemove[lockId] = lock; + res.push(lock); } } - //remove all - yield editorData.removeLocks(ctx, docId); - //set all - yield editorData.addLocks(ctx, docId, toCache); - return userLocks; + await editorData.removeLocks(ctx, docId, toRemove); + return res; } function* checkEndAuthLock(ctx, unlock, isSave, docId, userId, releaseLocks, deleteIndex, conn) { @@ -1987,7 +1988,7 @@ exports.install = function(server, callbackFunction) { var unlockRes = yield editorData.unlockAuth(ctx, docId, userId); if (commonDefines.c_oAscUnlockRes.Unlocked === unlockRes) { const participantsMap = yield getParticipantMap(ctx, docId); - yield* publish(ctx, { + yield publish(ctx, { type: commonDefines.c_oPublishType.auth, ctx: ctx, docId: docId, @@ -2001,10 +2002,10 @@ exports.install = function(server, callbackFunction) { //Release locks if (releaseLocks && conn) { - const userLocks = yield* removeUserLocks(ctx, docId, userId); + const userLocks = yield removeUserLocks(ctx, docId, userId); if (0 < userLocks.length) { sendReleaseLock(ctx, conn, userLocks); - yield* publish(ctx, { + yield publish(ctx, { type: commonDefines.c_oPublishType.releaseLock, ctx: ctx, docId: docId, @@ -2084,20 +2085,20 @@ exports.install = function(server, callbackFunction) { // Recalculation only for foreign Lock when saving on a client that added/deleted rows or columns function _recalcLockArray(userId, _locks, oRecalcIndexColumns, oRecalcIndexRows) { + let res = {}; if (null == _locks) { - return false; + return res; } - var count = _locks.length; var element = null, oRangeOrObjectId = null; - var i; var sheetId = -1; - var isModify = false; - for (i = 0; i < count; ++i) { + for (let lockId in _locks) { + let isModify = false; + let lock = _locks[lockId]; // we do not count for ourselves - if (userId === _locks[i].user) { + if (userId === lock.user) { continue; } - element = _locks[i].block; + element = lock.block; if (c_oAscLockTypeElem.Range !== element["type"] || c_oAscLockTypeElemSubType.InsertColumns === element["subType"] || c_oAscLockTypeElemSubType.InsertRows === element["subType"]) { @@ -2119,8 +2120,11 @@ exports.install = function(server, callbackFunction) { oRangeOrObjectId["r2"] = oRecalcIndexRows[sheetId].getLockMe2(oRangeOrObjectId["r2"]); isModify = true; } + if (isModify) { + res[lockId] = lock; + } } - return isModify; + return res; } function _addRecalcIndex(oRecalcIndex) { @@ -2255,7 +2259,10 @@ exports.install = function(server, callbackFunction) { return name; } function isEditMode(permissions, mode) { - //as in web-apps/apps/documenteditor/main/app/controller/Main.js + //like this.api.asc_setViewMode(!this.appOptions.isEdit && !this.appOptions.isRestrictedEdit); + //https://github.com/ONLYOFFICE/web-apps/blob/4a7879b4f88f315fe94d9f7d97c0ed8aa9f82221/apps/documenteditor/main/app/controller/Main.js#L1743 + //todo permissions in embed editor + //https://github.com/ONLYOFFICE/web-apps/blob/72b8350c71e7b314b63b8eec675e76156bb4a2e4/apps/documenteditor/forms/app/controller/ApplicationController.js#L627 return (!mode || mode !== 'view') && (!permissions || permissions.edit !== false || permissions.review === true || permissions.comment === true || permissions.fillForms === true); } @@ -2272,10 +2279,11 @@ exports.install = function(server, callbackFunction) { } if (decoded.queryParams) { let queryParams = decoded.queryParams; - data.lang = queryParams.lang || queryParams.ui || "en-US"; + data.lang = queryParams.lang || queryParams.ui || constants.TEMPLATES_DEFAULT_LOCALE; } - if (decoded.fileInfo) { + if (wopiClient.isWopiJwtToken(decoded)) { let fileInfo = decoded.fileInfo; + let queryParams = decoded.queryParams; if (openCmd) { openCmd.format = wopiClient.getFileTypeByInfo(fileInfo); openCmd.title = fileInfo.BreadcrumbDocName || fileInfo.BaseFileName; @@ -2291,11 +2299,15 @@ exports.install = function(server, callbackFunction) { openCmd.userid = fileInfo.UserId; } } + let permissionsEdit = !fileInfo.ReadOnly && fileInfo.UserCanWrite && queryParams.formsubmit !== "1"; + let permissionsFillForm = permissionsEdit || queryParams.formsubmit === "1"; let permissions = { - edit: !fileInfo.ReadOnly && fileInfo.UserCanWrite, + edit: permissionsEdit, review: (fileInfo.SupportsReviewing === false) ? false : (fileInfo.UserCanReview === false ? false : fileInfo.UserCanReview), copy: fileInfo.CopyPasteRestrictions !== "CurrentDocumentOnly" && fileInfo.CopyPasteRestrictions !== "BlockAll", - print: !fileInfo.DisablePrint && !fileInfo.HidePrintOption + print: !fileInfo.DisablePrint && !fileInfo.HidePrintOption, + chat: queryParams.dchat!=="1", + fillForms: permissionsFillForm }; //todo (review: undefiend) // res = deepEqual(data.permissions, permissions, {strict: true}); @@ -2373,7 +2385,8 @@ exports.install = function(server, callbackFunction) { if (null != edit.lang) { data.lang = edit.lang; } - if (null != edit.mode) { + //allow to restrict rights so don't use token mode in case of 'view' + if (null != edit.mode && 'view' !== data.mode) { data.mode = edit.mode; } if (edit.coEditing?.mode) { @@ -2429,7 +2442,7 @@ exports.install = function(server, callbackFunction) { } //todo make required fields - if (decoded.url || decoded.payload|| (decoded.key && !decoded.fileInfo)) { + if (decoded.url || decoded.payload|| (decoded.key && !wopiClient.isWopiJwtToken(decoded))) { ctx.logger.warn('fillDataFromJwt token has invalid format'); res = false; } @@ -2470,7 +2483,7 @@ exports.install = function(server, callbackFunction) { isDecoded = true; let decoded = checkJwtRes.decoded; let fillDataFromJwtRes = false; - if (decoded.fileInfo) { + if (wopiClient.isWopiJwtToken(decoded)) { //wopi fillDataFromJwtRes = fillDataFromWopiJwt(decoded, data); } else if (decoded.editorConfig && undefined !== decoded.editorConfig.ds_view) { @@ -2537,7 +2550,7 @@ exports.install = function(server, callbackFunction) { upsertRes = yield canvasService.commandOpenStartPromise(ctx, docId, utils.getBaseUrlByConnection(ctx, conn), data.documentCallbackUrl, format); curIndexUser = upsertRes.insertId; //todo update additional in commandOpenStartPromise - if ((upsertRes.isInsert || (wopiParams && 2 === curIndexUser)) && (undefined !== data.timezoneOffset || ctx.shardKey)) { + if ((upsertRes.isInsert || (wopiParams && 2 === curIndexUser)) && (undefined !== data.timezoneOffset || ctx.shardKey || ctx.wopiSrc)) { //todo insert in commandOpenStartPromise. insert here for database compatibility if (false === canvasService.hasAdditionalCol) { let selectRes = yield taskResult.select(ctx, docId); @@ -2554,6 +2567,9 @@ exports.install = function(server, callbackFunction) { if (ctx.shardKey) { task.additional += sqlBase.DocumentAdditional.prototype.setShardKey(ctx.shardKey); } + if (ctx.wopiSrc) { + task.additional += sqlBase.DocumentAdditional.prototype.setWopiSrc(ctx.wopiSrc); + } yield taskResult.update(ctx, task); } else { ctx.logger.warn('auth unknown column "additional"'); @@ -2569,7 +2585,7 @@ exports.install = function(server, callbackFunction) { const curUserId = curUserIdOriginal + curIndexUser; conn.tenant = tenantManager.getTenantByConnection(ctx, conn); conn.docId = data.docid; - conn.permissions = data.permissions; + conn.permissions = data.permissions || {}; conn.user = { id: curUserId, idOriginal: curUserIdOriginal, @@ -2592,6 +2608,7 @@ exports.install = function(server, callbackFunction) { } conn.unsyncTime = null; conn.encrypted = data.encrypted; + conn.lang = data.lang; conn.supportAuthChangesAck = data.supportAuthChangesAck; const c_LR = constants.LICENSE_RESULT; @@ -2750,7 +2767,7 @@ exports.install = function(server, callbackFunction) { if (bIsSuccessRestore) { // check locks var arrayBlocks = data['block']; - var getLockRes = yield* getLock(ctx, conn, data, true); + var getLockRes = yield getLock(ctx, conn, data, true); if (arrayBlocks && (0 === arrayBlocks.length || getLockRes)) { yield* authRestore(ctx, conn, data.sessionId); } else { @@ -2869,7 +2886,7 @@ exports.install = function(server, callbackFunction) { //closing could happen during async action return false; } - yield* publish(ctx, {type: commonDefines.c_oPublishType.participantsState, ctx: ctx, docId: docId, userId: tmpUser.id, participantsTimestamp: participantsTimestamp, participants: participantsMap, waitAuthUserId: waitAuthUserId}, docId, tmpUser.id); + yield publish(ctx, {type: commonDefines.c_oPublishType.participantsState, ctx: ctx, docId: docId, userId: tmpUser.id, participantsTimestamp: participantsTimestamp, participants: participantsMap, waitAuthUserId: waitAuthUserId}, docId, tmpUser.id); return res; } @@ -2972,17 +2989,13 @@ exports.install = function(server, callbackFunction) { const tenTypesUpload = ctx.getCfg('services.CoAuthoring.utils.limits_image_types_upload', cfgTypesUpload); const docId = conn.docId; - let docLock; - if(EditorTypes.document == conn.editorType){ - docLock = {}; - let elem; - const allLocks = yield* getAllLocks(ctx, docId); - for(let i = 0 ; i < allLocks.length; ++i) { - elem = allLocks[i]; - docLock[elem.block] = elem; + let docLock = yield editorData.getLocks(ctx, docId); + if (EditorTypes.document !== conn.editorType){ + let docLockList = []; + for (let lockId in docLock) { + docLockList.push(docLock[lockId]); } - } else { - docLock = yield* getAllLocks(ctx, docId); + docLock = docLockList; } let allMessages = yield editorData.getMessages(ctx, docId); allMessages = allMessages.length > 0 ? allMessages : undefined;//todo client side @@ -3015,7 +3028,7 @@ exports.install = function(server, callbackFunction) { } function* onMessage(ctx, conn, data) { - if (false === conn.permissions.chat) { + if (false === conn.permissions?.chat) { ctx.logger.warn("insert message permissions.chat==false"); return; } @@ -3028,7 +3041,7 @@ exports.install = function(server, callbackFunction) { var messages = [msg]; sendDataMessage(ctx, conn, messages); - yield* publish(ctx, {type: commonDefines.c_oPublishType.message, ctx: ctx, docId: docId, userId: userId, messages: messages}, docId, userId); + yield publish(ctx, {type: commonDefines.c_oPublishType.message, ctx: ctx, docId: docId, userId: userId, messages: messages}, docId, userId); } function* onCursor(ctx, conn, data) { @@ -3039,100 +3052,55 @@ exports.install = function(server, callbackFunction) { ctx.logger.info("send cursor: %s", msg); var messages = [msg]; - yield* publish(ctx, {type: commonDefines.c_oPublishType.cursor, ctx: ctx, docId: docId, userId: userId, messages: messages}, docId, userId); + yield publish(ctx, {type: commonDefines.c_oPublishType.cursor, ctx: ctx, docId: docId, userId: userId, messages: messages}, docId, userId); } - - function* getLock(ctx, conn, data, bIsRestore) { - ctx.logger.info("getLock"); - var fLock = null; + // For Word block is now string "guid" + // For Excel block is now object { sheetId, type, rangeOrObjectId, guid } + // For presentations, this is an object { type, val } or { type, slideId, objId } + async function getLock(ctx, conn, data, bIsRestore) { + ctx.logger.debug("getLock"); + var fCheckLock = null; switch (conn.editorType) { case EditorTypes.document: // Word - fLock = getLockWord; + fCheckLock = _checkLockWord; break; case EditorTypes.spreadsheet: // Excel - fLock = getLockExcel; + fCheckLock = _checkLockExcel; break; case EditorTypes.presentation: // PP - fLock = getLockPresentation; + fCheckLock = _checkLockPresentation; break; + default: + return false; } - return fLock ? yield* fLock(ctx, conn, data, bIsRestore) : false; - } - - function* getLockWord(ctx, conn, data, bIsRestore) { - var docId = conn.docId, userId = conn.user.id, arrayBlocks = data.block; - var i; - var checkRes = yield* _checkLock(ctx, docId, arrayBlocks); - var documentLocks = checkRes.documentLocks; - if (checkRes.res) { - //Ok. take lock - var toCache = []; - for (i = 0; i < arrayBlocks.length; ++i) { - var block = arrayBlocks[i]; - var elem = {time: Date.now(), user: userId, block: block}; - documentLocks[block] = elem; - toCache.push(elem); + let docId = conn.docId, userId = conn.user.id, arrayBlocks = data.block; + let locks = arrayBlocks.reduce(function(map, block) { + //todo use one id + map[block.guid || block] = {time: Date.now(), user: userId, block: block}; + return map; + }, {}); + let addRes = await editorData.addLocksNX(ctx, docId, locks); + let documentLocks = addRes.allLocks; + let isAllAdded = Object.keys(addRes.lockConflict).length === 0; + if (!isAllAdded || !fCheckLock(ctx, docId, documentLocks, locks, arrayBlocks, userId)) { + //remove new locks + let toRemove = {}; + for (let lockId in locks) { + if (!addRes.lockConflict[lockId]) { + toRemove[lockId] = locks[lockId]; + delete documentLocks[lockId]; + } } - yield editorData.addLocks(ctx, docId, toCache); - } else if (bIsRestore) { - return false; - } - //to the one who made the request we return as quickly as possible - sendData(ctx, conn, {type: "getLock", locks: documentLocks}); - yield* publish(ctx, {type: commonDefines.c_oPublishType.getLock, ctx: ctx, docId: docId, userId: userId, documentLocks: documentLocks}, docId, userId); - return true; - } - - // For Excel block is now object { sheetId, type, rangeOrObjectId, guid } - function* getLockExcel(ctx, conn, data, bIsRestore) { - var docId = conn.docId, userId = conn.user.id, arrayBlocks = data.block; - var i; - var checkRes = yield* _checkLockExcel(ctx, docId, arrayBlocks, userId); - var documentLocks = checkRes.documentLocks; - if (checkRes.res) { - //Ok. take lock - var toCache = []; - for (i = 0; i < arrayBlocks.length; ++i) { - var block = arrayBlocks[i]; - var elem = {time: Date.now(), user: userId, block: block}; - documentLocks.push(elem); - toCache.push(elem); + await editorData.removeLocks(ctx, docId, toRemove); + if (bIsRestore) { + return false; } - yield editorData.addLocks(ctx, docId, toCache); - } else if (bIsRestore) { - return false; } - //to the one who made the request we return as quickly as possible sendData(ctx, conn, {type: "getLock", locks: documentLocks}); - yield* publish(ctx, {type: commonDefines.c_oPublishType.getLock, ctx: ctx, docId: docId, userId: userId, documentLocks: documentLocks}, docId, userId); - return true; - } - - // For presentations, this is an object { type, val } or { type, slideId, objId } - function* getLockPresentation(ctx, conn, data, bIsRestore) { - var docId = conn.docId, userId = conn.user.id, arrayBlocks = data.block; - var i; - var checkRes = yield* _checkLockPresentation(ctx, docId, arrayBlocks, userId); - var documentLocks = checkRes.documentLocks; - if (checkRes.res) { - //Ok. take lock - var toCache = []; - for (i = 0; i < arrayBlocks.length; ++i) { - var block = arrayBlocks[i]; - var elem = {time: Date.now(), user: userId, block: block}; - documentLocks.push(elem); - toCache.push(elem); - } - yield editorData.addLocks(ctx, docId, toCache); - } else if (bIsRestore) { - return false; - } - //to the one who made the request we return as quickly as possible - sendData(ctx, conn, {type: "getLock", locks: documentLocks}); - yield* publish(ctx, {type: commonDefines.c_oPublishType.getLock, ctx: ctx, docId: docId, userId: userId, documentLocks: documentLocks}, docId, userId); + await publish(ctx, {type: commonDefines.c_oPublishType.getLock, ctx: ctx, docId: docId, userId: userId, documentLocks: documentLocks}, docId, userId); return true; } @@ -3216,14 +3184,10 @@ exports.install = function(server, callbackFunction) { const oRecalcIndexRows = _addRecalcIndex(tmpAdditionalInfo["indexRows"]); // Now we need to recalculate indexes for lock elements if (null !== oRecalcIndexColumns || null !== oRecalcIndexRows) { - const docLock = yield* getAllLocks(ctx, docId); - if (_recalcLockArray(userId, docLock, oRecalcIndexColumns, oRecalcIndexRows)) { - let toCache = []; - for (let i = 0; i < docLock.length; ++i) { - toCache.push(docLock[i]); - } - yield editorData.removeLocks(ctx, docId); - yield editorData.addLocks(ctx, docId, toCache); + let docLock = yield editorData.getLocks(ctx, docId); + let docLockMod = _recalcLockArray(userId, docLock, oRecalcIndexColumns, oRecalcIndexRows); + if (Object.keys(docLockMod).length > 0) { + yield editorData.addLocks(ctx, docId, docLockMod); } } } @@ -3231,7 +3195,7 @@ exports.install = function(server, callbackFunction) { let userLocks = []; if (data.releaseLocks) { //Release locks - userLocks = yield* removeUserLocks(ctx, docId, userId); + userLocks = yield removeUserLocks(ctx, docId, userId); } // For this user, we remove Lock from the document if the unlock flag has arrived const checkEndAuthLockRes = yield* checkEndAuthLock(ctx, data.unlock, false, docId, userId); @@ -3252,14 +3216,14 @@ exports.install = function(server, callbackFunction) { value.time = value.time.getTime(); }) } - yield* publish(ctx, {type: commonDefines.c_oPublishType.changes, ctx: ctx, docId: docId, userId: userId, + yield publish(ctx, {type: commonDefines.c_oPublishType.changes, ctx: ctx, docId: docId, userId: userId, changes: changesToSend, startIndex: startIndex, changesIndex: puckerIndex, syncChangesIndex: puckerIndex, locks: arrLocks, excelAdditionalInfo: data.excelAdditionalInfo, endSaveChanges: data.endSaveChanges}, docId, userId); } // Automatically remove the lock ourselves and send the index to save yield* unSaveLock(ctx, conn, changesIndex, newChangesLastTime, puckerIndex); //last save - let changeInfo = getExternalChangeInfo(conn.user, newChangesLastTime); + let changeInfo = getExternalChangeInfo(conn.user, newChangesLastTime, conn.lang); yield resetForceSaveAfterChanges(ctx, docId, newChangesLastTime, puckerIndex, utils.getBaseUrlByConnection(ctx, conn), changeInfo); } else { let changesToSend = arrNewDocumentChanges; @@ -3270,13 +3234,13 @@ exports.install = function(server, callbackFunction) { value.time = value.time.getTime(); }) } - let isPublished = yield* publish(ctx, {type: commonDefines.c_oPublishType.changes, ctx: ctx, docId: docId, userId: userId, + let isPublished = yield publish(ctx, {type: commonDefines.c_oPublishType.changes, ctx: ctx, docId: docId, userId: userId, changes: changesToSend, startIndex: startIndex, changesIndex: puckerIndex, syncChangesIndex: puckerIndex, locks: [], excelAdditionalInfo: undefined, endSaveChanges: data.endSaveChanges}, docId, userId); sendData(ctx, conn, {type: 'savePartChanges', changesIndex: changesIndex, syncChangesIndex: puckerIndex}); if (!isPublished) { //stub for lockDocumentsTimerId - yield* publish(ctx, {type: commonDefines.c_oPublishType.changesNotify, ctx: ctx, docId: docId}); + yield publish(ctx, {type: commonDefines.c_oPublishType.changesNotify, ctx: ctx, docId: docId}); } } } @@ -3332,45 +3296,21 @@ exports.install = function(server, callbackFunction) { sendDataMessage(ctx, conn, allMessages); } - function* _checkLock(ctx, docId, arrayBlocks) { - // Data is array now - var isLock = false; - var allLocks = yield* getAllLocks(ctx, docId); - var documentLocks = {}; - for(var i = 0 ; i < allLocks.length; ++i) { - var elem = allLocks[i]; - documentLocks[elem.block] =elem; - } - if (arrayBlocks.length > 0) { - for (var i = 0; i < arrayBlocks.length; ++i) { - var block = arrayBlocks[i]; - ctx.logger.info("getLock id: %s", block); - if (documentLocks.hasOwnProperty(block) && documentLocks[block] !== null) { - isLock = true; - break; - } - } - } else { - isLock = true; - } - return {res: !isLock, documentLocks: documentLocks}; + function _checkLockWord(ctx, docId, documentLocks, newLocks, arrayBlocks, userId) { + return true; } - - function* _checkLockExcel(ctx, docId, arrayBlocks, userId) { + function _checkLockExcel(ctx, docId, documentLocks, newLocks, arrayBlocks, userId) { // Data is array now var documentLock; var isLock = false; var isExistInArray = false; var i, blockRange; - var documentLocks = yield* getAllLocks(ctx, docId); var lengthArray = (arrayBlocks) ? arrayBlocks.length : 0; for (i = 0; i < lengthArray && false === isLock; ++i) { blockRange = arrayBlocks[i]; - for (var keyLockInArray in documentLocks) { - if (true === isLock) { - break; - } - if (!documentLocks.hasOwnProperty(keyLockInArray)) { + for (let keyLockInArray in documentLocks) { + if (newLocks[keyLockInArray]) { + //skip just added continue; } documentLock = documentLocks[keyLockInArray]; @@ -3407,41 +3347,43 @@ exports.install = function(server, callbackFunction) { continue; } isLock = compareExcelBlock(blockRange, documentLock.block); + if (true === isLock) { + break; + } } } if (0 === lengthArray) { isLock = true; } - return {res: !isLock && !isExistInArray, documentLocks: documentLocks}; + return !isLock && !isExistInArray; } - function* _checkLockPresentation(ctx, docId, arrayBlocks, userId) { + function _checkLockPresentation(ctx, docId, documentLocks, newLocks, arrayBlocks, userId) { // Data is array now var isLock = false; - var i, documentLock, blockRange; - var documentLocks = yield* getAllLocks(ctx, docId); + var i, blockRange; var lengthArray = (arrayBlocks) ? arrayBlocks.length : 0; for (i = 0; i < lengthArray && false === isLock; ++i) { blockRange = arrayBlocks[i]; - for (var keyLockInArray in documentLocks) { - if (true === isLock) { - break; - } - if (!documentLocks.hasOwnProperty(keyLockInArray)) { + for (let keyLockInArray in documentLocks) { + if (newLocks[keyLockInArray]) { + //skip just added continue; } - documentLock = documentLocks[keyLockInArray]; - + let documentLock = documentLocks[keyLockInArray]; if (documentLock.user === userId || !(documentLock.block)) { continue; } isLock = comparePresentationBlock(blockRange, documentLock.block); + if (true === isLock) { + break; + } } } if (0 === lengthArray) { isLock = true; } - return {res: !isLock, documentLocks: documentLocks}; + return !isLock; } function _checkLicense(ctx, conn) { @@ -3508,13 +3450,13 @@ exports.install = function(server, callbackFunction) { if (licenseInfo.usersCount) { const nowUTC = getLicenseNowUtc(); if(isLiveViewer) { - const arrUsers = yield editorData.getPresenceUniqueViewUser(ctx, nowUTC); + const arrUsers = yield editorStat.getPresenceUniqueViewUser(ctx, nowUTC); if (arrUsers.length >= licenseInfo.usersViewCount && (-1 === arrUsers.findIndex((element) => {return element.userid === userId}))) { licenseType = c_LR.UsersViewCount; } licenseWarningLimitUsersView = licenseInfo.usersViewCount * tenWarningLimitPercents <= arrUsers.length; } else { - const arrUsers = yield editorData.getPresenceUniqueUser(ctx, nowUTC); + const arrUsers = yield editorStat.getPresenceUniqueUser(ctx, nowUTC); if (arrUsers.length >= licenseInfo.usersCount && (-1 === arrUsers.findIndex((element) => {return element.userid === userId}))) { licenseType = c_LR.UsersCount; } @@ -3522,14 +3464,14 @@ exports.install = function(server, callbackFunction) { } } else if(isLiveViewer) { const connectionsLiveCount = licenseInfo.connectionsView; - const liveViewerConnectionsCount = yield editorData.getLiveViewerConnectionsCount(ctx, connections); + const liveViewerConnectionsCount = yield editorStat.getLiveViewerConnectionsCount(ctx, connections); if (liveViewerConnectionsCount >= connectionsLiveCount) { licenseType = c_LR.ConnectionsLive; } licenseWarningLimitConnectionsLive = connectionsLiveCount * tenWarningLimitPercents <= liveViewerConnectionsCount; } else { const connectionsCount = licenseInfo.connections; - const editConnectionsCount = yield editorData.getEditorConnectionsCount(ctx, connections); + const editConnectionsCount = yield editorStat.getEditorConnectionsCount(ctx, connections); if (editConnectionsCount >= connectionsCount) { licenseType = c_LR.Connections; } @@ -3725,7 +3667,8 @@ exports.install = function(server, callbackFunction) { case commonDefines.c_oPublishType.shutdown: //flag prevent new socket connections and receive data from exist connections shutdownFlag = data.status; - ctx.logger.warn('start shutdown:%b', shutdownFlag); + wopiClient.setIsShutdown(shutdownFlag); + ctx.logger.warn('start shutdown:%s', shutdownFlag); if (shutdownFlag) { ctx.logger.warn('active connections: %d', connections.length); //do not stop the server, because sockets and all requests will be unavailable @@ -3773,7 +3716,7 @@ exports.install = function(server, callbackFunction) { if (hasChanges) { let participants = yield getParticipantMap(ctx, data.docId); let participantsTimestamp = Date.now(); - yield* publish(ctx, {type: commonDefines.c_oPublishType.participantsState, ctx: ctx, docId: data.docId, userId: null, participantsTimestamp: participantsTimestamp, participants: participants}); + yield publish(ctx, {type: commonDefines.c_oPublishType.participantsState, ctx: ctx, docId: data.docId, userId: null, participantsTimestamp: participantsTimestamp, participants: participants}); } break; case commonDefines.c_oPublishType.rpc: @@ -3793,7 +3736,7 @@ exports.install = function(server, callbackFunction) { function* collectStats(ctx, countEdit, countLiveView, countView) { let now = Date.now(); - yield editorData.setEditorConnections(ctx, countEdit, countLiveView, countView, now, PRECISION); + yield editorStat.setEditorConnections(ctx, countEdit, countLiveView, countView, now, PRECISION); } function expireDoc() { return co(function* () { @@ -3870,16 +3813,16 @@ exports.install = function(server, callbackFunction) { ctx.setTenant(tenantId); let tenant = tenants[tenantId]; yield* collectStats(ctx, tenant.countEditByShard, tenant.countLiveViewByShard, tenant.countViewByShard); - yield editorData.setEditorConnectionsCountByShard(ctx, SHARD_ID, tenant.countEditByShard); - yield editorData.setLiveViewerConnectionsCountByShard(ctx, SHARD_ID, tenant.countLiveViewByShard); - yield editorData.setViewerConnectionsCountByShard(ctx, SHARD_ID, tenant.countViewByShard); + yield editorStat.setEditorConnectionsCountByShard(ctx, SHARD_ID, tenant.countEditByShard); + yield editorStat.setLiveViewerConnectionsCountByShard(ctx, SHARD_ID, tenant.countLiveViewByShard); + yield editorStat.setViewerConnectionsCountByShard(ctx, SHARD_ID, tenant.countViewByShard); if (clientStatsD) { //todo with multitenant - let countEdit = yield editorData.getEditorConnectionsCount(ctx, connections); + let countEdit = yield editorStat.getEditorConnectionsCount(ctx, connections); clientStatsD.gauge('expireDoc.connections.edit', countEdit); - let countLiveView = yield editorData.getLiveViewerConnectionsCount(ctx, connections); + let countLiveView = yield editorStat.getLiveViewerConnectionsCount(ctx, connections); clientStatsD.gauge('expireDoc.connections.liveview', countLiveView); - let countView = yield editorData.getViewerConnectionsCount(ctx, connections); + let countView = yield editorStat.getViewerConnectionsCount(ctx, connections); clientStatsD.gauge('expireDoc.connections.view', countView); } } @@ -3890,9 +3833,9 @@ exports.install = function(server, callbackFunction) { aggregationCtx.init(tenantManager.getDefautTenant(), ctx.docId, ctx.userId); //yield ctx.initTenantCache();//no need yield* collectStats(aggregationCtx, countEditByShard, countLiveViewByShard, countViewByShard); - yield editorData.setEditorConnectionsCountByShard(aggregationCtx, SHARD_ID, countEditByShard); - yield editorData.setLiveViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, countLiveViewByShard); - yield editorData.setViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, countViewByShard); + yield editorStat.setEditorConnectionsCountByShard(aggregationCtx, SHARD_ID, countEditByShard); + yield editorStat.setLiveViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, countLiveViewByShard); + yield editorStat.setViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, countViewByShard); } ctx.initDefault(); } catch (err) { @@ -3972,13 +3915,13 @@ exports.install = function(server, callbackFunction) { if (checkResult.includes(false)) { return; } - - editorData.connect().then( - () => { - callbackFunction(); - }, - error => operationContext.global.logger.error('editorData error: %s', error.stack) - ); + editorData + .connect() + .then(() => editorStat.connect()) + .then(() => callbackFunction()) + .catch(err => { + operationContext.global.logger.error('editorData error: %s', err.stack); + }); }, error => operationContext.global.logger.error('getTableColumns error: %s', error.stack) ); @@ -4000,13 +3943,18 @@ exports.healthCheck = function(req, res) { yield sqlBase.healthCheck(ctx); ctx.logger.debug('healthCheck database'); //check redis connection - if (editorData.isConnected()) { - yield editorData.ping(); + const healthData = yield editorData.healthCheck(); + if (healthData) { ctx.logger.debug('healthCheck editorData'); } else { - throw new Error('redis disconnected'); + throw new Error('editorData'); + } + const healthStat = yield editorStat.healthCheck(); + if (healthStat) { + ctx.logger.debug('healthCheck editorStat'); + } else { + throw new Error('editorStat'); } - const healthPubsub = yield pubsub.healthCheck(); if (healthPubsub) { ctx.logger.debug('healthCheck pubsub'); @@ -4021,18 +3969,12 @@ exports.healthCheck = function(req, res) { } //storage - const clusterId = cluster.isWorker ? cluster.worker.id : ''; - const tempName = 'hc_' + os.hostname() + '_' + clusterId + '_' + Math.round(Math.random() * HEALTH_CHECK_KEY_MAX); - const tempBuffer = Buffer.from([1, 2, 3, 4, 5]); - //It's proper to putObject one tempName - yield storage.putObject(ctx, tempName, tempBuffer, tempBuffer.length); - try { - //try to prevent case, when another process can remove same tempName - yield storage.deleteObject(ctx, tempName); - } catch (err) { - ctx.logger.warn('healthCheck error %s', err.stack); - } + yield storage.healthCheck(ctx); ctx.logger.debug('healthCheck storage'); + if (storage.isDiffrentPersistentStorage()) { + yield storage.healthCheck(ctx, cfgForgottenFiles); + ctx.logger.debug('healthCheck storage persistent'); + } output = true; ctx.logger.info('healthCheck end'); @@ -4094,7 +4036,7 @@ exports.licenseInfo = function(req, res) { view: {min: 0, avr: 0, max: 0} }; } - var redisRes = yield editorData.getEditorConnections(ctx); + var redisRes = yield editorStat.getEditorConnections(ctx); const now = Date.now(); if (redisRes.length > 0) { let expDocumentsStep95 = expDocumentsStep * 0.95; @@ -4155,8 +4097,8 @@ exports.licenseInfo = function(req, res) { } const nowUTC = getLicenseNowUtc(); let execRes; - execRes = yield editorData.getPresenceUniqueUser(ctx, nowUTC); - output.quota.edit.connectionsCount = yield editorData.getEditorConnectionsCount(ctx, connections); + execRes = yield editorStat.getPresenceUniqueUser(ctx, nowUTC); + output.quota.edit.connectionsCount = yield editorStat.getEditorConnectionsCount(ctx, connections); output.quota.edit.usersCount.unique = execRes.length; execRes.forEach(function(elem) { if (elem.anonym) { @@ -4164,8 +4106,8 @@ exports.licenseInfo = function(req, res) { } }); - execRes = yield editorData.getPresenceUniqueViewUser(ctx, nowUTC); - output.quota.view.connectionsCount = yield editorData.getLiveViewerConnectionsCount(ctx, connections); + execRes = yield editorStat.getPresenceUniqueViewUser(ctx, nowUTC); + output.quota.view.connectionsCount = yield editorStat.getLiveViewerConnectionsCount(ctx, connections); output.quota.view.usersCount.unique = execRes.length; execRes.forEach(function(elem) { if (elem.anonym) { @@ -4173,8 +4115,8 @@ exports.licenseInfo = function(req, res) { } }); - let byMonth = yield editorData.getPresenceUniqueUsersOfMonth(ctx); - let byMonthView = yield editorData.getPresenceUniqueViewUsersOfMonth(ctx); + let byMonth = yield editorStat.getPresenceUniqueUsersOfMonth(ctx); + let byMonthView = yield editorStat.getPresenceUniqueViewUsersOfMonth(ctx); let byMonthMerged = []; for (let i in byMonth) { if (byMonth.hasOwnProperty(i)) { @@ -4256,8 +4198,8 @@ function* findForgottenFile(ctx, docId) { function* commandLicense(ctx) { const nowUTC = getLicenseNowUtc(); - const users = yield editorData.getPresenceUniqueUser(ctx, nowUTC); - const users_view = yield editorData.getPresenceUniqueViewUser(ctx, nowUTC); + const users = yield editorStat.getPresenceUniqueUser(ctx, nowUTC); + const users_view = yield editorStat.getPresenceUniqueViewUser(ctx, nowUTC); const licenseInfo = yield tenantManager.getTenantLicense(ctx); return { @@ -4294,7 +4236,7 @@ function* commandHandle(ctx, params, req, output) { } case 'drop': { if (params.userid) { - yield* publish(ctx, {type: commonDefines.c_oPublishType.drop, ctx: ctx, docId: docId, users: [params.userid], description: params.description}); + yield publish(ctx, {type: commonDefines.c_oPublishType.drop, ctx: ctx, docId: docId, users: [params.userid], description: params.description}); } else if (params.users) { const users = (typeof params.users === 'string') ? JSON.parse(params.users) : params.users; yield* dropUsersFromDocument(ctx, docId, users); @@ -4321,7 +4263,7 @@ function* commandHandle(ctx, params, req, output) { } case 'meta': { if (params.meta) { - yield* publish(ctx, {type: commonDefines.c_oPublishType.meta, ctx: ctx, docId: docId, meta: params.meta}); + yield publish(ctx, {type: commonDefines.c_oPublishType.meta, ctx: ctx, docId: docId, meta: params.meta}); } else { output.error = commonDefines.c_oAscServerCommandErrors.UnknownCommand; } @@ -4415,7 +4357,7 @@ exports.shutdown = function(req, res) { ctx.initFromRequest(req); yield ctx.initTenantCache(); ctx.logger.info('shutdown start'); - output = yield shutdown.shutdown(ctx, editorData, req.method === 'PUT'); + output = yield shutdown.shutdown(ctx, editorStat, req.method === 'PUT'); } catch (err) { ctx.logger.error('shutdown error %s', err.stack); } finally { diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index f055a235..7a7f968b 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -53,6 +53,7 @@ var statsDClient = require('./../../Common/sources/statsdclient'); var operationContext = require('./../../Common/sources/operationContext'); var tenantManager = require('./../../Common/sources/tenantManager'); var config = require('config'); +const path = require("path"); const cfgTypesUpload = config.get('services.CoAuthoring.utils.limits_image_types_upload'); const cfgImageSize = config.get('services.CoAuthoring.server.limits_image_size'); @@ -70,6 +71,7 @@ const cfgAssemblyFormatAsOrigin = config.get('services.CoAuthoring.server.assemb const cfgDownloadMaxBytes = config.get('FileConverter.converter.maxDownloadBytes'); const cfgDownloadTimeout = config.get('FileConverter.converter.downloadTimeout'); const cfgDownloadFileAllowExt = config.get('services.CoAuthoring.server.downloadFileAllowExt'); +const cfgNewFileTemplate = config.get('services.CoAuthoring.server.newFileTemplate'); var SAVE_TYPE_PART_START = 0; var SAVE_TYPE_PART = 1; @@ -160,7 +162,7 @@ function getOpenedAt(row) { function getOpenedAtJSONParams(row) { let openedAt = getOpenedAt(row); if (openedAt) { - return JSON.stringify({'documentLayout': {'openedAt': openedAt}}); + return {'documentLayout': {'openedAt': openedAt}}; } return undefined; } @@ -278,17 +280,13 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd outputData.setData(statusInfo); break; case commonDefines.FileStatus.Err: + outputData.setStatus('err'); + outputData.setData(statusInfo); + break; case commonDefines.FileStatus.ErrToReload: outputData.setStatus('err'); outputData.setData(statusInfo); - if (commonDefines.FileStatus.ErrToReload == status) { - let userAuthStr = sqlBase.UserCallback.prototype.getCallbackByUserIndex(ctx, row.callback); - let wopiParams = wopiClient.parseWopiCallback(ctx, userAuthStr); - if (!wopiParams) { - //todo rework ErrToReload to clean up on next open - yield cleanupCache(ctx, key); - } - } + yield cleanupErrToReload(ctx, key); break; case commonDefines.FileStatus.None: //this status has no handler @@ -377,38 +375,40 @@ function getSaveTask(ctx, cmd) { //} return queueData; } -function* getUpdateResponse(ctx, cmd) { +async function getUpdateResponse(ctx, cmd) { const tenOpenProtectedFile = ctx.getCfg('services.CoAuthoring.server.openProtectedFile', cfgOpenProtectedFile); var updateTask = new taskResult.TaskResultData(); updateTask.tenant = ctx.tenant; updateTask.key = cmd.getSaveKey() ? cmd.getSaveKey() : cmd.getDocId(); var statusInfo = cmd.getStatusInfo(); - if (constants.NO_ERROR == statusInfo) { + if (constants.NO_ERROR === statusInfo) { updateTask.status = commonDefines.FileStatus.Ok; let password = cmd.getPassword(); if (password) { if (false === hasPasswordCol) { - let selectRes = yield taskResult.select(ctx, updateTask.key); + let selectRes = await taskResult.select(ctx, updateTask.key); hasPasswordCol = selectRes.length > 0 && undefined !== selectRes[0].password; } if(hasPasswordCol) { updateTask.password = password; } } - } else if (constants.CONVERT_DOWNLOAD == statusInfo) { + } else if (constants.CONVERT_DOWNLOAD === statusInfo) { updateTask.status = commonDefines.FileStatus.ErrToReload; - } else if (constants.CONVERT_NEED_PARAMS == statusInfo) { + } else if (constants.CONVERT_LIMITS === statusInfo) { + updateTask.status = commonDefines.FileStatus.ErrToReload; + } else if (constants.CONVERT_NEED_PARAMS === statusInfo) { updateTask.status = commonDefines.FileStatus.NeedParams; - } else if (constants.CONVERT_DRM == statusInfo || constants.CONVERT_PASSWORD == statusInfo) { + } else if (constants.CONVERT_DRM === statusInfo || constants.CONVERT_PASSWORD === statusInfo) { if (tenOpenProtectedFile) { updateTask.status = commonDefines.FileStatus.NeedPassword; } else { updateTask.status = commonDefines.FileStatus.Err; } - } else if (constants.CONVERT_DRM_UNSUPPORTED == statusInfo) { + } else if (constants.CONVERT_DRM_UNSUPPORTED === statusInfo) { updateTask.status = commonDefines.FileStatus.Err; - } else if (constants.CONVERT_DEAD_LETTER == statusInfo) { + } else if (constants.CONVERT_DEAD_LETTER === statusInfo) { updateTask.status = commonDefines.FileStatus.ErrToReload; } else { updateTask.status = commonDefines.FileStatus.Err; @@ -439,6 +439,14 @@ var cleanupCacheIf = co.wrap(function* (ctx, mask) { ctx.logger.debug("cleanupCacheIf db.affectedRows=%d", removeRes.affectedRows); return res; }); +async function cleanupErrToReload(ctx, key) { + let updateTask = new taskResult.TaskResultData(); + updateTask.tenant = ctx.tenant; + updateTask.key = key; + updateTask.status = commonDefines.FileStatus.None; + updateTask.statusInfo = constants.NO_ERROR; + await taskResult.update(ctx, updateTask); +} function commandOpenStartPromise(ctx, docId, baseUrl, opt_documentCallbackUrl, opt_format) { var task = new taskResult.TaskResultData(); @@ -616,10 +624,11 @@ let commandSfctByCmd = co.wrap(function*(ctx, cmd, opt_priority, opt_expiration, var selectRes = yield taskResult.select(ctx, cmd.getDocId()); var row = selectRes.length > 0 ? selectRes[0] : null; if (!row) { - return; + return false; } if (opt_initShardKey) { ctx.setShardKey(sqlBase.DocumentAdditional.prototype.getShardKey(row.additional)); + ctx.setWopiSrc(sqlBase.DocumentAdditional.prototype.getWopiSrc(row.additional)); } yield* addRandomKeyTaskCmd(ctx, cmd); addPasswordToCmd(ctx, cmd, row.password); @@ -627,11 +636,12 @@ let commandSfctByCmd = co.wrap(function*(ctx, cmd, opt_priority, opt_expiration, let userAuthStr = sqlBase.UserCallback.prototype.getCallbackByUserIndex(ctx, row.callback); cmd.setWopiParams(wopiClient.parseWopiCallback(ctx, userAuthStr, row.callback)); cmd.setOutputFormat(changeFormatByOrigin(ctx, row, cmd.getOutputFormat())); - cmd.setJsonParams(getOpenedAtJSONParams(row)); + cmd.appendJsonParams(getOpenedAtJSONParams(row)); var queueData = getSaveTask(ctx, cmd); queueData.setFromChanges(true); let priority = null != opt_priority ? opt_priority : constants.QUEUE_PRIORITY_LOW; yield* docsCoServer.addTask(queueData, priority, opt_queue, opt_expiration); + return true; }); function isDisplayedImage(strName) { var res = 0; @@ -827,6 +837,7 @@ function* commandSaveFromOrigin(ctx, cmd, outputData, password) { if (docPassword.initial) { cmd.setPassword(docPassword.initial); } + //todo setLCID in browser var queueData = getSaveTask(ctx, cmd); queueData.setFromOrigin(true); queueData.setFromChanges(true); @@ -847,8 +858,11 @@ function* commandSetPassword(ctx, conn, cmd, outputData) { hasDocumentPassword = true; } } - ctx.logger.debug('commandSetPassword isEnterCorrectPassword=%s, hasDocumentPassword=%s, hasPasswordCol=%s', conn.isEnterCorrectPassword, hasDocumentPassword, hasPasswordCol); - if (tenOpenProtectedFile && (conn.isEnterCorrectPassword || !hasDocumentPassword) && hasPasswordCol) { + //https://github.com/ONLYOFFICE/web-apps/blob/4a7879b4f88f315fe94d9f7d97c0ed8aa9f82221/apps/documenteditor/main/app/controller/Main.js#L1652 + //this.appOptions.isPasswordSupport = this.appOptions.isEdit && this.api.asc_isProtectionSupport() && (this.permissions.protect!==false); + let isPasswordSupport = tenOpenProtectedFile && !conn.user?.view && false !== conn.permissions?.protect; + ctx.logger.debug('commandSetPassword isEnterCorrectPassword=%s, hasDocumentPassword=%s, hasPasswordCol=%s, isPasswordSupport=%s', conn.isEnterCorrectPassword, hasDocumentPassword, hasPasswordCol, isPasswordSupport); + if (isPasswordSupport && (conn.isEnterCorrectPassword || !hasDocumentPassword) && hasPasswordCol) { let updateMask = new taskResult.TaskResultData(); updateMask.tenant = ctx.tenant; updateMask.key = cmd.getDocId(); @@ -861,7 +875,7 @@ function* commandSetPassword(ctx, conn, cmd, outputData) { task.password = cmd.getPassword() || ""; let changeInfo = null; if (conn.user) { - changeInfo = task.innerPasswordChange = docsCoServer.getExternalChangeInfo(conn.user, newChangesLastDate.getTime()); + changeInfo = task.innerPasswordChange = docsCoServer.getExternalChangeInfo(conn.user, newChangesLastDate.getTime(), conn.lang); } var upsertRes = yield taskResult.updateIf(ctx, task, updateMask); @@ -1066,8 +1080,17 @@ const commandSfcCallback = co.wrap(function*(ctx, cmd, isSfcm, isEncrypted) { } else { try { if (wopiParams) { - let isAutoSave = forceSaveType !== commonDefines.c_oAscForceSaveTypes.Button && forceSaveType !== commonDefines.c_oAscForceSaveTypes.Form; - replyStr = yield processWopiPutFile(ctx, docId, wopiParams, savePathDoc, userLastChangeId, true, isAutoSave, false); + if (outputSfc.getUrl()) { + if (forceSaveType === commonDefines.c_oAscForceSaveTypes.Form) { + yield processWopiSaveAs(ctx, cmd); + replyStr = JSON.stringify({error: 0}); + } else { + let isAutoSave = forceSaveType !== commonDefines.c_oAscForceSaveTypes.Button && forceSaveType !== commonDefines.c_oAscForceSaveTypes.Form; + replyStr = yield processWopiPutFile(ctx, docId, wopiParams, savePathDoc, userLastChangeId, true, isAutoSave, false); + } + } else { + replyStr = JSON.stringify({error: 1, descr: "wopi: no file"}); + } } else { replyStr = yield docsCoServer.sendServerRequest(ctx, uri, outputSfc, checkAndFixAuthorizationLength); } @@ -1100,7 +1123,11 @@ const commandSfcCallback = co.wrap(function*(ctx, cmd, isSfcm, isEncrypted) { updateMask.statusInfo = updateIfTask.statusInfo; try { if (wopiParams) { - replyStr = yield processWopiPutFile(ctx, docId, wopiParams, savePathDoc, userLastChangeId, !notModified, false, true); + if (outputSfc.getUrl()) { + replyStr = yield processWopiPutFile(ctx, docId, wopiParams, savePathDoc, userLastChangeId, !notModified, false, true); + } else { + replyStr = JSON.stringify({error: 1, descr: "wopi: no file"}); + } } else { replyStr = yield docsCoServer.sendServerRequest(ctx, uri, outputSfc, checkAndFixAuthorizationLength); } @@ -1167,6 +1194,8 @@ const commandSfcCallback = co.wrap(function*(ctx, cmd, isSfcm, isEncrypted) { } if (!isSfcm) { //todo simultaneous opening + //clean redis (redisKeyPresenceSet and redisKeyPresenceHash removed with last element) + yield docsCoServer.editorData.cleanDocumentOnExit(ctx, docId); //to unlock wopi file yield docsCoServer.unlockWopiDoc(ctx, docId, callbackUserIndex); //cleanupRes can be false in case of simultaneous opening. it is OK @@ -1194,7 +1223,7 @@ const commandSfcCallback = co.wrap(function*(ctx, cmd, isSfcm, isEncrypted) { if ((docsCoServer.getIsShutdown() && !isSfcm) || cmd.getRedisKey()) { let keyRedis = cmd.getRedisKey() ? cmd.getRedisKey() : redisKeyShutdown; - yield docsCoServer.editorData.removeShutdown(keyRedis, docId); + yield docsCoServer.editorStat.removeShutdown(keyRedis, docId); } ctx.logger.debug('End commandSfcCallback'); return replyStr; @@ -1513,7 +1542,10 @@ function getPrintFileUrl(ctx, docId, baseUrl, filename) { let userFriendlyName = encodeURIComponent(filename.replace(/\//g, "%2f")); let res = `${baseUrl}/printfile/${encodeURIComponent(docId)}/${userFriendlyName}?token=${encodeURIComponent(token)}`; if (ctx.shardKey) { - res += `&${constants.SHARED_KEY_NAME}=${encodeURIComponent(ctx.shardKey)}`; + res += `&${constants.SHARD_KEY_API_NAME}=${encodeURIComponent(ctx.shardKey)}`; + } + if (ctx.wopiSrc) { + res += `&${constants.SHARD_KEY_WOPI_NAME}=${encodeURIComponent(ctx.wopiSrc)}`; } res += `&filename=${userFriendlyName}`; return res; @@ -1594,10 +1626,12 @@ exports.downloadFile = function(req, res) { const tenDownloadMaxBytes = ctx.getCfg('FileConverter.converter.maxDownloadBytes', cfgDownloadMaxBytes); const tenDownloadTimeout = ctx.getCfg('FileConverter.converter.downloadTimeout', cfgDownloadTimeout); const tenDownloadFileAllowExt = ctx.getCfg('services.CoAuthoring.server.downloadFileAllowExt', cfgDownloadFileAllowExt); + const tenNewFileTemplate = ctx.getCfg('services.CoAuthoring.server.newFileTemplate', cfgNewFileTemplate); let authorization; let isInJwtToken = false; let errorDescription; + let headers, fromTemplate; let authRes = yield docsCoServer.getRequestParams(ctx, req); if (authRes.code === constants.NO_ERROR) { let decoded = authRes.params; @@ -1610,6 +1644,21 @@ exports.downloadFile = function(req, res) { } else if (decoded.url && -1 !== tenDownloadFileAllowExt.indexOf(decoded.fileType)) { url = decoded.url; isInJwtToken = true; + } else if (wopiClient.isWopiJwtToken(decoded)) { + if (decoded.fileInfo.Size === 0) { + //editnew case + fromTemplate = pathModule.extname(decoded.fileInfo.BaseFileName).substring(1); + } else { + ({url, headers} = yield wopiClient.getWopiFileUrl(ctx, decoded.fileInfo, decoded.userAuth)); + let filterStatus = yield wopiClient.checkIpFilter(ctx, url); + if (0 === filterStatus) { + //todo false? (true because it passed checkIpFilter for wopi) + //todo use directIfIn + isInJwtToken = true; + } else { + errorDescription = 'access deny'; + } + } } else if (!tenTokenEnableBrowser) { //todo token required if (decoded.url) { @@ -1627,45 +1676,59 @@ exports.downloadFile = function(req, res) { res.sendStatus(403); return; } - if (utils.canIncludeOutboxAuthorization(ctx, url)) { - let secret = yield tenantManager.getTenantSecret(ctx, commonDefines.c_oAscSecretType.Outbox); - authorization = utils.fillJwtForRequest(ctx, {url: url}, secret, false); - } - let urlParsed = urlModule.parse(url); - let filterStatus = yield* utils.checkHostFilter(ctx, urlParsed.hostname); - if (0 !== filterStatus) { - ctx.logger.warn('Error downloadFile checkIpFilter error: url = %s', url); - res.sendStatus(filterStatus); - return; - } - let headers; - if (req.get('Range')) { - headers = { - 'Range': req.get('Range') + if (fromTemplate) { + ctx.logger.debug('downloadFile from file template: %s', fromTemplate); + let locale = constants.TEMPLATES_DEFAULT_LOCALE; + let fileTemplatePath = pathModule.join(tenNewFileTemplate, locale, 'new.' + fromTemplate); + res.sendFile(pathModule.resolve(fileTemplatePath)); + } else { + if (utils.canIncludeOutboxAuthorization(ctx, url)) { + let secret = yield tenantManager.getTenantSecret(ctx, commonDefines.c_oAscSecretType.Outbox); + authorization = utils.fillJwtForRequest(ctx, {url: url}, secret, false); + } + let urlParsed = urlModule.parse(url); + let filterStatus = yield* utils.checkHostFilter(ctx, urlParsed.hostname); + if (0 !== filterStatus) { + ctx.logger.warn('Error downloadFile checkIpFilter error: url = %s', url); + res.sendStatus(filterStatus); + return; } - } - yield utils.downloadUrlPromise(ctx, url, tenDownloadTimeout, tenDownloadMaxBytes, authorization, isInJwtToken, headers, res); + if (req.get('Range')) { + if (!headers) { + headers = {}; + } + headers['Range'] = req.get('Range'); + } + + yield utils.downloadUrlPromise(ctx, url, tenDownloadTimeout, tenDownloadMaxBytes, authorization, isInJwtToken, headers, res); + } if (clientStatsD) { clientStatsD.timing('coauth.downloadFile', new Date() - startDate); } } catch (err) { - ctx.logger.error('Error downloadFile: %s', err.stack); - //catch errors because status may be sent while piping to response - try { - if (err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT') { - res.sendStatus(408); - } else if (err.code === 'EMSGSIZE') { - res.sendStatus(413); - } else if (err.response) { - res.sendStatus(err.response.statusCode); - } else { - res.sendStatus(400); - } - } catch (err) { + if (err.code === "ERR_STREAM_PREMATURE_CLOSE") { + ctx.logger.debug('Error downloadFile: %s', err.stack); + } else { ctx.logger.error('Error downloadFile: %s', err.stack); + //catch errors because status may be sent while piping to response + if (!res.headersSent) { + try { + if (err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT') { + res.sendStatus(408); + } else if (err.code === 'EMSGSIZE') { + res.sendStatus(413); + } else if (err.response) { + res.sendStatus(err.response.statusCode); + } else { + res.sendStatus(400); + } + } catch (err) { + ctx.logger.error('Error downloadFile: %s', err.stack); + } + } } } finally { @@ -1673,7 +1736,7 @@ exports.downloadFile = function(req, res) { } }); }; -exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId, opt_userIndex, opt_queue, opt_initShardKey) { +exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId, opt_userIndex, opt_userLcid, opt_queue, opt_initShardKey) { return co(function* () { try { var startDate = null; @@ -1690,6 +1753,7 @@ exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId } if (opt_initShardKey) { ctx.setShardKey(sqlBase.DocumentAdditional.prototype.getShardKey(row.additional)); + ctx.setWopiSrc(sqlBase.DocumentAdditional.prototype.getWopiSrc(row.additional)); } var cmd = new commonDefines.InputCommand(); cmd.setCommand('sfc'); @@ -1698,7 +1762,9 @@ exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId cmd.setStatusInfoIn(statusInfo); cmd.setUserActionId(opt_userId); cmd.setUserActionIndex(opt_userIndex); - cmd.setJsonParams(getOpenedAtJSONParams(row)); + cmd.appendJsonParams(getOpenedAtJSONParams(row)); + //todo lang and region are different + cmd.setLCID(opt_userLcid); let userAuthStr = sqlBase.UserCallback.prototype.getCallbackByUserIndex(ctx, row.callback); cmd.setWopiParams(wopiClient.parseWopiCallback(ctx, userAuthStr, row.callback)); addPasswordToCmd(ctx, cmd, row && row.password); @@ -1708,7 +1774,7 @@ exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId queueData.setFromChanges(true); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_NORMAL, opt_queue); if (docsCoServer.getIsShutdown()) { - yield docsCoServer.editorData.addShutdown(redisKeyShutdown, docId); + yield docsCoServer.editorStat.addShutdown(redisKeyShutdown, docId); } ctx.logger.debug('AddTask saveFromChanges'); } else { @@ -1725,6 +1791,18 @@ exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId } }); }; + +async function processWopiSaveAs(ctx, cmd) { + const info = await docsCoServer.getCallback(ctx, cmd.getDocId(), cmd.getUserIndex()); + // info.wopiParams is null if it is not wopi + if (info.wopiParams) { + const suggestedTargetType = `.${formatChecker.getStringFromFormat(cmd.getOutputFormat())}`; + const storageFilePath = `${cmd.getSaveKey()}/${cmd.getOutputPath()}`; + const stream = await storage.createReadStream(ctx, storageFilePath); + const { wopiSrc, access_token } = info.wopiParams.userAuth; + await wopiClient.putRelativeFile(ctx, wopiSrc, access_token, null, stream.readStream, stream.contentLength, suggestedTargetType, false); + } +} exports.receiveTask = function(data, ack) { return co(function* () { let ctx = new operationContext.Context(); @@ -1735,29 +1813,33 @@ exports.receiveTask = function(data, ack) { ctx.initFromTaskQueueData(task); yield ctx.initTenantCache(); ctx.logger.info('receiveTask start: %s', data); - var updateTask = yield* getUpdateResponse(ctx, cmd); + var updateTask = yield getUpdateResponse(ctx, cmd); var updateRes = yield taskResult.update(ctx, updateTask); if (updateRes.affectedRows > 0) { var outputData = new OutputData(cmd.getCommand()); var command = cmd.getCommand(); var additionalOutput = {needUrlKey: null, needUrlMethod: null, needUrlType: null, needUrlIsCorrectPassword: undefined, creationDate: undefined, openedAt: undefined}; - if ('open' == command || 'reopen' == command) { + if ('open' === command || 'reopen' === command) { yield getOutputData(ctx, cmd, outputData, cmd.getDocId(), null, additionalOutput); - } else if ('save' == command || 'savefromorigin' == command) { - yield getOutputData(ctx, cmd, outputData, cmd.getSaveKey(), null, additionalOutput); - } else if ('sfcm' == command) { + } else if ('save' === command || 'savefromorigin' === command) { + let status = yield getOutputData(ctx, cmd, outputData, cmd.getSaveKey(), null, additionalOutput); + if (commonDefines.FileStatus.Ok === status && cmd.getIsSaveAs()) { + yield processWopiSaveAs(ctx, cmd); + //todo in case of wopi no need to send url. send it to avoid stubs in sdk + } + } else if ('sfcm' === command) { yield commandSfcCallback(ctx, cmd, true); - } else if ('sfc' == command) { + } else if ('sfc' === command) { yield commandSfcCallback(ctx, cmd, false); - } else if ('sendmm' == command) { + } else if ('sendmm' === command) { yield* commandSendMMCallback(ctx, cmd); - } else if ('conv' == command) { + } else if ('conv' === command) { //nothing } if (outputData.getStatus()) { ctx.logger.debug('receiveTask publish: %s', JSON.stringify(outputData)); var output = new OutputDataWrap('documentOpen', outputData); - yield* docsCoServer.publish(ctx, { + yield docsCoServer.publish(ctx, { type: commonDefines.c_oPublishType.receiveTask, ctx: ctx, cmd: cmd, output: output, needUrlKey: additionalOutput.needUrlKey, needUrlMethod: additionalOutput.needUrlMethod, @@ -1780,6 +1862,7 @@ exports.receiveTask = function(data, ack) { exports.cleanupCache = cleanupCache; exports.cleanupCacheIf = cleanupCacheIf; +exports.cleanupErrToReload = cleanupErrToReload; exports.getOpenedAt = getOpenedAt; exports.commandSfctByCmd = commandSfctByCmd; exports.commandOpenStartPromise = commandOpenStartPromise; diff --git a/DocService/sources/changes2forgotten.js b/DocService/sources/changes2forgotten.js index e0014b1d..e42b5269 100644 --- a/DocService/sources/changes2forgotten.js +++ b/DocService/sources/changes2forgotten.js @@ -44,7 +44,9 @@ const operationContext = require('./../../Common/sources/operationContext'); const sqlBase = require('./databaseConnectors/baseConnector'); const docsCoServer = require('./DocsCoServer'); const taskResult = require('./taskresult'); -const editorDataStorage = require('./' + config.get('services.CoAuthoring.server.editorDataStorage')); +const cfgEditorDataStorage = config.get('services.CoAuthoring.server.editorDataStorage'); +const cfgEditorStatStorage = config.get('services.CoAuthoring.server.editorStatStorage'); +const editorStatStorage = require('./' + (cfgEditorStatStorage || cfgEditorDataStorage)); const cfgForgottenFiles = config.get('services.CoAuthoring.server.forgottenfiles'); const cfgTableResult = config.get('services.CoAuthoring.sql.tableResult'); @@ -81,7 +83,7 @@ function shutdown() { var res = true; let ctx = new operationContext.Context(); try { - let editorData = new editorDataStorage(); + let editorStat = editorStatStorage.EditorStat ? new editorStatStorage.EditorStat() : new editorStatStorage(); ctx.logger.debug('shutdown start:' + EXEC_TIMEOUT); //redisKeyShutdown is not a simple counter, so it doesn't get decremented by a build that started before Shutdown started @@ -130,9 +132,9 @@ function shutdown() { yield ctx.initTenantCache(); yield updateDoc(ctx, docId, commonDefines.FileStatus.Ok, ""); - yield editorData.addShutdown(redisKeyShutdown, docId); + yield editorStat.addShutdown(redisKeyShutdown, docId); ctx.logger.debug('shutdown createSaveTimerPromise %s', docId); - yield docsCoServer.createSaveTimer(ctx, docId, null, null, queue, true); + yield docsCoServer.createSaveTimer(ctx, docId, null, null, null, queue, true); } ctx.initDefault(); //sleep because of bugs in createSaveTimerPromise @@ -140,7 +142,7 @@ function shutdown() { let startTime = new Date().getTime(); while (true) { - let remainingFiles = yield editorData.getShutdownCount(redisKeyShutdown); + let remainingFiles = yield editorStat.getShutdownCount(redisKeyShutdown); ctx.logger.debug('shutdown remaining files:%d', remainingFiles); let curTime = new Date().getTime() - startTime; if (curTime >= EXEC_TIMEOUT || remainingFiles <= 0) { @@ -169,7 +171,7 @@ function shutdown() { //todo needs to check queues, because there may be long conversions running before Shutdown //clean up - yield editorData.cleanupShutdown(redisKeyShutdown); + yield editorStat.cleanupShutdown(redisKeyShutdown); yield pubsub.close(); yield queue.close(); diff --git a/DocService/sources/converterservice.js b/DocService/sources/converterservice.js index 27d6aab8..a6bb2a4d 100644 --- a/DocService/sources/converterservice.js +++ b/DocService/sources/converterservice.js @@ -35,7 +35,6 @@ const path = require('path'); var config = require('config'); var co = require('co'); -const locale = require('windows-locale'); const mime = require('mime'); var taskResult = require('./taskresult'); var utils = require('./../../Common/sources/utils'); @@ -50,6 +49,7 @@ var statsDClient = require('./../../Common/sources/statsdclient'); var storageBase = require('./../../Common/sources/storage-base'); var operationContext = require('./../../Common/sources/operationContext'); const sqlBase = require('./databaseConnectors/baseConnector'); +const utilsDocService = require("./utilsDocService"); const cfgTokenEnableBrowser = config.get('services.CoAuthoring.token.enable.browser'); @@ -83,12 +83,12 @@ function* getConvertStatus(ctx, docId, encryptedUserPassword, selectRes, opt_che } break; case commonDefines.FileStatus.Err: + status.err = row.status_info; + break; case commonDefines.FileStatus.ErrToReload: case commonDefines.FileStatus.NeedPassword: status.err = row.status_info; - if (commonDefines.FileStatus.ErrToReload == row.status || commonDefines.FileStatus.NeedPassword == row.status) { - yield canvasService.cleanupCache(ctx, docId); - } + yield canvasService.cleanupErrToReload(ctx, docId); break; case commonDefines.FileStatus.NeedParams: case commonDefines.FileStatus.SaveVersion: @@ -147,7 +147,8 @@ function* convertByCmd(ctx, cmd, async, opt_fileTo, opt_taskExist, opt_priority, if (!bCreate) { selectRes = yield taskResult.select(ctx, docId); status = yield* getConvertStatus(ctx, cmd.getDocId() ,cmd.getPassword(), selectRes, opt_checkPassword); - } else { + } + if (bCreate || (commonDefines.FileStatus.None === selectRes?.[0]?.status)) { var queueData = new commonDefines.TaskQueueData(); queueData.setCtx(ctx); queueData.setCmd(cmd); @@ -182,8 +183,9 @@ function* convertByCmd(ctx, cmd, async, opt_fileTo, opt_taskExist, opt_priority, return status; } -async function convertFromChanges(ctx, docId, baseUrl, forceSave, externalChangeInfo, opt_userdata, opt_formdata, opt_userConnectionId, - opt_userConnectionDocId, opt_responseKey, opt_priority, opt_expiration, opt_queue, opt_redisKey, opt_initShardKey) { +async function convertFromChanges(ctx, docId, baseUrl, forceSave, externalChangeInfo, opt_userdata, opt_formdata, + opt_userConnectionId, opt_userConnectionDocId, opt_responseKey, opt_priority, + opt_expiration, opt_queue, opt_redisKey, opt_initShardKey, opt_jsonParams) { var cmd = new commonDefines.InputCommand(); cmd.setCommand('sfcm'); cmd.setDocId(docId); @@ -193,6 +195,10 @@ async function convertFromChanges(ctx, docId, baseUrl, forceSave, externalChange cmd.setDelimiter(commonDefines.c_oAscCsvDelimiter.Comma); cmd.setForceSave(forceSave); cmd.setExternalChangeInfo(externalChangeInfo); + if (externalChangeInfo.lang) { + //todo lang and region are different + cmd.setLCID(utilsDocService.localeToLCID(externalChangeInfo.lang)); + } if (opt_userdata) { cmd.setUserData(opt_userdata); } @@ -212,8 +218,14 @@ async function convertFromChanges(ctx, docId, baseUrl, forceSave, externalChange if (opt_redisKey) { cmd.setRedisKey(opt_redisKey); } + if (opt_jsonParams) { + cmd.appendJsonParams(opt_jsonParams); + } - await canvasService.commandSfctByCmd(ctx, cmd, opt_priority, opt_expiration, opt_queue, opt_initShardKey); + let commandSfctByCmdRes = await canvasService.commandSfctByCmd(ctx, cmd, opt_priority, opt_expiration, opt_queue, opt_initShardKey); + if (!commandSfctByCmdRes) { + return new commonDefines.ConvertStatus(constants.UNKNOWN); + } var fileTo = constants.OUTPUT_NAME; let outputExt = formatChecker.getStringFromFormat(cmd.getOutputFormat()); if (outputExt) { @@ -268,6 +280,17 @@ function convertRequest(req, res, isJson) { utils.fillResponse(req, res, new commonDefines.ConvertStatus(constants.CONVERT_PARAMS), isJson); return; } + if (params.pdf) { + if (true === params.pdf.pdfa && constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === outputFormat) { + outputFormat = constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDFA; + } else if (false === params.pdf.pdfa && constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDFA === outputFormat) { + outputFormat = constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF; + } + if (params.pdf.form && (constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === outputFormat || + constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDFA === outputFormat)) { + outputFormat = constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_OFORM_PDF; + } + } var cmd = new commonDefines.InputCommand(); cmd.setCommand('conv'); cmd.setUrl(params.url); @@ -281,8 +304,8 @@ function convertRequest(req, res, isJson) { cmd.setDelimiter(parseIntParam(params.delimiter) || commonDefines.c_oAscCsvDelimiter.Comma); if(undefined != params.delimiterChar) cmd.setDelimiterChar(params.delimiterChar); - if (params.region && locale[params.region.toLowerCase()]) { - cmd.setLCID(locale[params.region.toLowerCase()].id); + if (params.region) { + cmd.setLCID(utilsDocService.localeToLCID(params.region)); } let jsonParams = {}; if (params.documentLayout) { @@ -295,7 +318,7 @@ function convertRequest(req, res, isJson) { jsonParams['watermark'] = params.watermark; } if (Object.keys(jsonParams).length > 0) { - cmd.setJsonParams(JSON.stringify(jsonParams)); + cmd.appendJsonParams(jsonParams); } if (params.password) { if (params.password.length > constants.PASSWORD_MAX_LENGTH) { @@ -530,22 +553,22 @@ function convertTo(req, res) { cmd.setOutputFormat(outputFormat); cmd.setCodepage(commonDefines.c_oAscCodePageUtf8); cmd.setDelimiter(commonDefines.c_oAscCsvDelimiter.Comma); - if (lang && locale[lang.toLowerCase()]) { - cmd.setLCID(locale[lang.toLowerCase()].id); + if (lang) { + cmd.setLCID(utilsDocService.localeToLCID(lang)); } if (fullSheetPreview) { - cmd.setJsonParams(JSON.stringify({'spreadsheetLayout': { + cmd.appendJsonParams({'spreadsheetLayout': { "ignorePrintArea": true, "fitToWidth": 1, "fitToHeight": 1 - }})); + }}); } else { - cmd.setJsonParams(JSON.stringify({'spreadsheetLayout': { + cmd.appendJsonParams({'spreadsheetLayout': { "ignorePrintArea": true, "fitToWidth": 0, "fitToHeight": 0, "scale": 100 - }})); + }}); } if (password) { let encryptedPassword = yield utils.encryptPassword(ctx, password); @@ -651,7 +674,7 @@ function getConverterHtmlHandler(req, res) { ctx.setDocId(docId); if (!(wopiSrc && access_token && access_token && targetext && docId) || constants.AVS_OFFICESTUDIO_FILE_UNKNOWN === formatChecker.getFormatFromString(targetext)) { - ctx.logger.debug('convert-and-edit-handler invalid params: wopiSrc=%s; access_token=%s; targetext=%s; docId=%s', wopiSrc, access_token, targetext, docId); + ctx.logger.debug('convert-and-edit-handler invalid params: WOPISrc=%s; access_token=%s; targetext=%s; docId=%s', wopiSrc, access_token, targetext, docId); utils.fillResponse(req, res, new commonDefines.ConvertStatus(constants.CONVERT_PARAMS), isJson); return; } diff --git a/DocService/sources/databaseConnectors/connectorUtilities.js b/DocService/sources/databaseConnectors/connectorUtilities.js index bfa09374..6ff41dcd 100644 --- a/DocService/sources/databaseConnectors/connectorUtilities.js +++ b/DocService/sources/databaseConnectors/connectorUtilities.js @@ -200,6 +200,23 @@ DocumentAdditional.prototype.getShardKey = function(str) { return res; }; +DocumentAdditional.prototype.setWopiSrc = function(wopiSrc) { + let additional = new DocumentAdditional(); + additional.data.push({wopiSrc}); + return additional.toSQLInsert(); +}; +DocumentAdditional.prototype.getWopiSrc = function(str) { + let res; + let val = new DocumentAdditional(); + val.fromString(str); + val.data.forEach((elem) => { + if (elem.wopiSrc) { + res = elem.wopiSrc; + } + }); + return res; +}; + module.exports = { UserCallback, DocumentPassword, diff --git a/DocService/sources/databaseConnectors/mysqlConnector.js b/DocService/sources/databaseConnectors/mysqlConnector.js index 05ed5cf0..d46d4511 100644 --- a/DocService/sources/databaseConnectors/mysqlConnector.js +++ b/DocService/sources/databaseConnectors/mysqlConnector.js @@ -32,14 +32,14 @@ 'use strict'; -const mysql = require('mysql2'); +const mysql = require('mysql2/promise'); const connectorUtilities = require('./connectorUtilities'); const config = require('config'); const configSql = config.get('services.CoAuthoring.sql'); -const cfgTableResult = config.get('services.CoAuthoring.sql.tableResult'); +const cfgTableResult = configSql.get('tableResult'); -const pool = mysql.createPool({ +const connectionConfiguration = { host : configSql.get('dbHost'), port : parseInt(configSql.get('dbPort')), user : configSql.get('dbUser'), @@ -49,120 +49,120 @@ const pool = mysql.createPool({ connectionLimit : configSql.get('connectionlimit'), timezone : 'Z', flags : '-FOUND_ROWS' -}); +}; + +const additionalOptions = configSql.get('mysqlExtraOptions'); +const configuration = Object.assign({}, connectionConfiguration, additionalOptions); + +let pool = mysql.createPool(configuration); function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes = false, opt_noLog = false, opt_values = []) { - pool.getConnection(function(connectionError, connection) { - if (connectionError) { - if (!opt_noLog) { - ctx.logger.error('pool.getConnection error: %s', connectionError); - } + return executeQuery(ctx, sqlCommand, opt_values, opt_noModifyRes, opt_noLog).then( + result => callbackFunction?.(null, result), + error => callbackFunction?.(error) + ); +} - callbackFunction?.(connectionError, null); +async function executeQuery(ctx, sqlCommand, values = [], noModifyRes = false, noLog = false) { + let connection = null; + try { + connection = await pool.getConnection(); - return; + const result = await connection.query(sqlCommand, values); + + let output; + if (!noModifyRes) { + output = result[0]?.affectedRows ? { affectedRows: result[0].affectedRows } : result[0]; + } else { + output = result[0]; } - let queryCallback = function (error, result) { - connection.release(); - if (error && !opt_noLog) { - ctx.logger.error('_______________________error______________________'); - ctx.logger.error('sqlQuery: %s sqlCommand: %s', error.code, sqlCommand); - ctx.logger.error(error); - ctx.logger.error('_____________________end_error____________________'); + return output ?? { rows: [], affectedRows: 0 }; + } catch (error) { + if (!noLog) { + ctx.logger.error(`sqlQuery() error while executing query: ${sqlCommand}\n${error.stack}`); + } + + throw error; + } finally { + if (connection) { + try { + // Put the connection back in the pool + connection.release(); + } catch (error) { + if (!noLog) { + ctx.logger.error(`connection.release() error while executing query: ${sqlCommand}\n${error.stack}`); + } } - - let output; - if (!opt_noModifyRes) { - output = result?.affectedRows ? { affectedRows: result.affectedRows } : result; - } else { - output = result; - } - - output = output ?? { rows: [], affectedRows: 0 }; - - callbackFunction?.(error, output); - }; - - connection.query(sqlCommand, opt_values, queryCallback); - }); + } + } } -function closePool() { - return new Promise((resolve, reject) => { - pool.end((error) => { - if (error) { - reject(error); - } else { - resolve(); - } - }); - }); +async function closePool() { + return await pool.end(); } -function addSqlParameter(val, values) { - values.push(val); +function addSqlParameter(parameter, accumulatedArray) { + accumulatedArray.push(parameter); return '?'; } -function concatParams(val1, val2) { - return `CONCAT(COALESCE(${val1}, ''), COALESCE(${val2}, ''))`; +function concatParams(firstParameter, secondParameter) { + return `CONCAT(COALESCE(${firstParameter}, ''), COALESCE(${secondParameter}, ''))`; } -function upsert(ctx, task) { - return new Promise(function(resolve, reject) { - task.completeDefaults(); - let dateNow = new Date(); - let values = []; - let cbInsert = task.callback; - if (task.callback) { - let userCallback = new connectorUtilities.UserCallback(); - userCallback.fromValues(task.userIndex, task.callback); - cbInsert = userCallback.toSQLInsert(); - } - let p0 = addSqlParameter(task.tenant, values); - let p1 = addSqlParameter(task.key, values); - let p2 = addSqlParameter(task.status, values); - let p3 = addSqlParameter(task.statusInfo, values); - let p4 = addSqlParameter(dateNow, values); - let p5 = addSqlParameter(task.userIndex, values); - let p6 = addSqlParameter(task.changeId, values); - let p7 = addSqlParameter(cbInsert, values); - let p8 = addSqlParameter(task.baseurl, values); - let p9 = addSqlParameter(dateNow, values); - var sqlCommand = `INSERT INTO ${cfgTableResult} (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl)`+ - ` VALUES (${p0}, ${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8}) ON DUPLICATE KEY UPDATE` + - ` last_open_date = ${p9}`; - if (task.callback) { - let p10 = addSqlParameter(JSON.stringify(task.callback), values); - sqlCommand += `, callback = CONCAT(callback , '${connectorUtilities.UserCallback.prototype.delimiter}{"userIndex":' , (user_index + 1) , ',"callback":', ${p10}, '}')`; - } - if (task.baseurl) { - let p11 = addSqlParameter(task.baseurl, values); - sqlCommand += `, baseurl = ${p11}`; - } +async function upsert(ctx, task) { + task.completeDefaults(); + const dateNow = new Date(); - sqlCommand += ', user_index = LAST_INSERT_ID(user_index + 1);'; + let cbInsert = task.callback; + if (task.callback) { + const userCallback = new connectorUtilities.UserCallback(); + userCallback.fromValues(task.userIndex, task.callback); + cbInsert = userCallback.toSQLInsert(); + } - sqlQuery(ctx, sqlCommand, function(error, result) { - if (error) { - reject(error); - } else { - const insertId = result.affectedRows === 1 ? task.userIndex : result.insertId; - //if CLIENT_FOUND_ROWS don't specify 1 row is inserted , 2 row is updated, and 0 row is set to its current values - //http://dev.mysql.com/doc/refman/5.7/en/insert-on-duplicate.html - const isInsert = result.affectedRows === 1; + const values = []; + const valuesPlaceholder = [ + addSqlParameter(task.tenant, values), + addSqlParameter(task.key, values), + addSqlParameter(task.status, values), + addSqlParameter(task.statusInfo, values), + addSqlParameter(dateNow, values), + addSqlParameter(task.userIndex, values), + addSqlParameter(task.changeId, values), + addSqlParameter(cbInsert, values), + addSqlParameter(task.baseurl, values) + ]; - resolve({ isInsert, insertId }); - } - }, true, false, values); - }); + let updateStatement = `last_open_date = ${addSqlParameter(dateNow, values)}`; + if (task.callback) { + let callbackPlaceholder = addSqlParameter(JSON.stringify(task.callback), values); + updateStatement += `, callback = CONCAT(callback , '${connectorUtilities.UserCallback.prototype.delimiter}{"userIndex":' , (user_index + 1) , ',"callback":', ${callbackPlaceholder}, '}')`; + } + + if (task.baseurl) { + let baseUrlPlaceholder = addSqlParameter(task.baseurl, values); + updateStatement += `, baseurl = ${baseUrlPlaceholder}`; + } + + updateStatement += ', user_index = LAST_INSERT_ID(user_index + 1);'; + + const sqlCommand = `INSERT INTO ${cfgTableResult} (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl) `+ + `VALUES (${valuesPlaceholder.join(', ')}) ` + + `ON DUPLICATE KEY UPDATE ${updateStatement}`; + + const result = await executeQuery(ctx, sqlCommand, values, true); + const insertId = result.affectedRows === 1 ? task.userIndex : result.insertId; + //if CLIENT_FOUND_ROWS don't specify 1 row is inserted , 2 row is updated, and 0 row is set to its current values + //http://dev.mysql.com/doc/refman/5.7/en/insert-on-duplicate.html + const isInsert = result.affectedRows === 1; + + return { isInsert, insertId }; } -module.exports = { - sqlQuery, - closePool, - addSqlParameter, - concatParams, - upsert -} \ No newline at end of file +module.exports.sqlQuery = sqlQuery; +module.exports.closePool = closePool; +module.exports.addSqlParameter = addSqlParameter; +module.exports.concatParams = concatParams; +module.exports.upsert = upsert; diff --git a/DocService/sources/editorDataMemory.js b/DocService/sources/editorDataMemory.js index 1c72766b..ab76a32e 100644 --- a/DocService/sources/editorDataMemory.js +++ b/DocService/sources/editorDataMemory.js @@ -39,19 +39,29 @@ const tenantManager = require('./../../Common/sources/tenantManager'); const cfgExpMonthUniqueUsers = ms(config.get('services.CoAuthoring.expire.monthUniqueUsers')); +function EditorCommon() { +} +EditorCommon.prototype.connect = async function () {}; +EditorCommon.prototype.isConnected = function() { + return true; +}; +EditorCommon.prototype.ping = async function() {return "PONG"}; +EditorCommon.prototype.close = async function() {}; +EditorCommon.prototype.healthCheck = async function() { + if (this.isConnected()) { + await this.ping(); + return true; + } + return false; +}; + function EditorData() { + EditorCommon.call(this); this.data = {}; this.forceSaveTimer = {}; - this.uniqueUser = {}; - this.uniqueUsersOfMonth = {}; - this.uniqueViewUser = {}; - this.uniqueViewUsersOfMonth = {}; - this.shutdown = {}; - this.stat = {}; } -EditorData.prototype.connect = function() { - return Promise.resolve(); -}; +EditorData.prototype = Object.create(EditorCommon.prototype); +EditorData.prototype.constructor = EditorData; EditorData.prototype._getDocumentData = function(ctx, docId) { let tenantData = this.data[ctx.tenant]; if (!tenantData) { @@ -73,7 +83,7 @@ EditorData.prototype._checkAndLock = function(ctx, name, docId, fencingToken, tt const expireAt = now + ttl * 1000; data[name] = {fencingToken: fencingToken, expireAt: expireAt}; } - return Promise.resolve(res); + return res; }; EditorData.prototype._checkAndUnlock = function(ctx, name, docId, fencingToken) { let data = this._getDocumentData(ctx, docId); @@ -90,106 +100,117 @@ EditorData.prototype._checkAndUnlock = function(ctx, name, docId, fencingToken) res = commonDefines.c_oAscUnlockRes.Empty; delete data[name]; } - return Promise.resolve(res); + return res; }; -EditorData.prototype.addPresence = function(ctx, docId, userId, userInfo) { - return Promise.resolve(); -}; -EditorData.prototype.updatePresence = function(ctx, docId, userId) { - return Promise.resolve(); -}; -EditorData.prototype.removePresence = function(ctx, docId, userId) { - return Promise.resolve(); -}; -EditorData.prototype.getPresence = function(ctx, docId, connections) { +EditorData.prototype.addPresence = async function(ctx, docId, userId, userInfo) {}; +EditorData.prototype.updatePresence = async function(ctx, docId, userId) {}; +EditorData.prototype.removePresence = async function(ctx, docId, userId) {}; +EditorData.prototype.getPresence = async function(ctx, docId, connections) { let hvals = []; - for (let i = 0; i < connections.length; ++i) { - let conn = connections[i]; - if (conn.docId === docId && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { - hvals.push(utils.getConnectionInfoStr(conn)); + if (connections) { + for (let i = 0; i < connections.length; ++i) { + let conn = connections[i]; + if (conn.docId === docId && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { + hvals.push(utils.getConnectionInfoStr(conn)); + } } } - return Promise.resolve(hvals); + return hvals; }; -EditorData.prototype.lockSave = function(ctx, docId, userId, ttl) { +EditorData.prototype.lockSave = async function(ctx, docId, userId, ttl) { return this._checkAndLock(ctx, 'lockSave', docId, userId, ttl); }; -EditorData.prototype.unlockSave = function(ctx, docId, userId) { +EditorData.prototype.unlockSave = async function(ctx, docId, userId) { return this._checkAndUnlock(ctx, 'lockSave', docId, userId); }; -EditorData.prototype.lockAuth = function(ctx, docId, userId, ttl) { +EditorData.prototype.lockAuth = async function(ctx, docId, userId, ttl) { return this._checkAndLock(ctx, 'lockAuth', docId, userId, ttl); }; -EditorData.prototype.unlockAuth = function(ctx, docId, userId) { +EditorData.prototype.unlockAuth = async function(ctx, docId, userId) { return this._checkAndUnlock(ctx, 'lockAuth', docId, userId); }; -EditorData.prototype.getDocumentPresenceExpired = function(now) { - return Promise.resolve([]); -}; -EditorData.prototype.removePresenceDocument = function(ctx, docId) { - return Promise.resolve(); +EditorData.prototype.getDocumentPresenceExpired = async function(now) { + return []; }; +EditorData.prototype.removePresenceDocument = async function(ctx, docId) {}; -EditorData.prototype.addLocks = function(ctx, docId, locks) { +EditorData.prototype.addLocks = async function(ctx, docId, locks) { let data = this._getDocumentData(ctx, docId); if (!data.locks) { - data.locks = []; + data.locks = {}; } - data.locks = data.locks.concat(locks); - return Promise.resolve(); + Object.assign(data.locks, locks); }; -EditorData.prototype.removeLocks = function(ctx, docId) { +EditorData.prototype.addLocksNX = async function(ctx, docId, locks) { + let data = this._getDocumentData(ctx, docId); + if (!data.locks) { + data.locks = {}; + } + let lockConflict = {}; + for (let lockId in locks) { + if (undefined === data.locks[lockId]) { + data.locks[lockId] = locks[lockId]; + } else { + lockConflict[lockId] = locks[lockId]; + } + } + return {lockConflict, allLocks: data.locks}; +}; +EditorData.prototype.removeLocks = async function(ctx, docId, locks) { + let data = this._getDocumentData(ctx, docId); + if (data.locks) { + for (let lockId in locks) { + delete data.locks[lockId]; + } + } +}; +EditorData.prototype.removeAllLocks = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); data.locks = undefined; - return Promise.resolve(); }; -EditorData.prototype.getLocks = function(ctx, docId) { +EditorData.prototype.getLocks = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); - return Promise.resolve(data.locks || []); + return data.locks || {}; }; -EditorData.prototype.addMessage = function(ctx, docId, msg) { +EditorData.prototype.addMessage = async function(ctx, docId, msg) { let data = this._getDocumentData(ctx, docId); if (!data.messages) { data.messages = []; } data.messages.push(msg); - return Promise.resolve(); }; -EditorData.prototype.removeMessages = function(ctx, docId) { +EditorData.prototype.removeMessages = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); data.messages = undefined; - return Promise.resolve(); }; -EditorData.prototype.getMessages = function(ctx, docId) { +EditorData.prototype.getMessages = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); - return Promise.resolve(data.messages || []); + return data.messages || []; }; -EditorData.prototype.setSaved = function(ctx, docId, status) { +EditorData.prototype.setSaved = async function(ctx, docId, status) { let data = this._getDocumentData(ctx, docId); data.saved = status; - return Promise.resolve(); }; -EditorData.prototype.getdelSaved = function(ctx, docId) { +EditorData.prototype.getdelSaved = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); let res = data.saved; - data.saved = undefined; - return Promise.resolve(res); + data.saved = null; + return res; }; -EditorData.prototype.setForceSave = function(ctx, docId, time, index, baseUrl, changeInfo, convertInfo) { +EditorData.prototype.setForceSave = async function(ctx, docId, time, index, baseUrl, changeInfo, convertInfo) { let data = this._getDocumentData(ctx, docId); data.forceSave = {time, index, baseUrl, changeInfo, started: false, ended: false, convertInfo}; - return Promise.resolve(); }; -EditorData.prototype.getForceSave = function(ctx, docId) { +EditorData.prototype.getForceSave = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); - return Promise.resolve(data.forceSave || null); + return data.forceSave || null; }; -EditorData.prototype.checkAndStartForceSave = function(ctx, docId) { +EditorData.prototype.checkAndStartForceSave = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); let res; if (data.forceSave && !data.forceSave.started) { @@ -197,9 +218,9 @@ EditorData.prototype.checkAndStartForceSave = function(ctx, docId) { data.forceSave.ended = false; res = data.forceSave; } - return Promise.resolve(res); + return res; }; -EditorData.prototype.checkAndSetForceSave = function(ctx, docId, time, index, started, ended, convertInfo) { +EditorData.prototype.checkAndSetForceSave = async function(ctx, docId, time, index, started, ended, convertInfo) { let data = this._getDocumentData(ctx, docId); let res; if (data.forceSave && time === data.forceSave.time && index === data.forceSave.index) { @@ -208,15 +229,14 @@ EditorData.prototype.checkAndSetForceSave = function(ctx, docId, time, index, st data.forceSave.convertInfo = convertInfo; res = data.forceSave; } - return Promise.resolve(res); + return res; }; -EditorData.prototype.removeForceSave = function(ctx, docId) { +EditorData.prototype.removeForceSave = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); data.forceSave = undefined; - return Promise.resolve(); }; -EditorData.prototype.cleanDocumentOnExit = function(ctx, docId) { +EditorData.prototype.cleanDocumentOnExit = async function(ctx, docId) { let tenantData = this.data[ctx.tenant]; if (tenantData) { delete tenantData[docId]; @@ -225,10 +245,9 @@ EditorData.prototype.cleanDocumentOnExit = function(ctx, docId) { if (tenantTimer) { delete tenantTimer[docId]; } - return Promise.resolve(); }; -EditorData.prototype.addForceSaveTimerNX = function(ctx, docId, expireAt) { +EditorData.prototype.addForceSaveTimerNX = async function(ctx, docId, expireAt) { let tenantTimer = this.forceSaveTimer[ctx.tenant]; if (!tenantTimer) { this.forceSaveTimer[ctx.tenant] = tenantTimer = {}; @@ -236,9 +255,8 @@ EditorData.prototype.addForceSaveTimerNX = function(ctx, docId, expireAt) { if (!tenantTimer[docId]) { tenantTimer[docId] = expireAt; } - return Promise.resolve(); }; -EditorData.prototype.getForceSaveTimer = function(now) { +EditorData.prototype.getForceSaveTimer = async function(now) { let res = []; for (let tenant in this.forceSaveTimer) { if (this.forceSaveTimer.hasOwnProperty(tenant)) { @@ -253,18 +271,29 @@ EditorData.prototype.getForceSaveTimer = function(now) { } } } - return Promise.resolve(res); + return res; }; -EditorData.prototype.addPresenceUniqueUser = function(ctx, userId, expireAt, userInfo) { +function EditorStat() { + EditorCommon.call(this); + this.uniqueUser = {}; + this.uniqueUsersOfMonth = {}; + this.uniqueViewUser = {}; + this.uniqueViewUsersOfMonth = {}; + this.stat = {}; + this.shutdown = {}; + this.license = {}; +} +EditorStat.prototype = Object.create(EditorCommon.prototype); +EditorStat.prototype.constructor = EditorStat; +EditorStat.prototype.addPresenceUniqueUser = async function(ctx, userId, expireAt, userInfo) { let tenantUser = this.uniqueUser[ctx.tenant]; if (!tenantUser) { this.uniqueUser[ctx.tenant] = tenantUser = {}; } tenantUser[userId] = {expireAt: expireAt, userInfo: userInfo}; - return Promise.resolve(); }; -EditorData.prototype.getPresenceUniqueUser = function(ctx, nowUTC) { +EditorStat.prototype.getPresenceUniqueUser = async function(ctx, nowUTC) { let res = []; let tenantUser = this.uniqueUser[ctx.tenant]; if (!tenantUser) { @@ -282,9 +311,9 @@ EditorData.prototype.getPresenceUniqueUser = function(ctx, nowUTC) { } } } - return Promise.resolve(res); + return res; }; -EditorData.prototype.addPresenceUniqueUsersOfMonth = function(ctx, userId, period, userInfo) { +EditorStat.prototype.addPresenceUniqueUsersOfMonth = async function(ctx, userId, period, userInfo) { let tenantUser = this.uniqueUsersOfMonth[ctx.tenant]; if (!tenantUser) { this.uniqueUsersOfMonth[ctx.tenant] = tenantUser = {}; @@ -294,9 +323,8 @@ EditorData.prototype.addPresenceUniqueUsersOfMonth = function(ctx, userId, perio tenantUser[period] = {expireAt: expireAt, data: {}}; } tenantUser[period].data[userId] = userInfo; - return Promise.resolve(); }; -EditorData.prototype.getPresenceUniqueUsersOfMonth = function(ctx) { +EditorStat.prototype.getPresenceUniqueUsersOfMonth = async function(ctx) { let res = {}; let nowUTC = Date.now(); let tenantUser = this.uniqueUsersOfMonth[ctx.tenant]; @@ -313,18 +341,17 @@ EditorData.prototype.getPresenceUniqueUsersOfMonth = function(ctx) { } } } - return Promise.resolve(res); + return res; }; -EditorData.prototype.addPresenceUniqueViewUser = function(ctx, userId, expireAt, userInfo) { +EditorStat.prototype.addPresenceUniqueViewUser = async function(ctx, userId, expireAt, userInfo) { let tenantUser = this.uniqueViewUser[ctx.tenant]; if (!tenantUser) { this.uniqueViewUser[ctx.tenant] = tenantUser = {}; } tenantUser[userId] = {expireAt: expireAt, userInfo: userInfo}; - return Promise.resolve(); }; -EditorData.prototype.getPresenceUniqueViewUser = function(ctx, nowUTC) { +EditorStat.prototype.getPresenceUniqueViewUser = async function(ctx, nowUTC) { let res = []; let tenantUser = this.uniqueViewUser[ctx.tenant]; if (!tenantUser) { @@ -342,9 +369,9 @@ EditorData.prototype.getPresenceUniqueViewUser = function(ctx, nowUTC) { } } } - return Promise.resolve(res); + return res; }; -EditorData.prototype.addPresenceUniqueViewUsersOfMonth = function(ctx, userId, period, userInfo) { +EditorStat.prototype.addPresenceUniqueViewUsersOfMonth = async function(ctx, userId, period, userInfo) { let tenantUser = this.uniqueViewUsersOfMonth[ctx.tenant]; if (!tenantUser) { this.uniqueViewUsersOfMonth[ctx.tenant] = tenantUser = {}; @@ -354,9 +381,8 @@ EditorData.prototype.addPresenceUniqueViewUsersOfMonth = function(ctx, userId, p tenantUser[period] = {expireAt: expireAt, data: {}}; } tenantUser[period].data[userId] = userInfo; - return Promise.resolve(); }; -EditorData.prototype.getPresenceUniqueViewUsersOfMonth = function(ctx) { +EditorStat.prototype.getPresenceUniqueViewUsersOfMonth = async function(ctx) { let res = {}; let nowUTC = Date.now(); let tenantUser = this.uniqueViewUsersOfMonth[ctx.tenant]; @@ -373,10 +399,9 @@ EditorData.prototype.getPresenceUniqueViewUsersOfMonth = function(ctx) { } } } - return Promise.resolve(res); + return res; }; - -EditorData.prototype.setEditorConnections = function(ctx, countEdit, countLiveView, countView, now, precision) { +EditorStat.prototype.setEditorConnections = async function(ctx, countEdit, countLiveView, countView, now, precision) { let tenantStat = this.stat[ctx.tenant]; if (!tenantStat) { this.stat[ctx.tenant] = tenantStat = []; @@ -387,79 +412,69 @@ EditorData.prototype.setEditorConnections = function(ctx, countEdit, countLiveVi i++; } tenantStat.splice(0, i); - return Promise.resolve(); }; -EditorData.prototype.getEditorConnections = function(ctx) { +EditorStat.prototype.getEditorConnections = async function(ctx) { let tenantStat = this.stat[ctx.tenant]; if (!tenantStat) { this.stat[ctx.tenant] = tenantStat = []; } - return Promise.resolve(tenantStat); + return tenantStat; }; -EditorData.prototype.setEditorConnectionsCountByShard = function(ctx, shardId, count) { - return Promise.resolve(); -}; -EditorData.prototype.incrEditorConnectionsCountByShard = function(ctx, shardId, count) { - return Promise.resolve(); -}; -EditorData.prototype.getEditorConnectionsCount = function(ctx, connections) { +EditorStat.prototype.setEditorConnectionsCountByShard = async function(ctx, shardId, count) {}; +EditorStat.prototype.incrEditorConnectionsCountByShard = async function(ctx, shardId, count) {}; +EditorStat.prototype.getEditorConnectionsCount = async function(ctx, connections) { let count = 0; - for (let i = 0; i < connections.length; ++i) { - let conn = connections[i]; - if (!(conn.isCloseCoAuthoring || (conn.user && conn.user.view)) && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { - count++; + if (connections) { + for (let i = 0; i < connections.length; ++i) { + let conn = connections[i]; + if (!(conn.isCloseCoAuthoring || (conn.user && conn.user.view)) && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { + count++; + } } } - return Promise.resolve(count); + return count; }; -EditorData.prototype.setViewerConnectionsCountByShard = function(ctx, shardId, count) { - return Promise.resolve(); -}; -EditorData.prototype.incrViewerConnectionsCountByShard = function(ctx, shardId, count) { - return Promise.resolve(); -}; -EditorData.prototype.getViewerConnectionsCount = function(ctx, connections) { +EditorStat.prototype.setViewerConnectionsCountByShard = async function(ctx, shardId, count) {}; +EditorStat.prototype.incrViewerConnectionsCountByShard = async function(ctx, shardId, count) {}; +EditorStat.prototype.getViewerConnectionsCount = async function(ctx, connections) { let count = 0; - for (let i = 0; i < connections.length; ++i) { - let conn = connections[i]; - if (conn.isCloseCoAuthoring || (conn.user && conn.user.view) && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { - count++; + if (connections) { + for (let i = 0; i < connections.length; ++i) { + let conn = connections[i]; + if (conn.isCloseCoAuthoring || (conn.user && conn.user.view) && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { + count++; + } } } - return Promise.resolve(count); + return count; }; -EditorData.prototype.setLiveViewerConnectionsCountByShard = function(ctx, shardId, count) { - return Promise.resolve(); -}; -EditorData.prototype.incrLiveViewerConnectionsCountByShard = function(ctx, shardId, count) { - return Promise.resolve(); -}; -EditorData.prototype.getLiveViewerConnectionsCount = function(ctx, connections) { +EditorStat.prototype.setLiveViewerConnectionsCountByShard = async function(ctx, shardId, count) {}; +EditorStat.prototype.incrLiveViewerConnectionsCountByShard = async function(ctx, shardId, count) {}; +EditorStat.prototype.getLiveViewerConnectionsCount = async function(ctx, connections) { let count = 0; - for (let i = 0; i < connections.length; ++i) { - let conn = connections[i]; - if (utils.isLiveViewer(conn) && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { - count++; + if (connections) { + for (let i = 0; i < connections.length; ++i) { + let conn = connections[i]; + if (utils.isLiveViewer(conn) && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { + count++; + } } } - return Promise.resolve(count); + return count; }; - -EditorData.prototype.addShutdown = function(key, docId) { +EditorStat.prototype.addShutdown = async function(key, docId) { if (!this.shutdown[key]) { this.shutdown[key] = {}; } this.shutdown[key][docId] = 1; - return Promise.resolve(); }; -EditorData.prototype.removeShutdown = function(key, docId) { +EditorStat.prototype.removeShutdown = async function(key, docId) { if (!this.shutdown[key]) { this.shutdown[key] = {}; } delete this.shutdown[key][docId]; - return Promise.resolve(); }; -EditorData.prototype.getShutdownCount = function(key) { +EditorStat.prototype.getShutdownCount = async function(key) { let count = 0; if (this.shutdown[key]) { for (let docId in this.shutdown[key]) { @@ -468,31 +483,22 @@ EditorData.prototype.getShutdownCount = function(key) { } } } - return Promise.resolve(count); + return count; }; -EditorData.prototype.cleanupShutdown = function(key) { +EditorStat.prototype.cleanupShutdown = async function(key) { delete this.shutdown[key]; - return Promise.resolve(); +}; +EditorStat.prototype.setLicense = async function(key, val) { + this.license[key] = val; +}; +EditorStat.prototype.getLicense = async function(key) { + return this.license[key] || null; +}; +EditorStat.prototype.removeLicense = async function(key) { + delete this.license[key]; }; -EditorData.prototype.setLicense = function(key, val) { - return Promise.resolve(); -}; -EditorData.prototype.getLicense = function(key) { - return Promise.resolve(null); -}; -EditorData.prototype.removeLicense = function(key) { - return Promise.resolve(); -}; - -EditorData.prototype.isConnected = function() { - return true; -}; -EditorData.prototype.ping = function() { - return Promise.resolve(); -}; -EditorData.prototype.close = function() { - return Promise.resolve(); -}; - -module.exports = EditorData; +module.exports = { + EditorData, + EditorStat +} diff --git a/DocService/sources/gc.js b/DocService/sources/gc.js index 3d2ac1d4..b647ff49 100644 --- a/DocService/sources/gc.js +++ b/DocService/sources/gc.js @@ -78,7 +78,8 @@ var checkFileExpire = function(expireSeconds) { let tenant = expired[i].tenant; let docId = expired[i].id; let shardKey = sqlBase.DocumentAdditional.prototype.getShardKey(expired[i].additional); - ctx.init(tenant, docId, ctx.userId, shardKey); + let wopiSrc = sqlBase.DocumentAdditional.prototype.getWopiSrc(expired[i].additional); + ctx.init(tenant, docId, ctx.userId, shardKey, wopiSrc); yield ctx.initTenantCache(); //todo tenant //check that no one is in the document @@ -125,7 +126,7 @@ var checkDocumentExpire = function() { var hasChanges = yield docsCoServer.hasChanges(ctx, docId); if (hasChanges) { //todo opt_initShardKey from getDocumentPresenceExpired data or from db - yield docsCoServer.createSaveTimer(ctx, docId, null, null, queue, true, true); + yield docsCoServer.createSaveTimer(ctx, docId, null, null, null, queue, true, true); startSaveCount++; } else { yield docsCoServer.cleanDocumentOnExitNoChangesPromise(ctx, docId); diff --git a/DocService/sources/routes/static.js b/DocService/sources/routes/static.js new file mode 100644 index 00000000..6051ae2c --- /dev/null +++ b/DocService/sources/routes/static.js @@ -0,0 +1,104 @@ +/* + * (c) Copyright Ascensio System SIA 2010-2023 + * + * This program is a free software product. You can redistribute it and/or + * modify it under the terms of the GNU Affero General Public License (AGPL) + * version 3 as published by the Free Software Foundation. In accordance with + * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect + * that Ascensio System SIA expressly excludes the warranty of non-infringement + * of any third-party rights. + * + * This program is distributed WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For + * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html + * + * You can contact Ascensio System SIA at 20A-6 Ernesta Birznieka-Upish + * street, Riga, Latvia, EU, LV-1050. + * + * The interactive user interfaces in modified source and object code versions + * of the Program must display Appropriate Legal Notices, as required under + * Section 5 of the GNU AGPL version 3. + * + * Pursuant to Section 7(b) of the License you must retain the original Product + * logo when distributing the program. Pursuant to Section 7(e) we decline to + * grant you any rights under trademark law for use of our trademarks. + * + * All the Product's GUI elements, including illustrations and icon sets, as + * well as technical writing content are licensed under the terms of the + * Creative Commons Attribution-ShareAlike 4.0 International. See the License + * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode + * + */ + +'use strict'; +const express = require('express'); +const config = require("config"); +const operationContext = require('./../../../Common/sources/operationContext'); +const utils = require('./../../../Common/sources/utils'); +const storage = require('./../../../Common/sources/storage-base'); +const urlModule = require("url"); +const path = require("path"); +const mime = require("mime"); + +const cfgStaticContent = config.has('services.CoAuthoring.server.static_content') ? config.get('services.CoAuthoring.server.static_content') : {}; +const cfgCacheStorage = config.get('storage'); +const cfgPersistentStorage = utils.deepMergeObjects({}, cfgCacheStorage, config.get('persistentStorage')); +const cfgForgottenFiles = config.get('services.CoAuthoring.server.forgottenfiles'); +const cfgErrorFiles = config.get('FileConverter.converter.errorfiles'); + +const router = express.Router(); + +function initCacheRouter(cfgStorage, routs) { + const bucketName = cfgStorage.bucketName; + const storageFolderName = cfgStorage.storageFolderName; + const folderPath = cfgStorage.fs.folderPath; + routs.forEach((rout) => { + //special dirs are empty by default + if (!rout) { + return; + } + let rootPath = path.join(folderPath, rout); + router.use(`/${bucketName}/${storageFolderName}/${rout}`, (req, res, next) => { + const index = req.url.lastIndexOf('/'); + if ('GET' === req.method && index > 0) { + let sendFileOptions = { + root: rootPath, dotfiles: 'deny', headers: { + 'Content-Disposition': 'attachment' + } + }; + const urlParsed = urlModule.parse(req.url); + if (urlParsed && urlParsed.pathname) { + const filename = decodeURIComponent(path.basename(urlParsed.pathname)); + sendFileOptions.headers['Content-Type'] = mime.getType(filename); + } + const realUrl = decodeURI(req.url.substring(0, index)); + res.sendFile(realUrl, sendFileOptions, (err) => { + if (err) { + operationContext.global.logger.error(err); + res.status(400).end(); + } + }); + } else { + res.sendStatus(404); + } + }); + }); +} + +for (let i in cfgStaticContent) { + if (cfgStaticContent.hasOwnProperty(i)) { + router.use(i, express.static(cfgStaticContent[i]['path'], cfgStaticContent[i]['options'])); + } +} +if (storage.needServeStatic()) { + initCacheRouter(cfgCacheStorage, [cfgCacheStorage.cacheFolderName]); +} +if (storage.needServeStatic(cfgForgottenFiles)) { + let persistentRouts = [cfgForgottenFiles, cfgErrorFiles]; + persistentRouts.filter((rout) => {return rout && rout.length > 0;}); + if (persistentRouts.length > 0) { + initCacheRouter(cfgPersistentStorage, [cfgForgottenFiles, cfgErrorFiles]); + } +} + +module.exports = router; diff --git a/DocService/sources/server.js b/DocService/sources/server.js index 1a71621d..27a21d86 100644 --- a/DocService/sources/server.js +++ b/DocService/sources/server.js @@ -57,7 +57,7 @@ const utils = require('./../../Common/sources/utils'); const commonDefines = require('./../../Common/sources/commondefines'); const operationContext = require('./../../Common/sources/operationContext'); const tenantManager = require('./../../Common/sources/tenantManager'); -const configStorage = config.get('storage'); +const staticRouter = require('./routes/static'); const cfgWopiEnable = config.get('wopi.enable'); const cfgWopiDummyEnable = config.get('wopi.dummy.enable'); @@ -132,44 +132,6 @@ updateLicense(); fs.watchFile(cfgLicenseFile, updateLicense); setInterval(updateLicense, 86400000); -if (config.has('services.CoAuthoring.server.static_content')) { - const staticContent = config.get('services.CoAuthoring.server.static_content'); - for (let i in staticContent) { - if (staticContent.hasOwnProperty(i)) { - app.use(i, express.static(staticContent[i]['path'], staticContent[i]['options'])); - } - } -} - -if (configStorage.has('fs.folderPath')) { - const cfgBucketName = configStorage.get('bucketName'); - const cfgStorageFolderName = configStorage.get('storageFolderName'); - app.use('/' + cfgBucketName + '/' + cfgStorageFolderName, (req, res, next) => { - const index = req.url.lastIndexOf('/'); - if ('GET' === req.method && index > 0) { - let sendFileOptions = { - root: configStorage.get('fs.folderPath'), dotfiles: 'deny', headers: { - 'Content-Disposition': 'attachment' - } - }; - const urlParsed = urlModule.parse(req.url); - if (urlParsed && urlParsed.pathname) { - const filename = decodeURIComponent(path.basename(urlParsed.pathname)); - sendFileOptions.headers['Content-Type'] = mime.getType(filename); - } - const realUrl = decodeURI(req.url.substring(0, index)); - res.sendFile(realUrl, sendFileOptions, (err) => { - if (err) { - operationContext.global.logger.error(err); - res.status(400).end(); - } - }); - } else { - res.sendStatus(404); - } - }); -} - try { fs.watch(config.get('services.CoAuthoring.plugins.path'), updatePlugins); } catch (e) { @@ -211,6 +173,9 @@ docsCoServer.install(server, () => { } }); }); + + app.use('/', staticRouter); + const rawFileParser = bodyParser.raw( {inflate: true, limit: config.get('services.CoAuthoring.server.limits_tempfile_upload'), type: function() {return true;}}); const urleEcodedParser = bodyParser.urlencoded({ extended: false }); diff --git a/DocService/sources/shutdown.js b/DocService/sources/shutdown.js index ca40908e..6a88fa5b 100644 --- a/DocService/sources/shutdown.js +++ b/DocService/sources/shutdown.js @@ -47,7 +47,7 @@ var WAIT_TIMEOUT = 30000; var LOOP_TIMEOUT = 1000; var EXEC_TIMEOUT = WAIT_TIMEOUT + utils.getConvertionTimeout(undefined); -exports.shutdown = function(ctx, editorData, status) { +exports.shutdown = function(ctx, editorStat, status) { return co(function*() { var res = true; try { @@ -55,7 +55,7 @@ exports.shutdown = function(ctx, editorData, status) { //redisKeyShutdown is not a simple counter, so it doesn't get decremented by a build that started before Shutdown started //reset redisKeyShutdown just in case the previous run didn't finish - yield editorData.cleanupShutdown(redisKeyShutdown); + yield editorStat.cleanupShutdown(redisKeyShutdown); var pubsub = new pubsubService(); yield pubsub.initPromise(); @@ -76,7 +76,7 @@ exports.shutdown = function(ctx, editorData, status) { ctx.logger.debug('shutdown timeout'); break; } - var remainingFiles = yield editorData.getShutdownCount(redisKeyShutdown); + var remainingFiles = yield editorStat.getShutdownCount(redisKeyShutdown); ctx.logger.debug('shutdown remaining files:%d', remainingFiles); if (!isStartWait && remainingFiles <= 0) { break; @@ -85,7 +85,7 @@ exports.shutdown = function(ctx, editorData, status) { } //todo need to check the queues, because there may be long conversions running before Shutdown //clean up - yield editorData.cleanupShutdown(redisKeyShutdown); + yield editorStat.cleanupShutdown(redisKeyShutdown); yield pubsub.close(); ctx.logger.debug('shutdown end'); diff --git a/DocService/sources/utilsDocService.js b/DocService/sources/utilsDocService.js index 0b80f48b..370ad3dd 100644 --- a/DocService/sources/utilsDocService.js +++ b/DocService/sources/utilsDocService.js @@ -34,6 +34,7 @@ const exifParser = require("exif-parser"); const Jimp = require("jimp"); +const locale = require('windows-locale'); async function fixImageExifRotation(ctx, buffer) { if (!buffer) { @@ -59,7 +60,17 @@ async function fixImageExifRotation(ctx, buffer) { } return buffer; } +/** + * + * @param {string} lang + * @returns {number | undefined} + */ +function localeToLCID(lang) { + let elem = locale[lang && lang.toLowerCase()]; + return elem && elem.id; +} module.exports = { - fixImageExifRotation + fixImageExifRotation, + localeToLCID }; \ No newline at end of file diff --git a/DocService/sources/wopiClient.js b/DocService/sources/wopiClient.js index bb053884..1c3a9dec 100644 --- a/DocService/sources/wopiClient.js +++ b/DocService/sources/wopiClient.js @@ -35,6 +35,7 @@ const path = require('path'); const { pipeline } = require('node:stream/promises'); const crypto = require('crypto'); +let util = require('util'); const {URL} = require('url'); const co = require('co'); const jwt = require('jsonwebtoken'); @@ -63,18 +64,22 @@ const cfgTokenEnableBrowser = config.get('services.CoAuthoring.token.enable.brow const cfgCallbackRequestTimeout = config.get('services.CoAuthoring.server.callbackRequestTimeout'); const cfgNewFileTemplate = config.get('services.CoAuthoring.server.newFileTemplate'); const cfgDownloadTimeout = config.get('FileConverter.converter.downloadTimeout'); +const cfgMaxDownloadBytes = config.get('FileConverter.converter.maxDownloadBytes'); const cfgWopiFileInfoBlockList = config.get('wopi.fileInfoBlockList'); const cfgWopiWopiZone = config.get('wopi.wopiZone'); const cfgWopiPdfView = config.get('wopi.pdfView'); +const cfgWopiPdfEdit = config.get('wopi.pdfEdit'); const cfgWopiWordView = config.get('wopi.wordView'); const cfgWopiWordEdit = config.get('wopi.wordEdit'); const cfgWopiCellView = config.get('wopi.cellView'); const cfgWopiCellEdit = config.get('wopi.cellEdit'); const cfgWopiSlideView = config.get('wopi.slideView'); const cfgWopiSlideEdit = config.get('wopi.slideEdit'); +const cfgWopiForms = config.get('wopi.forms'); const cfgWopiFavIconUrlWord = config.get('wopi.favIconUrlWord'); const cfgWopiFavIconUrlCell = config.get('wopi.favIconUrlCell'); const cfgWopiFavIconUrlSlide = config.get('wopi.favIconUrlSlide'); +const cfgWopiFavIconUrlPdf = config.get('wopi.favIconUrlPdf'); const cfgWopiPublicKey = config.get('wopi.publicKey'); const cfgWopiModulus = config.get('wopi.modulus'); const cfgWopiExponent = config.get('wopi.exponent'); @@ -86,8 +91,12 @@ const cfgWopiPrivateKeyOld = config.get('wopi.privateKeyOld'); const cfgWopiHost = config.get('wopi.host'); const cfgWopiDummySampleFilePath = config.get('wopi.dummy.sampleFilePath'); +let cryptoSign = util.promisify(crypto.sign); + let templatesFolderLocalesCache = null; +let templatesFolderExtsCache = null; const templateFilesSizeCache = {}; +let shutdownFlag = false; let mimeTypesByExt = (function() { let mimeTypesByExt = {}; @@ -107,9 +116,24 @@ let mimeTypesByExt = (function() { return mimeTypesByExt; })(); +async function getTemplatesFolderExts(ctx){ + //find available template files + if (templatesFolderExtsCache === null) { + const tenNewFileTemplate = ctx.getCfg('services.CoAuthoring.server.newFileTemplate', cfgNewFileTemplate); + const dirContent = await readdir(`${tenNewFileTemplate}/${constants.TEMPLATES_DEFAULT_LOCALE}/`, { withFileTypes: true }); + templatesFolderExtsCache = dirContent + .filter(dirObject => dirObject.isFile()) + .reduce((result, item, index, array) => { + let ext = path.extname(item.name).substring(1); + result[ext] = ext; + return result; + }, {}); + } + return templatesFolderExtsCache; +} + function discovery(req, res) { return co(function*() { - let output = ''; const xml = xmlbuilder2.create({version: '1.0', encoding: 'utf-8'}); let ctx = new operationContext.Context(); try { @@ -118,15 +142,18 @@ function discovery(req, res) { ctx.logger.info('wopiDiscovery start'); const tenWopiWopiZone = ctx.getCfg('wopi.wopiZone', cfgWopiWopiZone); const tenWopiPdfView = ctx.getCfg('wopi.pdfView', cfgWopiPdfView); + const tenWopiPdfEdit = ctx.getCfg('wopi.pdfEdit', cfgWopiPdfEdit); const tenWopiWordView = ctx.getCfg('wopi.wordView', cfgWopiWordView); const tenWopiWordEdit = ctx.getCfg('wopi.wordEdit', cfgWopiWordEdit); const tenWopiCellView = ctx.getCfg('wopi.cellView', cfgWopiCellView); const tenWopiCellEdit = ctx.getCfg('wopi.cellEdit', cfgWopiCellEdit); const tenWopiSlideView = ctx.getCfg('wopi.slideView', cfgWopiSlideView); const tenWopiSlideEdit = ctx.getCfg('wopi.slideEdit', cfgWopiSlideEdit); + const tenWopiForms = ctx.getCfg('wopi.forms', cfgWopiForms); const tenWopiFavIconUrlWord = ctx.getCfg('wopi.favIconUrlWord', cfgWopiFavIconUrlWord); const tenWopiFavIconUrlCell = ctx.getCfg('wopi.favIconUrlCell', cfgWopiFavIconUrlCell); const tenWopiFavIconUrlSlide = ctx.getCfg('wopi.favIconUrlSlide', cfgWopiFavIconUrlSlide); + const tenWopiFavIconUrlPdf = ctx.getCfg('wopi.favIconUrlSlide', cfgWopiFavIconUrlPdf); const tenWopiPublicKey = ctx.getCfg('wopi.publicKey', cfgWopiPublicKey); const tenWopiModulus = ctx.getCfg('wopi.modulus', cfgWopiModulus); const tenWopiExponent = ctx.getCfg('wopi.exponent', cfgWopiExponent); @@ -136,20 +163,27 @@ function discovery(req, res) { const tenWopiHost = ctx.getCfg('wopi.host', cfgWopiHost); let baseUrl = tenWopiHost || utils.getBaseUrlByRequest(ctx, req); - let names = ['Word','Excel','PowerPoint']; - let favIconUrls = [tenWopiFavIconUrlWord, tenWopiFavIconUrlCell, tenWopiFavIconUrlSlide]; + let names = ['Word','Excel','PowerPoint','Pdf']; + let favIconUrls = [tenWopiFavIconUrlWord, tenWopiFavIconUrlCell, tenWopiFavIconUrlSlide, tenWopiFavIconUrlPdf]; let exts = [ - {targetext: 'docx', view: tenWopiPdfView.concat(tenWopiWordView), edit: tenWopiWordEdit}, + {targetext: 'docx', view: tenWopiWordView, edit: tenWopiWordEdit}, {targetext: 'xlsx', view: tenWopiCellView, edit: tenWopiCellEdit}, - {targetext: 'pptx', view: tenWopiSlideView, edit: tenWopiSlideEdit} + {targetext: 'pptx', view: tenWopiSlideView, edit: tenWopiSlideEdit}, + {targetext: null, view: tenWopiPdfView, edit: tenWopiPdfEdit} ]; + let documentTypes = [`word`, `cell`, `slide`, `pdf`]; + + let templatesFolderExtsCache = yield getTemplatesFolderExts(ctx); + let formsExts = tenWopiForms.reduce((result, item, index, array) => { + result[item] = item; + return result; + }, {}); let templateStart = `${baseUrl}/hosting/wopi`; let templateEnd = `<rs=DC_LLCC&><dchat=DISABLE_CHAT&><embed=EMBEDDED&>`; templateEnd += `<fs=FULLSCREEN&><hid=HOST_SESSION_ID&><rec=RECORDING&>`; templateEnd += `<sc=SESSION_CONTEXT&><thm=THEME_ID&><ui=UI_LLCC&>`; templateEnd += `<wopisrc=WOPI_SOURCE&>&`; - let documentTypes = [`word`, `cell`, `slide`]; let xmlZone = xml.ele('wopi-discovery').ele('net-zone', { name: tenWopiWopiZone }); //start section for MS WOPI connectors for(let i = 0; i < names.length; ++i) { @@ -164,12 +198,13 @@ function discovery(req, res) { let urlTemplateMobileView = `${templateStart}/${documentTypes[i]}/view?mobile=1&${templateEnd}`; let urlTemplateEdit = `${templateStart}/${documentTypes[i]}/edit?${templateEnd}`; let urlTemplateMobileEdit = `${templateStart}/${documentTypes[i]}/edit?mobile=1&${templateEnd}`; + let urlTemplateFormSubmit = `${templateStart}/${documentTypes[i]}/edit?formsubmit=1&${templateEnd}`; let xmlApp = xmlZone.ele('app', {name: name, favIconUrl: favIconUrl}); for (let j = 0; j < ext.view.length; ++j) { - xmlApp.ele('action', {name: 'view', ext: ext.view[j], urlsrc: urlTemplateView}).up(); + xmlApp.ele('action', {name: 'view', ext: ext.view[j], default: 'true', urlsrc: urlTemplateView}).up(); xmlApp.ele('action', {name: 'embedview', ext: ext.view[j], urlsrc: urlTemplateEmbedView}).up(); xmlApp.ele('action', {name: 'mobileView', ext: ext.view[j], urlsrc: urlTemplateMobileView}).up(); - if (-1 === tenWopiPdfView.indexOf(ext.view[j])) { + if (ext.targetext) { let urlConvert = `${templateStart}/convert-and-edit/${ext.view[j]}/${ext.targetext}?${templateEnd}`; xmlApp.ele('action', {name: 'convert', ext: ext.view[j], targetext: ext.targetext, requires: 'update', urlsrc: urlConvert}).up(); } @@ -178,12 +213,17 @@ function discovery(req, res) { xmlApp.ele('action', {name: 'view', ext: ext.edit[j], urlsrc: urlTemplateView}).up(); xmlApp.ele('action', {name: 'embedview', ext: ext.edit[j], urlsrc: urlTemplateEmbedView}).up(); xmlApp.ele('action', {name: 'mobileView', ext: ext.edit[j], urlsrc: urlTemplateMobileView}).up(); - xmlApp.ele('action', {name: 'edit', ext: ext.edit[j], default: 'true', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); + if (formsExts[ext.edit[j]]) { + xmlApp.ele('action', {name: 'formsubmit', ext: ext.edit[j], default: 'true', requires: 'locks,update', urlsrc: urlTemplateFormSubmit}).up(); + xmlApp.ele('action', {name: 'edit', ext: ext.edit[j], requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); + } else { + xmlApp.ele('action', {name: 'edit', ext: ext.edit[j], default: 'true', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); + } xmlApp.ele('action', {name: 'mobileEdit', ext: ext.edit[j], requires: 'locks,update', urlsrc: urlTemplateMobileEdit}).up(); + if (templatesFolderExtsCache[ext.edit[j]]) { + xmlApp.ele('action', {name: 'editnew', ext: ext.edit[j], requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); + } } - constants.SUPPORTED_TEMPLATES_EXTENSIONS[name].forEach( - extension => xmlApp.ele('action', {name: 'editnew', ext: extension, requires: 'locks,update', urlsrc: urlTemplateEdit}).up() - ); xmlApp.up(); } //end section for MS WOPI connectors @@ -195,6 +235,7 @@ function discovery(req, res) { let urlTemplateMobileView = `${templateStart}/${documentTypes[i]}/view?mobile=1&${templateEnd}`; let urlTemplateEdit = `${templateStart}/${documentTypes[i]}/edit?${templateEnd}`; let urlTemplateMobileEdit = `${templateStart}/${documentTypes[i]}/edit?mobile=1&${templateEnd}`; + let urlTemplateFormSubmit = `${templateStart}/${documentTypes[i]}/edit?formsubmit=1&${templateEnd}`; for (let j = 0; j < ext.view.length; ++j) { let mimeTypes = mimeTypesByExt[ext.view[j]]; if (mimeTypes) { @@ -203,7 +244,7 @@ function discovery(req, res) { xmlApp.ele('action', {name: 'view', ext: '', default: 'true', urlsrc: urlTemplateView}).up(); xmlApp.ele('action', {name: 'embedview', ext: '', urlsrc: urlTemplateEmbedView}).up(); xmlApp.ele('action', {name: 'mobileView', ext: '', urlsrc: urlTemplateMobileView}).up(); - if (-1 === tenWopiPdfView.indexOf(ext.view[j])) { + if (ext.targetext) { let urlConvert = `${templateStart}/convert-and-edit/${ext.view[j]}/${ext.targetext}?${templateEnd}`; xmlApp.ele('action', {name: 'convert', ext: '', targetext: ext.targetext, requires: 'update', urlsrc: urlConvert}).up(); } @@ -216,8 +257,16 @@ function discovery(req, res) { if (mimeTypes) { mimeTypes.forEach((value) => { let xmlApp = xmlZone.ele('app', {name: value}); - xmlApp.ele('action', {name: 'edit', ext: '', default: 'true', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); + if (formsExts[ext.edit[j]]) { + xmlApp.ele('action', {name: 'formsubmit', ext: '', default: 'true', requires: 'locks,update', urlsrc: urlTemplateFormSubmit}).up(); + xmlApp.ele('action', {name: 'edit', ext: '', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); + } else { + xmlApp.ele('action', {name: 'edit', ext: '', default: 'true', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); + } xmlApp.ele('action', {name: 'mobileEdit', ext: '', requires: 'locks,update', urlsrc: urlTemplateMobileEdit}).up(); + if (templatesFolderExtsCache[ext.edit[j]]) { + xmlApp.ele('action', {name: 'editnew', ext: '', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); + } xmlApp.up(); }); } @@ -229,9 +278,11 @@ function discovery(req, res) { //end section for collabora nexcloud connectors let xmlDiscovery = xmlZone.up(); if (tenWopiPublicKeyOld && tenWopiPublicKey) { + let exponent = numberToBase64(tenWopiExponent - 0); + let exponentOld = numberToBase64(tenWopiExponentOld - 0); xmlDiscovery.ele('proof-key', { - oldvalue: tenWopiPublicKeyOld, oldmodulus: tenWopiModulusOld, oldexponent: tenWopiExponentOld, - value: tenWopiPublicKey, modulus: tenWopiModulus, exponent: tenWopiExponent + oldvalue: tenWopiPublicKeyOld, oldmodulus: tenWopiModulusOld, oldexponent: exponentOld, + value: tenWopiPublicKey, modulus: tenWopiModulus, exponent: exponent }).up(); } xmlDiscovery.up(); @@ -289,6 +340,28 @@ function getFileTypeByInfo(fileInfo) { fileType = fileInfo.FileExtension ? fileInfo.FileExtension.substr(1) : fileType; return fileType.toLowerCase(); } +async function getWopiFileUrl(ctx, fileInfo, userAuth) { + const tenMaxDownloadBytes = ctx.getCfg('FileConverter.converter.maxDownloadBytes', cfgMaxDownloadBytes); + let url; + let headers = {'X-WOPI-MaxExpectedSize': tenMaxDownloadBytes}; + if (fileInfo?.FileUrl) { + //Requests to the FileUrl can not be signed using proof keys. The FileUrl is used exactly as provided by the host, so it does not necessarily include the access token, which is required to construct the expected proof. + url = fileInfo.FileUrl; + } else if (fileInfo?.TemplateSource) { + url = fileInfo.TemplateSource; + } else if (userAuth) { + url = `${userAuth.wopiSrc}/contents?access_token=${userAuth.access_token}`; + await fillStandardHeaders(ctx, headers, url, userAuth.access_token); + } + ctx.logger.debug('getWopiFileUrl url=%s; headers=%j', url, headers); + return {url, headers}; +} +function isWopiJwtToken(decoded) { + return !!decoded.fileInfo; +} +function setIsShutdown(val) { + shutdownFlag = val; +} function getLastModifiedTimeFromCallbacks(callbacks) { for (let i = callbacks.length; i >= 0; --i) { let callback = callbacks[i]; @@ -443,45 +516,47 @@ function getEditorHtml(req, res) { params.fileInfo = {}; return; } - //save common info - const fileType = getFileTypeByInfo(fileInfo); - if (undefined === lockId) { - lockId = crypto.randomBytes(16).toString('base64'); - let commonInfo = JSON.stringify({lockId: lockId, fileInfo: fileInfo}); - yield canvasService.commandOpenStartPromise(ctx, docId, utils.getBaseUrlByRequest(ctx, req), commonInfo, fileType); - } - - // TODO: throw error if format not supported? - if (fileInfo.Size === 0 && fileType.length !== 0) { - const wopiParams = getWopiParams(undefined, fileInfo, wopiSrc, access_token, access_token_ttl); - - if (templatesFolderLocalesCache === null) { - const dirContent = yield readdir(`${tenNewFileTemplate}/`, { withFileTypes: true }); - templatesFolderLocalesCache = dirContent.filter(dirObject => dirObject.isDirectory()).map(dirObject => dirObject.name); + if (!shutdownFlag) { + //save common info + const fileType = getFileTypeByInfo(fileInfo); + if (undefined === lockId) { + lockId = crypto.randomBytes(16).toString('base64'); + let commonInfo = JSON.stringify({lockId: lockId, fileInfo: fileInfo}); + yield canvasService.commandOpenStartPromise(ctx, docId, utils.getBaseUrlByRequest(ctx, req), commonInfo, fileType); } - const localePrefix = lang || ui || 'en'; - let locale = constants.TEMPLATES_FOLDER_LOCALE_COLLISON_MAP[localePrefix] ?? templatesFolderLocalesCache.find(locale => locale.startsWith(localePrefix)); - if (locale === undefined) { - locale = 'en-US'; + // TODO: throw error if format not supported? + if (fileInfo.Size === 0 && fileType.length !== 0) { + const wopiParams = getWopiParams(undefined, fileInfo, wopiSrc, access_token, access_token_ttl); + + if (templatesFolderLocalesCache === null) { + const dirContent = yield readdir(`${tenNewFileTemplate}/`, { withFileTypes: true }); + templatesFolderLocalesCache = dirContent.filter(dirObject => dirObject.isDirectory()).map(dirObject => dirObject.name); + } + + const localePrefix = lang || ui || 'en'; + let locale = constants.TEMPLATES_FOLDER_LOCALE_COLLISON_MAP[localePrefix] ?? templatesFolderLocalesCache.find(locale => locale.startsWith(localePrefix)); + if (locale === undefined) { + locale = constants.TEMPLATES_DEFAULT_LOCALE; + } + + const filePath = `${tenNewFileTemplate}/${locale}/new.${fileType}`; + if (!templateFilesSizeCache[filePath]) { + templateFilesSizeCache[filePath] = yield lstat(filePath); + } + + const templateFileInfo = templateFilesSizeCache[filePath]; + const templateFileStream = createReadStream(filePath); + yield putFile(ctx, wopiParams, undefined, templateFileStream, templateFileInfo.size, fileInfo.UserId, false, false, false); } - const filePath = `${tenNewFileTemplate}/${locale}/new.${fileType}`; - if (!templateFilesSizeCache[filePath]) { - templateFilesSizeCache[filePath] = yield lstat(filePath); - } - - const templateFileInfo = templateFilesSizeCache[filePath]; - const templateFileStream = createReadStream(filePath); - yield putFile(ctx, wopiParams, undefined, templateFileStream, templateFileInfo.size, fileInfo.UserId, false, false, false); - } - - //Lock - if ('view' !== mode) { - let lockRes = yield lock(ctx, 'LOCK', lockId, fileInfo, userAuth); - if (!lockRes) { - params.fileInfo = {}; - return; + //Lock + if ('view' !== mode) { + let lockRes = yield lock(ctx, 'LOCK', lockId, fileInfo, userAuth); + if (!lockRes) { + params.fileInfo = {}; + return; + } } } @@ -532,7 +607,7 @@ function getConverterHtml(req, res) { let targetext = req.params.targetext; if (!(wopiSrc && access_token && access_token_ttl && ext && targetext)) { - ctx.logger.debug('convert-and-edit invalid params: wopiSrc=%s; access_token=%s; access_token_ttl=%s; ext=%s; targetext=%s', wopiSrc, access_token, access_token_ttl, ext, targetext); + ctx.logger.debug('convert-and-edit invalid params: WOPISrc=%s; access_token=%s; access_token_ttl=%s; ext=%s; targetext=%s', wopiSrc, access_token, access_token_ttl, ext, targetext); return; } @@ -548,15 +623,15 @@ function getConverterHtml(req, res) { if (docId) { let baseUrl = tenWopiHost || utils.getBaseUrlByRequest(ctx, req); params.statusHandler = `${baseUrl}/hosting/wopi/convert-and-edit-handler`; - params.statusHandler += `?wopiSrc=${encodeURI(wopiSrc)}&access_token=${encodeURI(access_token)}`; - params.statusHandler += `&targetext=${encodeURI(targetext)}&docId=${encodeURI(docId)}`; + params.statusHandler += `?${constants.SHARD_KEY_WOPI_NAME}=${encodeURIComponent(wopiSrc)}&access_token=${encodeURIComponent(access_token)}`; + params.statusHandler += `&targetext=${encodeURIComponent(targetext)}&docId=${encodeURIComponent(docId)}`; if (tenTokenEnableBrowser) { let tokenData = {docId: docId}; let options = {algorithm: tenTokenOutboxAlgorithm, expiresIn: tenTokenOutboxExpires}; let secret = yield tenantManager.getTenantSecret(ctx, commonDefines.c_oAscSecretType.Browser); let token = jwt.sign(tokenData, secret, options); - params.statusHandler += `&token=${encodeURI(token)}`; + params.statusHandler += `&token=${encodeURIComponent(token)}`; } } } catch (err) { @@ -596,7 +671,7 @@ function putFile(ctx, wopiParams, data, dataStream, dataSize, userLastChangeId, let commonInfo = wopiParams.commonInfo; //todo add all the users who contributed changes to the document in this PutFile request to X-WOPI-Editors let headers = {'X-WOPI-Override': 'PUT', 'X-WOPI-Lock': commonInfo.lockId, 'X-WOPI-Editors': userLastChangeId}; - fillStandardHeaders(ctx, headers, uri, userAuth.access_token); + yield fillStandardHeaders(ctx, headers, uri, userAuth.access_token); headers['X-LOOL-WOPI-IsModifiedByUser'] = isModifiedByUser; headers['X-LOOL-WOPI-IsAutosave'] = isAutosave; headers['X-LOOL-WOPI-IsExitSave'] = isExitSave; @@ -604,6 +679,7 @@ function putFile(ctx, wopiParams, data, dataStream, dataSize, userLastChangeId, //collabora nexcloud connector headers['X-LOOL-WOPI-Timestamp'] = wopiParams.LastModifiedTime; } + headers['Content-Type'] = mime.getType(getFileTypeByInfo(fileInfo)); ctx.logger.debug('wopi PutFile request uri=%s headers=%j', uri, headers); postRes = yield utils.postRequestPromise(ctx, uri, data, dataStream, dataSize, tenCallbackRequestTimeout, undefined, headers); @@ -633,9 +709,12 @@ function putRelativeFile(ctx, wopiSrc, access_token, data, dataStream, dataSize, return postRes; } - let headers = {'X-WOPI-Override': 'PUT_RELATIVE', 'X-WOPI-SuggestedTarget': utf7.encode(suggestedTarget), - 'X-WOPI-FileConversion': isFileConversion}; - fillStandardHeaders(ctx, headers, uri, access_token); + let headers = {'X-WOPI-Override': 'PUT_RELATIVE', 'X-WOPI-SuggestedTarget': utf7.encode(suggestedTarget)}; + if (isFileConversion) { + headers['X-WOPI-FileConversion'] = isFileConversion; + } + yield fillStandardHeaders(ctx, headers, uri, access_token); + headers['Content-Type'] = mime.getType(suggestedTarget); ctx.logger.debug('wopi putRelativeFile request uri=%s headers=%j', uri, headers); postRes = yield utils.postRequestPromise(ctx, uri, data, dataStream, dataSize, tenCallbackRequestTimeout, undefined, headers); @@ -673,7 +752,7 @@ function renameFile(ctx, wopiParams, name) { let commonInfo = wopiParams.commonInfo; let headers = {'X-WOPI-Override': 'RENAME_FILE', 'X-WOPI-Lock': commonInfo.lockId, 'X-WOPI-RequestedName': utf7.encode(name)}; - fillStandardHeaders(ctx, headers, uri, userAuth.access_token); + yield fillStandardHeaders(ctx, headers, uri, userAuth.access_token); ctx.logger.debug('wopi RenameFile request uri=%s headers=%j', uri, headers); let postRes = yield utils.postRequestPromise(ctx, uri, undefined, undefined, undefined, tenCallbackRequestTimeout, undefined, headers); @@ -711,9 +790,10 @@ function checkFileInfo(ctx, wopiSrc, access_token, opt_sc) { if (opt_sc) { headers['X-WOPI-SessionContext'] = opt_sc; } - fillStandardHeaders(ctx, headers, uri, access_token); + yield fillStandardHeaders(ctx, headers, uri, access_token); ctx.logger.debug('wopi checkFileInfo request uri=%s headers=%j', uri, headers); - //todo false? + //todo false? (true because it passed checkIpFilter for wopi) + //todo use directIfIn let isInJwtToken = true; let getRes = yield utils.downloadUrlPromise(ctx, uri, tenDownloadTimeout, undefined, undefined, isInJwtToken, headers); ctx.logger.debug(`wopi checkFileInfo headers=%j body=%s`, getRes.response.headers, getRes.body); @@ -746,7 +826,7 @@ function lock(ctx, command, lockId, fileInfo, userAuth) { } let headers = {"X-WOPI-Override": command, "X-WOPI-Lock": lockId}; - fillStandardHeaders(ctx, headers, uri, access_token); + yield fillStandardHeaders(ctx, headers, uri, access_token); ctx.logger.debug('wopi %s request uri=%s headers=%j', command, uri, headers); let postRes = yield utils.postRequestPromise(ctx, uri, undefined, undefined, undefined, tenCallbackRequestTimeout, undefined, headers); ctx.logger.debug('wopi %s response headers=%j', command, postRes.response.headers); @@ -783,7 +863,7 @@ function unlock(ctx, wopiParams) { } let headers = {"X-WOPI-Override": "UNLOCK", "X-WOPI-Lock": lockId}; - fillStandardHeaders(ctx, headers, uri, access_token); + yield fillStandardHeaders(ctx, headers, uri, access_token); ctx.logger.debug('wopi Unlock request uri=%s headers=%j', uri, headers); let postRes = yield utils.postRequestPromise(ctx, uri, undefined, undefined, undefined, tenCallbackRequestTimeout, undefined, headers); ctx.logger.debug('wopi Unlock response headers=%j', postRes.response.headers); @@ -816,28 +896,32 @@ function generateProofBuffer(url, accessToken, timeStamp) { buffer.writeBigUInt64BE(timeStamp, offset); return buffer; } -function generateProofSign(url, accessToken, timeStamp, privateKey) { - let signer = crypto.createSign('RSA-SHA256'); - signer.update(generateProofBuffer(url, accessToken, timeStamp)); - return signer.sign({key:privateKey}, "base64"); + +async function generateProofSign(url, accessToken, timeStamp, privateKey) { + let data = generateProofBuffer(url, accessToken, timeStamp); + let sign = await cryptoSign('RSA-SHA256', data, privateKey); + return sign.toString('base64'); } -function generateProof(ctx, url, accessToken, timeStamp) { - const tenWopiPrivateKey = ctx.getCfg('wopi.privateKey', cfgWopiPrivateKey); - let privateKey = `-----BEGIN RSA PRIVATE KEY-----\n${tenWopiPrivateKey}\n-----END RSA PRIVATE KEY-----`; - return generateProofSign(url, accessToken, timeStamp, privateKey); + +function numberToBase64(val) { + // Convert to hexadecimal + let hexString = val.toString(16); + //Ensure the hexadecimal string has an even length + if (hexString.length % 2 !== 0) { + hexString = '0' + hexString; + } + //Convert the hexadecimal string to a buffer + const buffer = Buffer.from(hexString, 'hex'); + return buffer.toString('base64'); } -function generateProofOld(ctx, url, accessToken, timeStamp) { - const tenWopiPrivateKeyOld = ctx.getCfg('wopi.privateKeyOld', cfgWopiPrivateKeyOld); - let privateKey = `-----BEGIN RSA PRIVATE KEY-----\n${tenWopiPrivateKeyOld}\n-----END RSA PRIVATE KEY-----`; - return generateProofSign(url, accessToken, timeStamp, privateKey); -} -function fillStandardHeaders(ctx, headers, url, access_token) { + +async function fillStandardHeaders(ctx, headers, url, access_token) { let timeStamp = utils.getDateTimeTicks(new Date()); const tenWopiPrivateKey = ctx.getCfg('wopi.privateKey', cfgWopiPrivateKey); const tenWopiPrivateKeyOld = ctx.getCfg('wopi.privateKeyOld', cfgWopiPrivateKeyOld); if (tenWopiPrivateKey && tenWopiPrivateKeyOld) { - headers['X-WOPI-Proof'] = generateProof(ctx, url, access_token, timeStamp); - headers['X-WOPI-ProofOld'] = generateProofOld(ctx, url, access_token, timeStamp); + headers['X-WOPI-Proof'] = await generateProofSign(url, access_token, timeStamp, Buffer.from(tenWopiPrivateKey, 'base64')); + headers['X-WOPI-ProofOld'] = await generateProofSign(url, access_token, timeStamp, Buffer.from(tenWopiPrivateKeyOld, 'base64')); headers['X-WOPI-TimeStamp'] = timeStamp; headers['X-WOPI-ClientVersion'] = commonDefines.buildVersion + '.' + commonDefines.buildNumber; // todo @@ -923,7 +1007,12 @@ async function dummyGetFile(req, res) { res, ); } catch (err) { - ctx.logger.error('dummyGetFile error:%s', err.stack); + if (err.code === "ERR_STREAM_PREMATURE_CLOSE") { + //xhr.abort case + ctx.logger.debug('dummyGetFile error: %s', err.stack); + } else { + ctx.logger.error('dummyGetFile error:%s', err.stack); + } } finally { if (!res.headersSent) { res.sendStatus(400); @@ -934,6 +1023,7 @@ function dummyOk(req, res) { res.sendStatus(200); } +exports.checkIpFilter = checkIpFilter; exports.discovery = discovery; exports.collaboraCapabilities = collaboraCapabilities; exports.parseWopiCallback = parseWopiCallback; @@ -944,12 +1034,13 @@ exports.putRelativeFile = putRelativeFile; exports.renameFile = renameFile; exports.lock = lock; exports.unlock = unlock; -exports.generateProof = generateProof; -exports.generateProofOld = generateProofOld; exports.fillStandardHeaders = fillStandardHeaders; exports.getWopiUnlockMarker = getWopiUnlockMarker; exports.getWopiModifiedMarker = getWopiModifiedMarker; exports.getFileTypeByInfo = getFileTypeByInfo; +exports.getWopiFileUrl = getWopiFileUrl; +exports.isWopiJwtToken = isWopiJwtToken; +exports.setIsShutdown = setIsShutdown; exports.dummyCheckFileInfo = dummyCheckFileInfo; exports.dummyGetFile = dummyGetFile; exports.dummyOk = dummyOk; diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index f5afadd6..c3d29628 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -123,7 +123,7 @@ function TaskQueueDataConvert(ctx, task) { this.mailMergeSend = cmd.mailmergesend; this.thumbnail = cmd.thumbnail; this.textParams = cmd.getTextParams(); - this.jsonParams = cmd.getJsonParams(); + this.jsonParams = JSON.stringify(cmd.getJsonParams()); this.lcid = cmd.getLCID(); this.password = cmd.getPassword(); this.savePassword = cmd.getSavePassword(); @@ -341,13 +341,15 @@ function* isUselessConvertion(ctx, task, cmd) { return constants.NO_ERROR; } async function changeFormatToExtendedPdf(ctx, dataConvert, cmd) { + let forceSave = cmd.getForceSave(); + let isSendForm = forceSave && forceSave.getType() === commonDefines.c_oAscForceSaveTypes.Form; let originFormat = cmd.getOriginFormat(); let isOriginFormatWithForms = constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === originFormat || constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_OFORM === originFormat || constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_DOCXF === originFormat; let isFormatToPdf = constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === dataConvert.formatTo || constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDFA === dataConvert.formatTo; - if (isFormatToPdf && isOriginFormatWithForms) { + if (isFormatToPdf && isOriginFormatWithForms && !isSendForm) { let format = await formatChecker.getDocumentFormatByFile(dataConvert.fileFrom); if (constants.AVS_OFFICESTUDIO_FILE_CANVAS_WORD === format) { ctx.logger.debug('change format to extended pdf'); @@ -358,7 +360,7 @@ async function changeFormatToExtendedPdf(ctx, dataConvert, cmd) { function* replaceEmptyFile(ctx, fileFrom, ext, _lcid) { const tenNewFileTemplate = ctx.getCfg('services.CoAuthoring.server.newFileTemplate', cfgNewFileTemplate); if (!fs.existsSync(fileFrom) || 0 === fs.lstatSync(fileFrom).size) { - let locale = 'en-US'; + let locale = constants.TEMPLATES_DEFAULT_LOCALE; if (_lcid) { let localeNew = lcid.from(_lcid); if (localeNew) { @@ -370,14 +372,24 @@ function* replaceEmptyFile(ctx, fileFrom, ext, _lcid) { } } } - ctx.logger.debug('replaceEmptyFile format=%s locale=%s', ext, locale); - let format = formatChecker.getFormatFromString(ext); - if (formatChecker.isDocumentFormat(format)) { - fs.copyFileSync(path.join(tenNewFileTemplate, locale, 'new.docx'), fileFrom); - } else if (formatChecker.isSpreadsheetFormat(format)) { - fs.copyFileSync(path.join(tenNewFileTemplate, locale, 'new.xlsx'), fileFrom); - } else if (formatChecker.isPresentationFormat(format)) { - fs.copyFileSync(path.join(tenNewFileTemplate, locale, 'new.pptx'), fileFrom); + let fileTemplatePath = path.join(tenNewFileTemplate, locale, 'new.'); + if (fs.existsSync(fileTemplatePath + ext)) { + ctx.logger.debug('replaceEmptyFile format=%s locale=%s', ext, locale); + fs.copyFileSync(fileTemplatePath + ext, fileFrom); + } else { + let format = formatChecker.getFormatFromString(ext); + let editorFormat; + if (formatChecker.isDocumentFormat(format)) { + editorFormat = 'docx'; + } else if (formatChecker.isSpreadsheetFormat(format)) { + editorFormat = 'xlsx'; + } else if (formatChecker.isPresentationFormat(format)) { + editorFormat = 'pptx'; + } + if (fs.existsSync(fileTemplatePath + editorFormat)) { + ctx.logger.debug('replaceEmptyFile format=%s locale=%s', ext, locale); + fs.copyFileSync(fileTemplatePath + editorFormat, fileFrom); + } } } } @@ -429,7 +441,6 @@ function* downloadFile(ctx, uri, fileFrom, withAuthorization, isInJwtToken, opt_ return res; } function* downloadFileFromStorage(ctx, strPath, dir, opt_specialDir) { - const tenMaxDownloadBytes = ctx.getCfg('FileConverter.converter.maxDownloadBytes', cfgMaxDownloadBytes); var list = yield storage.listObjects(ctx, strPath, opt_specialDir); ctx.logger.debug('downloadFileFromStorage list %s', list.toString()); //create dirs @@ -1002,7 +1013,6 @@ function* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, g } function* ExecuteTask(ctx, task) { - const tenMaxDownloadBytes = ctx.getCfg('FileConverter.converter.maxDownloadBytes', cfgMaxDownloadBytes); const tenForgottenFiles = ctx.getCfg('services.CoAuthoring.server.forgottenfiles', cfgForgottenFiles); const tenForgottenFilesName = ctx.getCfg('services.CoAuthoring.server.forgottenfilesname', cfgForgottenFilesName); var startDate = null; @@ -1039,19 +1049,8 @@ function* ExecuteTask(ctx, task) { withAuthorization = false; isInJwtToken = true; let fileInfo = wopiParams.commonInfo?.fileInfo; - let userAuth = wopiParams.userAuth; fileSize = fileInfo?.Size; - if (fileInfo?.FileUrl) { - //Requests to the FileUrl can not be signed using proof keys. The FileUrl is used exactly as provided by the host, so it does not necessarily include the access token, which is required to construct the expected proof. - url = fileInfo.FileUrl; - } else if (fileInfo?.TemplateSource) { - url = fileInfo.TemplateSource; - } else if (userAuth) { - url = `${userAuth.wopiSrc}/contents?access_token=${userAuth.access_token}`; - headers = {'X-WOPI-MaxExpectedSize': tenMaxDownloadBytes}; - wopiClient.fillStandardHeaders(ctx, headers, url, userAuth.access_token); - } - ctx.logger.debug('wopi url=%s; headers=%j', url, headers); + ({url, headers} = yield wopiClient.getWopiFileUrl(ctx, fileInfo, wopiParams.userAuth)); } if (undefined === fileSize || fileSize > 0) { error = yield* downloadFile(ctx, url, dataConvert.fileFrom, withAuthorization, isInJwtToken, headers); diff --git a/SpellChecker/npm-shrinkwrap.json b/SpellChecker/npm-shrinkwrap.json index ecc51d9d..7cd0edf1 100644 --- a/SpellChecker/npm-shrinkwrap.json +++ b/SpellChecker/npm-shrinkwrap.json @@ -5,40 +5,54 @@ "requires": true, "dependencies": { "accepts": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz", - "integrity": "sha1-63d99gEXI6OxTopywIBcjoZ0a9I=", + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", "requires": { - "mime-types": "~2.1.18", - "negotiator": "0.6.1" + "mime-types": "~2.1.34", + "negotiator": "0.6.3" } }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" }, "body-parser": { - "version": "1.18.3", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz", - "integrity": "sha1-WykhmP/dVTs6DyDe0FkrlWlVyLQ=", + "version": "1.20.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", + "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", "requires": { - "bytes": "3.0.0", - "content-type": "~1.0.4", + "bytes": "3.1.2", + "content-type": "~1.0.5", "debug": "2.6.9", - "depd": "~1.1.2", - "http-errors": "~1.6.3", - "iconv-lite": "0.4.23", - "on-finished": "~2.3.0", - "qs": "6.5.2", - "raw-body": "2.3.3", - "type-is": "~1.6.16" + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.11.0", + "raw-body": "2.5.2", + "type-is": "~1.6.18", + "unpipe": "1.0.0" } }, "bytes": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", - "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" + }, + "call-bind": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", + "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", + "requires": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "set-function-length": "^1.2.1" + } }, "co": { "version": "4.6.0", @@ -54,24 +68,34 @@ } }, "content-disposition": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", - "integrity": "sha1-DPaLud318r55YcOoUXjLhdunjLQ=" + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "requires": { + "safe-buffer": "5.2.1" + }, + "dependencies": { + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + } + } }, "content-type": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==" }, "cookie": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz", - "integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s=" + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==" }, "cookie-signature": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" }, "debug": { "version": "2.6.9", @@ -81,71 +105,102 @@ "ms": "2.0.0" } }, + "define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "requires": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + } + }, "depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" }, "destroy": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", - "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==" }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==" + }, + "es-define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", + "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", + "requires": { + "get-intrinsic": "^1.2.4" + } + }, + "es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==" }, "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" }, "etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==" }, "express": { - "version": "4.16.4", - "resolved": "https://registry.npmjs.org/express/-/express-4.16.4.tgz", - "integrity": "sha512-j12Uuyb4FMrd/qQAm6uCHAkPtO8FDTRJZBDd5D2KOL2eLaz1yUNdUB/NOIyq0iU4q4cFarsUCrnFDPBcnksuOg==", + "version": "4.19.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", + "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", "requires": { - "accepts": "~1.3.5", + "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.18.3", - "content-disposition": "0.5.2", + "body-parser": "1.20.2", + "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.3.1", + "cookie": "0.6.0", "cookie-signature": "1.0.6", "debug": "2.6.9", - "depd": "~1.1.2", + "depd": "2.0.0", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "etag": "~1.8.1", - "finalhandler": "1.1.1", + "finalhandler": "1.2.0", "fresh": "0.5.2", + "http-errors": "2.0.0", "merge-descriptors": "1.0.1", "methods": "~1.1.2", - "on-finished": "~2.3.0", - "parseurl": "~1.3.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", "path-to-regexp": "0.1.7", - "proxy-addr": "~2.0.4", - "qs": "6.5.2", - "range-parser": "~1.2.0", - "safe-buffer": "5.1.2", - "send": "0.16.2", - "serve-static": "1.13.2", - "setprototypeof": "1.1.0", - "statuses": "~1.4.0", - "type-is": "~1.6.16", + "proxy-addr": "~2.0.7", + "qs": "6.11.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.18.0", + "serve-static": "1.15.0", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" + }, + "dependencies": { + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + } } }, "faye-websocket": { @@ -157,38 +212,90 @@ } }, "finalhandler": { - "version": "1.1.1", - "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", - "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==", + "version": "1.2.0", + "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", + "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", "requires": { "debug": "2.6.9", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", - "on-finished": "~2.3.0", - "parseurl": "~1.3.2", - "statuses": "~1.4.0", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", "unpipe": "~1.0.0" } }, "forwarded": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", - "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==" }, "fresh": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==" + }, + "function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==" + }, + "get-intrinsic": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", + "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "requires": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "hasown": "^2.0.0" + } + }, + "gopd": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", + "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "requires": { + "get-intrinsic": "^1.1.3" + } + }, + "has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "requires": { + "es-define-property": "^1.0.0" + } + }, + "has-proto": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", + "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==" + }, + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" + }, + "hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "requires": { + "function-bind": "^1.1.2" + } }, "http-errors": { - "version": "1.6.3", - "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", - "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", + "version": "2.0.0", + "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", "requires": { - "depd": "~1.1.2", - "inherits": "2.0.3", - "setprototypeof": "1.1.0", - "statuses": ">= 1.4.0 < 2" + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" } }, "http-parser-js": { @@ -197,22 +304,22 @@ "integrity": "sha512-t7hjvef/5HEK7RWTdUzVUhl8zkEu+LlaE0IYzdMuvbSDipxBRpOn4Uhw8ZyECEa808iVT8XCjzo6xmYt4CiLZg==" }, "iconv-lite": { - "version": "0.4.23", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz", - "integrity": "sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA==", + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", "requires": { "safer-buffer": ">= 2.1.2 < 3" } }, "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "ipaddr.js": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.8.0.tgz", - "integrity": "sha1-6qM9bd16zo9/b+DJygRA5wZzix4=" + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" }, "json5": { "version": "1.0.1", @@ -225,34 +332,34 @@ "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==" }, "merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", - "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" }, "methods": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==" }, "mime": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.4.1.tgz", - "integrity": "sha512-KI1+qOZu5DcW6wayYHSzR/tXKCDC5Om4s1z2QJjDULzLcmf3DvzS7oluY4HCTrc+9FiKmWUgeNLg7W3uIQvxtQ==" + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" }, "mime-db": { - "version": "1.37.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.37.0.tgz", - "integrity": "sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg==" + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" }, "mime-types": { - "version": "2.1.21", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.21.tgz", - "integrity": "sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg==", + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", "requires": { - "mime-db": "~1.37.0" + "mime-db": "1.52.0" } }, "minimist": { @@ -263,12 +370,12 @@ "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, "negotiator": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", - "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=" + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" }, "node-addon-api": { "version": "3.0.2", @@ -282,51 +389,59 @@ "node-addon-api": "*" } }, + "object-inspect": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", + "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==" + }, "on-finished": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", - "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", "requires": { "ee-first": "1.1.1" } }, "parseurl": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz", - "integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M=" + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" }, "path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" }, "proxy-addr": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.4.tgz", - "integrity": "sha512-5erio2h9jp5CHGwcybmxmVqHmnCBZeewlfJ0pex+UW7Qny7OOZXTtH56TGNyBizkgiOwhJtMKrVzDTeKcySZwA==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", "requires": { - "forwarded": "~0.1.2", - "ipaddr.js": "1.8.0" + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" } }, "qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "requires": { + "side-channel": "^1.0.4" + } }, "range-parser": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz", - "integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4=" + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" }, "raw-body": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.3.tgz", - "integrity": "sha512-9esiElv1BrZoI3rCDuOuKCBRbuApGGaDPQfjSflGxdy4oyzqghxu6klEkkVIvBje+FF0BX9coEv8KqW6X/7njw==", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", "requires": { - "bytes": "3.0.0", - "http-errors": "1.6.3", - "iconv-lite": "0.4.23", + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", "unpipe": "1.0.0" } }, @@ -341,40 +456,71 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "send": { - "version": "0.16.2", - "resolved": "https://registry.npmjs.org/send/-/send-0.16.2.tgz", - "integrity": "sha512-E64YFPUssFHEFBvpbbjr44NCLtI1AohxQ8ZSiJjQLskAdKuriYEP6VyGEsRDH8ScozGpkaX1BGvhanqCwkcEZw==", + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", + "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", "requires": { "debug": "2.6.9", - "depd": "~1.1.2", - "destroy": "~1.0.4", + "depd": "2.0.0", + "destroy": "1.2.0", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "etag": "~1.8.1", "fresh": "0.5.2", - "http-errors": "~1.6.2", - "mime": "1.4.1", - "ms": "2.0.0", - "on-finished": "~2.3.0", - "range-parser": "~1.2.0", - "statuses": "~1.4.0" + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "dependencies": { + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + } } }, "serve-static": { - "version": "1.13.2", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.13.2.tgz", - "integrity": "sha512-p/tdJrO4U387R9oMjb1oj7qSMaMfmOyd4j9hOFoxZe2baQszgHcSWjuya/CiT5kgZZKRudHNOA0pYXOl8rQ5nw==", + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", "requires": { "encodeurl": "~1.0.2", "escape-html": "~1.0.3", - "parseurl": "~1.3.2", - "send": "0.16.2" + "parseurl": "~1.3.3", + "send": "0.18.0" + } + }, + "set-function-length": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", + "requires": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" } }, "setprototypeof": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", - "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "side-channel": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", + "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", + "requires": { + "call-bind": "^1.0.7", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.4", + "object-inspect": "^1.13.1" + } }, "sockjs": { "version": "0.3.21", @@ -387,28 +533,33 @@ } }, "statuses": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", - "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==" + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==" + }, + "toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" }, "type-is": { - "version": "1.6.16", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz", - "integrity": "sha512-HRkVv/5qY2G6I8iab9cI7v1bOIdhm94dVjQCPFElW9W+3GeDOSHmy2EBYe4VTApuzolPcmgFTN3ftVJRKR2J9Q==", + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", "requires": { "media-typer": "0.3.0", - "mime-types": "~2.1.18" + "mime-types": "~2.1.24" } }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==" }, "utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==" }, "uuid": { "version": "3.4.0", @@ -418,7 +569,7 @@ "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==" }, "websocket-driver": { "version": "0.7.4", diff --git a/SpellChecker/package.json b/SpellChecker/package.json index ebb74c97..62d2d250 100644 --- a/SpellChecker/package.json +++ b/SpellChecker/package.json @@ -7,7 +7,7 @@ "dependencies": { "co": "4.6.0", "config": "2.0.1", - "express": "4.16.4", + "express": "4.19.2", "nodehun": "git+https://git@github.com/ONLYOFFICE/nodehun.git#2411a56828c7d58214c61781b4a5c63d18adba99", "sockjs": "0.3.21" }, diff --git a/tests/integration/withServerInstance/storage.tests.js b/tests/integration/withServerInstance/storage.tests.js index 19ff0a8c..26fde01c 100644 --- a/tests/integration/withServerInstance/storage.tests.js +++ b/tests/integration/withServerInstance/storage.tests.js @@ -53,7 +53,8 @@ const utils = require('../../../Common/sources/utils'); const commonDefines = require("../../../Common/sources/commondefines"); const config = require('../../../Common/node_modules/config'); -const cfgStorageName = config.get('storage.name'); +const cfgCacheStorage = config.get('storage'); +const cfgPersistentStorage = utils.deepMergeObjects({}, cfgCacheStorage, config.get('persistentStorage')); const ctx = operationContext.global; const rand = Math.floor(Math.random() * 1000000); @@ -64,9 +65,15 @@ let testFile1 = testDir + "/test1.txt"; let testFile2 = testDir + "/test2.txt"; let testFile3 = testDir + "/test3.txt"; let testFile4 = testDir + "/test4.txt"; +let specialDirCache = ""; +let specialDirForgotten = "forgotten"; console.debug(`testDir: ${testDir}`) +function getStorageCfg(specialDir) { + return specialDir ? cfgPersistentStorage : cfgCacheStorage; +} + function request(url) { return new Promise(resolve => { let module = url.startsWith('https') ? https : http; @@ -97,7 +104,7 @@ function runTestForDir(specialDir) { let list = await storage.listObjects(ctx, testDir, specialDir); expect(list.sort()).toEqual([testFile1, testFile2].sort()); }); - if ("storage-fs" === cfgStorageName) { + if ("storage-fs" === getStorageCfg(specialDir).name) { test("UploadObject", async () => { let res = await storage.uploadObject(ctx, testFile3, "createReadStream.txt", specialDir); expect(res).toEqual(undefined); @@ -113,6 +120,7 @@ function runTestForDir(specialDir) { let list = await storage.listObjects(ctx, testDir, specialDir); expect(spy).toHaveBeenCalled(); expect(list.sort()).toEqual([testFile1, testFile2, testFile3].sort()); + spy.mockRestore(); }); } test("copyObject", async () => { @@ -172,7 +180,7 @@ function runTestForDir(specialDir) { expect(outputText.toString("utf8")).toEqual(testFileData3); }); test("getSignedUrl", async () => { - let url, data; + let url, urls, data; url = await storage.getSignedUrl(ctx, baseUrl, testFile1, urlType, undefined, undefined, specialDir); data = await request(url); expect(data).toEqual(testFileData1); @@ -189,6 +197,33 @@ function runTestForDir(specialDir) { data = await request(url); expect(data).toEqual(testFileData4); }); + test("getSignedUrls", async () => { + let urls, data; + urls = await storage.getSignedUrls(ctx, baseUrl, testDir, urlType, undefined, specialDir); + data = []; + for(let i in urls) { + data.push(await request(urls[i])); + } + expect(data.sort()).toEqual([testFileData1, testFileData2, testFileData3, testFileData4].sort()); + }); + test("getSignedUrlsArrayByArray", async () => { + let urls, data; + urls = await storage.getSignedUrlsArrayByArray(ctx, baseUrl, [testFile1, testFile2], urlType, specialDir); + data = []; + for(let i = 0; i < urls.length; ++i) { + data.push(await request(urls[i])); + } + expect(data.sort()).toEqual([testFileData1, testFileData2].sort()); + }); + test("getSignedUrlsByArray", async () => { + let urls, data; + urls = await storage.getSignedUrlsByArray(ctx, baseUrl, [testFile3, testFile4], undefined, urlType, specialDir); + data = []; + for(let i in urls) { + data.push(await request(urls[i])); + } + expect(data.sort()).toEqual([testFileData3, testFileData4].sort()); + }); test("deleteObject", async () => { let list; list = await storage.listObjects(ctx, testDir, specialDir); @@ -215,9 +250,57 @@ function runTestForDir(specialDir) { // Assumed, that server is already up. describe('storage common dir', function () { - runTestForDir(""); + runTestForDir(specialDirCache); }); describe('storage forgotten dir', function () { - runTestForDir("forgotten"); -}); \ No newline at end of file + runTestForDir(specialDirForgotten); +}); + +describe('storage mix common and forgotten dir', function () { + test("putObject", async () => { + let buffer = Buffer.from(testFileData1); + let res = await storage.putObject(ctx, testFile1, buffer, buffer.length, specialDirCache); + expect(res).toEqual(undefined); + let list = await storage.listObjects(ctx, testDir, specialDirCache); + expect(list.sort()).toEqual([testFile1].sort()); + + buffer = Buffer.from(testFileData2); + res = await storage.putObject(ctx, testFile2, buffer, buffer.length, specialDirForgotten); + expect(res).toEqual(undefined); + list = await storage.listObjects(ctx, testDir, specialDirForgotten); + expect(list.sort()).toEqual([testFile2].sort()); + }); + + test("copyPath", async () => { + let list, res; + res = await storage.copyPath(ctx, testDir, testDir, specialDirCache, specialDirForgotten); + expect(res).toEqual(undefined); + + list = await storage.listObjects(ctx, testDir, specialDirForgotten); + expect(list.sort()).toEqual([testFile1, testFile2].sort()); + }); + test("copyObject", async () => { + let list, res; + res = await storage.copyObject(ctx, testFile2, testFile2, specialDirForgotten, specialDirCache); + expect(res).toEqual(undefined); + + list = await storage.listObjects(ctx, testDir, specialDirCache); + expect(list.sort()).toEqual([testFile1, testFile2].sort()); + }); + + test("deletePath", async () => { + let list, res; + res = await storage.deletePath(ctx, testDir, specialDirCache); + expect(res).toEqual(undefined); + + list = await storage.listObjects(ctx, testDir, specialDirCache); + expect(list.sort()).toEqual([].sort()); + + res = await storage.deletePath(ctx, testDir, specialDirForgotten); + expect(res).toEqual(undefined); + + list = await storage.listObjects(ctx, testDir, specialDirForgotten); + expect(list.sort()).toEqual([].sort()); + }); +});