pengjunjie/server
1
0
Fork 1
mirror of https://github.com/ONLYOFFICE/server.git synced 2026-02-10 18:05:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

[git actions] Postgres and mysql actions

Browse Source
This commit is contained in:
Georgii Petrov
2023-10-02 09:14:27 +03:00
parent d303205723
commit 096efaba4c
10 changed files with 138 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
.github/workflows/databasesTestsAaction.yml vendored Normal file
View File

@ -0,0 +1,60 @@
name: check
on: [push]
jobs:
mysql-tests:
name: MYSQL database tests
runs-on: ubuntu-latest
steps:
- name: Run Mysql DB docker container
run: docker run --name mysql -p 8080:3306 -p 8081:33060 -e MYSQL_HOST=127.0.0.1 -e MYSQL_ROOT_PASSWORD=onlyoffice -e MYSQL_DATABASE=onlyoffice -d mysql:latest
- name: Check out repository code
uses: actions/checkout@v3
- name: Install modules
run: |
npm ci
npm --prefix Common ci
npm --prefix DocService ci
- name: Configure local IP
run: |
echo '{"services": {"CoAuthoring": {"sql": {"type": "mysql", "dbHost": "127.0.0.1", "dbPort": "8080", "dbUser": "root", "dbPass": "onlyoffice"}}}}' >> Common/config/local.json
- name : Creating schema
run: |
docker cp ./schema/mysql/createdb.sql mysql:/
docker exec mysql mysql -h 127.0.0.1 -u root --password=onlyoffice -D onlyoffice -e 'source /createdb.sql'
- name: Run Jest
run: npm exec -c 'cd ./DocService && jest baseConnector.tests.js --inject-globals=false --config=../tests/jest.config.js'
postgres-tests:
name: Postgres database tests
runs-on: ubuntu-latest
steps:
- name: Run Postgres DB docker container
run: docker run --name postgres -p 5432:5432 -e POSTGRES_PASSWORD=onlyoffice -e POSTGRES_USER=onlyoffice -e POSTGRES_DB=onlyoffice -d postgres:latest
- name: Check out repository code
uses: actions/checkout@v3
- name: Install modules
run: |
npm ci
npm --prefix Common ci
npm --prefix DocService ci
- name: Configure local IP
run: |
echo '{"services": {"CoAuthoring": {"sql": {"dbHost": "127.0.0.1"}}}}' >> Common/config/local.json
- name : Creating schema
run: |
docker cp ./schema/postgresql/createdb.sql postgres:/
docker exec postgres psql -d onlyoffice -U onlyoffice -a -f /createdb.sql
- name: Run Jest
run: npm exec -c 'cd ./DocService && jest baseConnector.tests.js --inject-globals=false --config=../tests/jest.config.js'

6
DocService/npm-shrinkwrap.json generated
View File

@ -1120,9 +1120,9 @@
}
},
"config": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/config/-/config-2.0.1.tgz",
"integrity": "sha512-aTaviJnC8ZjQYx8kQf4u6tWqIxWolyQQ3LqXgnCLAsIb78JrUshHG0YuzIarzTaVVe1Pazms3TXImfYra8UsyQ==",
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/config/-/config-2.0.2.tgz",
"integrity": "sha512-duIbkKb0gls0bOtGwd1vaD4236MwepQlZcrMheOGrn3/9Px7oYFh8G4LB3ylGOlPr5wGoJRm8Grb2RihJZxuHQ==",
"requires": {
"json5": "^1.0.1"
}

2
DocService/sources/baseConnector.js
View File

@ -353,7 +353,7 @@ function getEmptyCallbacks(ctx) {
function getTableColumns(ctx, tableName) {
return new Promise(function(resolve, reject) {
const sqlCommand = `SELECT column_name as 'column_name' FROM information_schema.COLUMNS WHERE TABLE_NAME = '${tableName}';`;
const sqlCommand = `SELECT column_name as "column_name" FROM information_schema.COLUMNS WHERE TABLE_NAME = '${tableName}';`;
dbInstance.sqlQuery(ctx, sqlCommand, function(error, result) {
if (error) {
reject(error);

74
DocService/sources/postgreSqlBaseConnector.js
View File

@ -67,7 +67,7 @@ types.setTypeParser(1184, function(stringValue) {
var maxPacketSize = configSql.get('max_allowed_packet');
exports.sqlQuery = function(ctx, sqlCommand, callbackFunction, opt_noModifyRes, opt_noLog, opt_values) {
function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes, opt_noLog, opt_values) {
co(function *() {
var result = null;
var error = null;
@ -92,16 +92,21 @@ exports.sqlQuery = function(ctx, sqlCommand, callbackFunction, opt_noModifyRes,
}
}
});
};
let addSqlParam = function (val, values) {
}
function closePool() {
pool.end();
}
function addSqlParameter(val, values) {
values.push(val);
return '$' + values.length;
};
exports.addSqlParameter = addSqlParam;
let concatParams = function (val1, val2) {
}
function concatParams(val1, val2) {
return `COALESCE(${val1}, '') || COALESCE(${val2}, '')`;
};
exports.concatParams = concatParams;
}
var isSupportOnConflict = true;
function getUpsertString(task, values) {
@ -113,28 +118,28 @@ function getUpsertString(task, values) {
userCallback.fromValues(task.userIndex, task.callback);
cbInsert = userCallback.toSQLInsert();
}
let p0 = addSqlParam(task.tenant, values);
let p1 = addSqlParam(task.key, values);
let p2 = addSqlParam(task.status, values);
let p3 = addSqlParam(task.statusInfo, values);
let p4 = addSqlParam(dateNow, values);
let p5 = addSqlParam(task.userIndex, values);
let p6 = addSqlParam(task.changeId, values);
let p7 = addSqlParam(cbInsert, values);
let p8 = addSqlParam(task.baseurl, values);
let p0 = addSqlParameter(task.tenant, values);
let p1 = addSqlParameter(task.key, values);
let p2 = addSqlParameter(task.status, values);
let p3 = addSqlParameter(task.statusInfo, values);
let p4 = addSqlParameter(dateNow, values);
let p5 = addSqlParameter(task.userIndex, values);
let p6 = addSqlParameter(task.changeId, values);
let p7 = addSqlParameter(cbInsert, values);
let p8 = addSqlParameter(task.baseurl, values);
if (isSupportOnConflict) {
let p9 = addSqlParam(dateNow, values);
let p9 = addSqlParameter(dateNow, values);
//http://stackoverflow.com/questions/34762732/how-to-find-out-if-an-upsert-was-an-update-with-postgresql-9-5-upsert
let sqlCommand = `INSERT INTO ${cfgTableResult} (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl)`;
sqlCommand += ` VALUES (${p0}, ${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8})`;
sqlCommand += ` ON CONFLICT (tenant, id) DO UPDATE SET last_open_date = ${p9}`;
if (task.callback) {
let p10 = addSqlParam(JSON.stringify(task.callback), values);
let p10 = addSqlParameter(JSON.stringify(task.callback), values);
sqlCommand += `, callback = ${cfgTableResult}.callback || '${connectorUtilities.UserCallback.prototype.delimiter}{"userIndex":' `;
sqlCommand += ` || (${cfgTableResult}.user_index + 1)::text || ',"callback":' || ${p10}::text || '}'`;
}
if (task.baseurl) {
let p11 = addSqlParam(task.baseurl, values);
let p11 = addSqlParameter(task.baseurl, values);
sqlCommand += `, baseurl = ${p11}`;
}
sqlCommand += `, user_index = ${cfgTableResult}.user_index + 1 RETURNING user_index as userindex;`;
@ -143,17 +148,18 @@ function getUpsertString(task, values) {
return `SELECT * FROM merge_db(${p0}, ${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8});`;
}
}
exports.upsert = function(ctx, task) {
function upsert(ctx, task) {
return new Promise(function(resolve, reject) {
let values = [];
var sqlCommand = getUpsertString(task, values);
exports.sqlQuery(ctx, sqlCommand, function(error, result) {
sqlQuery(ctx, sqlCommand, function(error, result) {
if (error) {
if (isSupportOnConflict && '42601' === error.code) {
//SYNTAX ERROR
isSupportOnConflict = false;
ctx.logger.warn('checkIsSupportOnConflict false');
resolve(exports.upsert(ctx, task));
resolve(upsert(ctx, task));
} else {
reject(error);
}
@ -168,8 +174,9 @@ exports.upsert = function(ctx, task) {
}
}, true, undefined, values);
});
};
exports.insertChanges = function(ctx, tableChanges, startIndex, objChanges, docId, index, user, callback) {
}
function insertChanges(ctx, tableChanges, startIndex, objChanges, docId, index, user, callback) {
let i = startIndex;
if (i >= objChanges.length) {
return;
@ -193,7 +200,7 @@ exports.insertChanges = function(ctx, tableChanges, startIndex, objChanges, docI
//4 is max utf8 bytes per symbol
curLength += 4 * (docId.length + user.id.length + user.idOriginal.length + user.username.length + objChanges[i].change.length) + 4 + 8;
if (curLength >= maxPacketSize && i > startIndex) {
exports.sqlQuery(ctx, sqlCommand, function(error, output) {
sqlQuery(ctx, sqlCommand, function(error, output) {
if (error && '42883' == error.code) {
isSupported = false;
ctx.logger.warn('postgresql does not support UNNEST');
@ -201,7 +208,7 @@ exports.insertChanges = function(ctx, tableChanges, startIndex, objChanges, docI
if (error) {
callback(error, output, isSupported);
} else {
exports.insertChanges(ctx, tableChanges, i, objChanges, docId, index, user, callback);
insertChanges(ctx, tableChanges, i, objChanges, docId, index, user, callback);
}
}, undefined, undefined, values);
return;
@ -215,11 +222,20 @@ exports.insertChanges = function(ctx, tableChanges, startIndex, objChanges, docI
change.push(objChanges[i].change);
time.push(objChanges[i].time);
}
exports.sqlQuery(ctx, sqlCommand, function(error, output) {
sqlQuery(ctx, sqlCommand, function(error, output) {
if (error && '42883' == error.code) {
isSupported = false;
ctx.logger.warn('postgresql does not support UNNEST');
}
callback(error, output, isSupported);
}, undefined, undefined, values);
};
}
module.exports = {
sqlQuery,
closePool,
addSqlParameter,
concatParams,
upsert,
insertChanges
};

3
package.json
View File

@ -72,7 +72,8 @@
"perf-expired": "cd ./DocService&& cross-env NODE_ENV=development-windows NODE_CONFIG_DIR=../Common/config node ../tests/perf/checkFileExpire.js",
"perf-exif": "cd ./DocService&& cross-env NODE_ENV=development-windows NODE_CONFIG_DIR=../Common/config node ../tests/perf/fixImageExifRotation.js",
"unit tests": "cd ./DocService && jest unit --inject-globals=false --config=../tests/jest.config.js",
"integration tests": "cd ./DocService && jest integration --inject-globals=false --config=../tests/jest.config.js",
"integration tests with server instance": "cd ./DocService && jest integration/withServerInstance --inject-globals=false --config=../tests/jest.config.js",
"integration tests without server instance": "cd ./DocService && jest integration/(?!withServerInstance/) --inject-globals=false --config=../tests/jest.config.js",
"tests": "cd ./DocService && jest --inject-globals=false --config=../tests/jest.config.js",
"install:Common": "npm ci --prefix ./Common",
"install:DocService": "npm ci --prefix ./DocService",

2
schema/mysql/createdb.sql
View File

@ -54,7 +54,7 @@ CREATE TABLE IF NOT EXISTS `task_result` (
`id` varchar(255) NOT NULL,
`status` tinyint(3) NOT NULL,
`status_info` int(10) NOT NULL,
`created_at` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP,
`created_at` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
`last_open_date` datetime(6) NOT NULL,
`user_index` int(10) unsigned NOT NULL DEFAULT 1,
`change_id` int(10) unsigned NOT NULL DEFAULT 0,

3
tests/integration/baseConnector.tests.js
View File

@ -103,7 +103,8 @@ function createChanges(changesLength, date) {
async function getRowsCountById(table, id) {
const result = await executeSql(`SELECT COUNT(id) AS count FROM ${table} WHERE id = '${id}';`);
return result[0].count;
// Return type of COUNT() in postgres is bigint which treats as string by connector.
return +result[0].count;
}
async function noRowsExistenceCheck(table, id) {

15
tests/integration/forgottenFilesCommnads.tests.js → tests/integration/withServerInstance/forgottenFilesCommnads.tests.js
View File

@ -1,12 +1,13 @@
const { describe, test, expect, afterAll, beforeAll } = require('@jest/globals');
const http = require('http');
const { signToken } = require('../../DocService/sources/DocsCoServer');
const storage = require('../../Common/sources/storage-base');
const constants = require('../../Common/sources/commondefines');
const operationContext = require('../../Common/sources/operationContext');
const utils = require("../../Common/sources/utils");
const config = require('../../Common/node_modules/config');
const { signToken } = require('../../../DocService/sources/DocsCoServer');
const storage = require('../../../Common/sources/storage-base');
const constants = require('../../../Common/sources/commondefines');
const operationContext = require('../../../Common/sources/operationContext');
const utils = require("../../../Common/sources/utils");
const config = require('../../../Common/node_modules/config');
const cfgForgottenFiles = config.get('services.CoAuthoring.server.forgottenfiles');
const cfgForgottenFilesName = config.get('services.CoAuthoring.server.forgottenfilesname');
@ -18,7 +19,7 @@ const cfgStorageName = config.get('storage.name');
const cfgEndpoint = config.get('storage.endpoint');
const cfgBucketName = config.get('storage.bucketName');
const ctx = new operationContext.Context();
//yield ctx.initTenantCache();//no need
const testFilesNames = {
get: 'DocService-DocsCoServer-forgottenFilesCommands-getForgotten-integration-test',
delete1: 'DocService-DocsCoServer-forgottenFilesCommands-deleteForgotten-integration-test',

10
tests/integration/storage.tests.js → tests/integration/withServerInstance/storage.tests.js
View File

@ -15,11 +15,11 @@ jest.mock("fs/promises", () => ({
}));
const { cp } = require('fs/promises');
const operationContext = require('../../Common/sources/operationContext');
const storage = require('../../Common/sources/storage-base');
const utils = require('../../Common/sources/utils');
const commonDefines = require("../../Common/sources/commondefines");
const config = require('../../Common/node_modules/config');
const operationContext = require('../../../Common/sources/operationContext');
const storage = require('../../../Common/sources/storage-base');
const utils = require('../../../Common/sources/utils');
const commonDefines = require("../../../Common/sources/commondefines");
const config = require('../../../Common/node_modules/config');
const cfgStorageName = config.get('storage.name');

11
tests/unit/sample.tets.js Normal file
View File

@ -0,0 +1,11 @@
const { describe, test, expect, afterAll } = require('@jest/globals');
describe('Successful and failure tests', function () {
test('Successful test', function () {
expect(true).toBeTruthy();
});
test('Failure test', function () {
expect(true).toBeFalsy();
});
});