From 6728395f1e80e973ccef99b5faf0b3dd1bd21aab Mon Sep 17 00:00:00 2001 From: Georgii Petrov Date: Fri, 16 Jun 2023 16:25:59 +0300 Subject: [PATCH 01/13] [ds] Oracle base connector pt.1 --- DocService/npm-shrinkwrap.json | 5 + DocService/package.json | 1 + DocService/sources/baseConnector.js | 180 +++--------------- DocService/sources/connectorUtilities.js | 155 +++++++++++++++ DocService/sources/damengBaseConnector.js | 6 +- DocService/sources/mySqlBaseConnector.js | 6 +- DocService/sources/oracleBaseConnector.js | 174 +++++++++++++++++ DocService/sources/postgreSqlBaseConnector.js | 6 +- 8 files changed, 367 insertions(+), 166 deletions(-) create mode 100644 DocService/sources/connectorUtilities.js create mode 100644 DocService/sources/oracleBaseConnector.js diff --git a/DocService/npm-shrinkwrap.json b/DocService/npm-shrinkwrap.json index 0323903e..6d725786 100644 --- a/DocService/npm-shrinkwrap.json +++ b/DocService/npm-shrinkwrap.json @@ -1361,6 +1361,11 @@ "ee-first": "1.1.1" } }, + "oracledb": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/oracledb/-/oracledb-6.0.1.tgz", + "integrity": "sha512-N5/Y4IkCFQCumqjEXi3snrclDKjDMMeawq/z/5Ydm5+BxjV3kg9wCaTAp++JTLlCWASb4JMXqK70PctmAkZh3g==" + }, "packet-reader": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", diff --git a/DocService/package.json b/DocService/package.json index d14134c2..c59de5ac 100644 --- a/DocService/package.json +++ b/DocService/package.json @@ -33,6 +33,7 @@ "multi-integer-range": "^4.0.7", "multiparty": "^4.2.1", "mysql2": "^2.3.3", + "oracledb": "^6.0.1", "pg": "^8.8.0", "redis": "^4.6.5", "retry": "^0.12.0", diff --git a/DocService/sources/baseConnector.js b/DocService/sources/baseConnector.js index 05c30d77..b0e0b441 100644 --- a/DocService/sources/baseConnector.js +++ b/DocService/sources/baseConnector.js @@ -36,15 +36,18 @@ var sqlDataBaseType = { mySql : 'mysql', mariaDB : 'mariadb', postgreSql : 'postgres', - dameng : 'dameng' + dameng : 'dameng', + oracle: 'oracle' }; +const connectorUtilities = require('./connectorUtilities'); var bottleneck = require("bottleneck"); var config = require('config'); var configSql = config.get('services.CoAuthoring.sql'); +const dbType = configSql.get('type'); var baseConnector; -switch (configSql.get('type')) { +switch (dbType) { case sqlDataBaseType.mySql: case sqlDataBaseType.mariaDB: baseConnector = require('./mySqlBaseConnector'); @@ -52,11 +55,13 @@ switch (configSql.get('type')) { case sqlDataBaseType.dameng: baseConnector = require('./damengBaseConnector'); break; + case sqlDataBaseType.oracle: + baseConnector = require('./oracleBaseConnector'); + break; default: baseConnector = require('./postgreSqlBaseConnector'); break; } -let constants = require('./../../Common/sources/constants'); const cfgTableResult = configSql.get('tableResult'); const cfgTableChanges = configSql.get('tableChanges'); @@ -275,7 +280,18 @@ exports.healthCheck = function (ctx) { return new Promise(function(resolve, reject) { //SELECT 1; usefull for H2, MySQL, Microsoft SQL Server, PostgreSQL, SQLite //http://stackoverflow.com/questions/3668506/efficient-sql-test-query-or-validation-query-that-will-work-across-all-or-most - baseConnector.sqlQuery(ctx, 'SELECT 1;', function(error, result) { + let sql; + switch (dbType) { + case sqlDataBaseType.oracle: { + sql = 'SELECT 1 FROM DUAL;'; + break; + } + default: { + sql = 'SELECT 1;'; + } + } + + baseConnector.sqlQuery(ctx, sql, function(error, result) { if (error) { reject(error); } else { @@ -313,156 +329,6 @@ exports.getTableColumns = function(ctx, tableName) { }); } }; -function UserCallback() { - this.userIndex = undefined; - this.callback = undefined; -} -UserCallback.prototype.fromValues = function(userIndex, callback){ - if(null !== userIndex){ - this.userIndex = userIndex; - } - if(null !== callback){ - this.callback = callback; - } -}; -UserCallback.prototype.delimiter = constants.CHAR_DELIMITER; -UserCallback.prototype.toSQLInsert = function(){ - return this.delimiter + JSON.stringify(this); -}; -UserCallback.prototype.getCallbackByUserIndex = function(ctx, callbacksStr, opt_userIndex) { - ctx.logger.debug("getCallbackByUserIndex: userIndex = %s callbacks = %s", opt_userIndex, callbacksStr); - if (!callbacksStr || !callbacksStr.startsWith(UserCallback.prototype.delimiter)) { - let index = callbacksStr.indexOf(UserCallback.prototype.delimiter); - if (-1 === index) { - //old format - return callbacksStr; - } else { - //mix of old and new format - callbacksStr = callbacksStr.substring(index); - } - } - let callbacks = callbacksStr.split(UserCallback.prototype.delimiter); - let callbackUrl = ""; - for (let i = 1; i < callbacks.length; ++i) { - let callback = JSON.parse(callbacks[i]); - callbackUrl = callback.callback; - if (callback.userIndex === opt_userIndex) { - break; - } - } - return callbackUrl; -}; -UserCallback.prototype.getCallbacks = function(ctx, callbacksStr) { - ctx.logger.debug("getCallbacks: callbacks = %s", callbacksStr); - if (!callbacksStr || !callbacksStr.startsWith(UserCallback.prototype.delimiter)) { - let index = callbacksStr.indexOf(UserCallback.prototype.delimiter); - if (-1 === index) { - //old format - return [callbacksStr]; - } else { - //mix of old and new format - callbacksStr = callbacksStr.substring(index); - } - } - let callbacks = callbacksStr.split(UserCallback.prototype.delimiter); - let res = []; - for (let i = 1; i < callbacks.length; ++i) { - let callback = JSON.parse(callbacks[i]); - res.push(callback.callback); - } - return res; -}; -exports.UserCallback = UserCallback; - -function DocumentPassword() { - this.password = undefined; - this.change = undefined; -} -DocumentPassword.prototype.fromString = function(passwordStr){ - var parsed = JSON.parse(passwordStr); - this.fromValues(parsed.password, parsed.change); -}; -DocumentPassword.prototype.fromValues = function(password, change){ - if(null !== password){ - this.password = password; - } - if(null !== change) { - this.change = change; - } -}; -DocumentPassword.prototype.delimiter = constants.CHAR_DELIMITER; -DocumentPassword.prototype.toSQLInsert = function(){ - return this.delimiter + JSON.stringify(this); -}; -DocumentPassword.prototype.isInitial = function(){ - return !this.change; -}; -DocumentPassword.prototype.getDocPassword = function(ctx, docPasswordStr) { - let res = {initial: undefined, current: undefined, change: undefined}; - if (docPasswordStr) { - ctx.logger.debug("getDocPassword: passwords = %s", docPasswordStr); - let passwords = docPasswordStr.split(UserCallback.prototype.delimiter); - - for (let i = 1; i < passwords.length; ++i) { - let password = new DocumentPassword(); - password.fromString(passwords[i]); - if (password.isInitial()) { - res.initial = password.password; - } else { - res.change = password.change; - } - res.current = password.password; - } - } - return res; -}; -DocumentPassword.prototype.getCurPassword = function(ctx, docPasswordStr) { - let docPassword = this.getDocPassword(ctx, docPasswordStr); - return docPassword.current; -}; -DocumentPassword.prototype.hasPasswordChanges = function(ctx, docPasswordStr) { - let docPassword = this.getDocPassword(ctx, docPasswordStr); - return docPassword.initial !== docPassword.current; -}; -exports.DocumentPassword = DocumentPassword; - -function DocumentAdditional() { - this.data = []; -} -DocumentAdditional.prototype.delimiter = constants.CHAR_DELIMITER; -DocumentAdditional.prototype.toSQLInsert = function() { - if (this.data.length) { - let vals = this.data.map((currentValue) => { - return JSON.stringify(currentValue); - }); - return this.delimiter + vals.join(this.delimiter); - } else { - return null; - } -}; -DocumentAdditional.prototype.fromString = function(str) { - if (!str) { - return; - } - let vals = str.split(this.delimiter).slice(1); - this.data = vals.map((currentValue) => { - return JSON.parse(currentValue); - }); -}; -DocumentAdditional.prototype.setOpenedAt = function(time, timezoneOffset) { - let additional = new DocumentAdditional(); - additional.data.push({time: time, timezoneOffset: timezoneOffset}); - return additional.toSQLInsert(); -}; -DocumentAdditional.prototype.getOpenedAt = function(str) { - let res; - let val = new DocumentAdditional(); - val.fromString(str); - val.data.forEach((elem) => { - if (undefined !== elem.timezoneOffset) { - res = elem.time - (elem.timezoneOffset * 60 * 1000); - } - }); - return res; -}; -exports.DocumentAdditional = DocumentAdditional; +exports.UserCallback = connectorUtilities.UserCallback; +exports.DocumentPassword = connectorUtilities.DocumentPassword; +exports.DocumentAdditional = connectorUtilities.DocumentAdditional; \ No newline at end of file diff --git a/DocService/sources/connectorUtilities.js b/DocService/sources/connectorUtilities.js new file mode 100644 index 00000000..92c0c529 --- /dev/null +++ b/DocService/sources/connectorUtilities.js @@ -0,0 +1,155 @@ +const constants = require('./../../Common/sources/constants'); + +function UserCallback() { + this.userIndex = undefined; + this.callback = undefined; +} +UserCallback.prototype.fromValues = function(userIndex, callback){ + if(null !== userIndex){ + this.userIndex = userIndex; + } + if(null !== callback){ + this.callback = callback; + } +}; +UserCallback.prototype.delimiter = constants.CHAR_DELIMITER; +UserCallback.prototype.toSQLInsert = function(){ + return this.delimiter + JSON.stringify(this); +}; +UserCallback.prototype.getCallbackByUserIndex = function(ctx, callbacksStr, opt_userIndex) { + ctx.logger.debug("getCallbackByUserIndex: userIndex = %s callbacks = %s", opt_userIndex, callbacksStr); + if (!callbacksStr || !callbacksStr.startsWith(UserCallback.prototype.delimiter)) { + let index = callbacksStr.indexOf(UserCallback.prototype.delimiter); + if (-1 === index) { + //old format + return callbacksStr; + } else { + //mix of old and new format + callbacksStr = callbacksStr.substring(index); + } + } + let callbacks = callbacksStr.split(UserCallback.prototype.delimiter); + let callbackUrl = ""; + for (let i = 1; i < callbacks.length; ++i) { + let callback = JSON.parse(callbacks[i]); + callbackUrl = callback.callback; + if (callback.userIndex === opt_userIndex) { + break; + } + } + return callbackUrl; +}; +UserCallback.prototype.getCallbacks = function(ctx, callbacksStr) { + ctx.logger.debug("getCallbacks: callbacks = %s", callbacksStr); + if (!callbacksStr || !callbacksStr.startsWith(UserCallback.prototype.delimiter)) { + let index = callbacksStr.indexOf(UserCallback.prototype.delimiter); + if (-1 === index) { + //old format + return [callbacksStr]; + } else { + //mix of old and new format + callbacksStr = callbacksStr.substring(index); + } + } + let callbacks = callbacksStr.split(UserCallback.prototype.delimiter); + let res = []; + for (let i = 1; i < callbacks.length; ++i) { + let callback = JSON.parse(callbacks[i]); + res.push(callback.callback); + } + return res; +}; +exports.UserCallback = UserCallback; + +function DocumentPassword() { + this.password = undefined; + this.change = undefined; +} +DocumentPassword.prototype.fromString = function(passwordStr){ + var parsed = JSON.parse(passwordStr); + this.fromValues(parsed.password, parsed.change); +}; +DocumentPassword.prototype.fromValues = function(password, change){ + if(null !== password){ + this.password = password; + } + if(null !== change) { + this.change = change; + } +}; +DocumentPassword.prototype.delimiter = constants.CHAR_DELIMITER; +DocumentPassword.prototype.toSQLInsert = function(){ + return this.delimiter + JSON.stringify(this); +}; +DocumentPassword.prototype.isInitial = function(){ + return !this.change; +}; +DocumentPassword.prototype.getDocPassword = function(ctx, docPasswordStr) { + let res = {initial: undefined, current: undefined, change: undefined}; + if (docPasswordStr) { + ctx.logger.debug("getDocPassword: passwords = %s", docPasswordStr); + let passwords = docPasswordStr.split(UserCallback.prototype.delimiter); + + for (let i = 1; i < passwords.length; ++i) { + let password = new DocumentPassword(); + password.fromString(passwords[i]); + if (password.isInitial()) { + res.initial = password.password; + } else { + res.change = password.change; + } + res.current = password.password; + } + } + return res; +}; +DocumentPassword.prototype.getCurPassword = function(ctx, docPasswordStr) { + let docPassword = this.getDocPassword(ctx, docPasswordStr); + return docPassword.current; +}; +DocumentPassword.prototype.hasPasswordChanges = function(ctx, docPasswordStr) { + let docPassword = this.getDocPassword(ctx, docPasswordStr); + return docPassword.initial !== docPassword.current; +}; +exports.DocumentPassword = DocumentPassword; + +function DocumentAdditional() { + this.data = []; +} +DocumentAdditional.prototype.delimiter = constants.CHAR_DELIMITER; +DocumentAdditional.prototype.toSQLInsert = function() { + if (this.data.length) { + let vals = this.data.map((currentValue) => { + return JSON.stringify(currentValue); + }); + return this.delimiter + vals.join(this.delimiter); + } else { + return null; + } +}; +DocumentAdditional.prototype.fromString = function(str) { + if (!str) { + return; + } + let vals = str.split(this.delimiter).slice(1); + this.data = vals.map((currentValue) => { + return JSON.parse(currentValue); + }); +}; +DocumentAdditional.prototype.setOpenedAt = function(time, timezoneOffset) { + let additional = new DocumentAdditional(); + additional.data.push({time: time, timezoneOffset: timezoneOffset}); + return additional.toSQLInsert(); +}; +DocumentAdditional.prototype.getOpenedAt = function(str) { + let res; + let val = new DocumentAdditional(); + val.fromString(str); + val.data.forEach((elem) => { + if (undefined !== elem.timezoneOffset) { + res = elem.time - (elem.timezoneOffset * 60 * 1000); + } + }); + return res; +}; +exports.DocumentAdditional = DocumentAdditional; \ No newline at end of file diff --git a/DocService/sources/damengBaseConnector.js b/DocService/sources/damengBaseConnector.js index 1e8d3bed..815cb620 100644 --- a/DocService/sources/damengBaseConnector.js +++ b/DocService/sources/damengBaseConnector.js @@ -33,7 +33,7 @@ 'use strict'; const co = require('co'); -var sqlBase = require('./baseConnector'); +const connectorUtilities = require('./connectorUtilities'); const db = require("dmdb"); const config = require('config'); @@ -146,7 +146,7 @@ exports.upsert = function(ctx, task, opt_updateUserIndex) { let values = []; let cbInsert = task.callback; if (task.callback) { - let userCallback = new sqlBase.UserCallback(); + let userCallback = new connectorUtilities.UserCallback(); userCallback.fromValues(task.userIndex, task.callback); cbInsert = userCallback.toSQLInsert(); } @@ -166,7 +166,7 @@ exports.upsert = function(ctx, task, opt_updateUserIndex) { sqlCommand += `WHEN MATCHED THEN UPDATE SET last_open_date = ${p9}`; if (task.callback) { let p10 = addSqlParam(JSON.stringify(task.callback), values); - sqlCommand += `, callback = CONCAT(callback , '${sqlBase.UserCallback.prototype.delimiter}{"userIndex":' , (user_index + 1) , ',"callback":', ${p10}, '}')`; + sqlCommand += `, callback = CONCAT(callback , '${connectorUtilities.UserCallback.prototype.delimiter}{"userIndex":' , (user_index + 1) , ',"callback":', ${p10}, '}')`; } if (task.baseurl) { let p11 = addSqlParam(task.baseurl, values); diff --git a/DocService/sources/mySqlBaseConnector.js b/DocService/sources/mySqlBaseConnector.js index 5f5bfc5b..249d17a2 100644 --- a/DocService/sources/mySqlBaseConnector.js +++ b/DocService/sources/mySqlBaseConnector.js @@ -33,7 +33,7 @@ 'use strict'; var mysql = require('mysql2'); -var sqlBase = require('./baseConnector'); +var connectorUtilities = require('./connectorUtilities'); const config = require('config'); const configSql = config.get('services.CoAuthoring.sql'); @@ -92,7 +92,7 @@ exports.upsert = function(ctx, task, opt_updateUserIndex) { let values = []; let cbInsert = task.callback; if (task.callback) { - let userCallback = new sqlBase.UserCallback(); + let userCallback = new connectorUtilities.UserCallback(); userCallback.fromValues(task.userIndex, task.callback); cbInsert = userCallback.toSQLInsert(); } @@ -111,7 +111,7 @@ exports.upsert = function(ctx, task, opt_updateUserIndex) { ` last_open_date = ${p9}`; if (task.callback) { let p10 = addSqlParam(JSON.stringify(task.callback), values); - sqlCommand += `, callback = CONCAT(callback , '${sqlBase.UserCallback.prototype.delimiter}{"userIndex":' , (user_index + 1) , ',"callback":', ${p10}, '}')`; + sqlCommand += `, callback = CONCAT(callback , '${connectorUtilities.UserCallback.prototype.delimiter}{"userIndex":' , (user_index + 1) , ',"callback":', ${p10}, '}')`; } if (task.baseurl) { let p11 = addSqlParam(task.baseurl, values); diff --git a/DocService/sources/oracleBaseConnector.js b/DocService/sources/oracleBaseConnector.js new file mode 100644 index 00000000..b58182b3 --- /dev/null +++ b/DocService/sources/oracleBaseConnector.js @@ -0,0 +1,174 @@ +/* + * (c) Copyright Ascensio System SIA 2010-2023 + * + * This program is a free software product. You can redistribute it and/or + * modify it under the terms of the GNU Affero General Public License (AGPL) + * version 3 as published by the Free Software Foundation. In accordance with + * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect + * that Ascensio System SIA expressly excludes the warranty of non-infringement + * of any third-party rights. + * + * This program is distributed WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For + * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html + * + * You can contact Ascensio System SIA at 20A-6 Ernesta Birznieka-Upish + * street, Riga, Latvia, EU, LV-1050. + * + * The interactive user interfaces in modified source and object code versions + * of the Program must display Appropriate Legal Notices, as required under + * Section 5 of the GNU AGPL version 3. + * + * Pursuant to Section 7(b) of the License you must retain the original Product + * logo when distributing the program. Pursuant to Section 7(e) we decline to + * grant you any rights under trademark law for use of our trademarks. + * + * All the Product's GUI elements, including illustrations and icon sets, as + * well as technical writing content are licensed under the terms of the + * Creative Commons Attribution-ShareAlike 4.0 International. See the License + * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode + * + */ + +'use strict'; + +const oracledb = require('oracledb'); +const config = require('config'); +const connectorUtilities = require("./connectorUtilities"); + +const configSql = config.get('services.CoAuthoring.sql'); +const cfgTableResult = config.get('services.CoAuthoring.sql.tableResult'); + +const connectionConfiguration = { + user: 'pdbadmin', // configSql.get('dbUser') + password: 'admin', // configSql.get('dbPass') + connectString: '192.168.5.7:1521/xepdb1', // `${configSql.get('dbHost')}:${configSql.get('dbPort')}/${configSql.get('dbName')}` + poolMin: 0, + poolMax: 10 // configSql.get('connectionlimit') +}; +let pool = null; + +async function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes, opt_noLog, opt_values) { + try { + if (!pool) { + pool = await oracledb.createPool(connectionConfiguration); + } + + const connection = await oracledb.getConnection(); + + const handler = (error, result) => { + if (error) { + if (!opt_noLog) { + ctx.logger.error('sqlQuery error sqlCommand: %s: %s', sqlCommand.slice(0, 50), error.stack); + } + + callbackFunction?.(error); + + return; + } + + let output = result; + if (!opt_noModifyRes) { + output = { rows: [], affectedRows: 0 }; + + if (result?.rowsAffected) { + output = { affectedRows: result.rowsAffected }; + } + + if (result?.rows) { + output = { rows: result.rows }; + } + } + + connection.close(); + callbackFunction?.(error, output); + }; + + if (opt_values) { + connection.execute(sqlCommand, opt_values, handler); + } else { + connection.execute(sqlCommand, handler); + } + } catch (error) { + if (!opt_noLog) { + ctx.logger.error('sqlQuery error while pool manipulation: %s', error.stack); + } + + callbackFunction?.(error); + } +} + +function addSqlParameter(parameter, accumulatedArray) { + const currentIndex = accumulatedArray.push(parameter) - 1; + return `:${currentIndex}`; +} + +function concatParams(val1, val2) { + return `COALESCE(${val1}, '') || COALESCE(${val2}, '')`; +} + +function upsert(ctx, task, opt_updateUserIndex) { + return new Promise((resolve, reject) => { + task.completeDefaults(); + + let cbInsert = task.callback; + if (task.callback) { + const userCallback = new connectorUtilities.UserCallback(); + userCallback.fromValues(task.userIndex, task.callback); + cbInsert = userCallback.toSQLInsert(); + } + + const dateNow = new Date(); + const values = []; + const valuesPlaceholder = [ + addSqlParameter(task.tenant, values), + addSqlParameter(task.key, values), + addSqlParameter(task.status, values), + addSqlParameter(task.statusInfo, values), + addSqlParameter(dateNow, values), + addSqlParameter(task.userIndex, values), + addSqlParameter(task.changeId, values), + addSqlParameter(cbInsert, values), + addSqlParameter(task.baseurl, values) + ]; + + let callback = ''; + if (task.callback) { + const parameter = addSqlParameter(JSON.stringify(task.callback), values); + callback = `, callback = CONCAT(callback , '${connectorUtilities.UserCallback.prototype.delimiter}{"userIndex":' , (user_index + 1) , ',"callback":', ${parameter}, '}')`; + } + + let baseUrl = ''; + if (task.baseurl) { + const parameter = addSqlParameter(task.baseurl, values); + baseUrl = `, baseurl = ${parameter}`; + } + + let userIndex = ''; + if (opt_updateUserIndex) { + userIndex = ', user_index = user_index + 1'; + } + + const updateQuery = `last_open_date = ${addSqlParameter(dateNow, values)}${callback}${baseUrl}${userIndex}` + const condition = `tenant = ${valuesPlaceholder[0]} AND id = ${valuesPlaceholder[1]}` + + let mergeSqlCommand = `MERGE INTO ${cfgTableResult} USING DUAL searching_alias ON (${condition})` + + ` WHEN MATCHED THEN UPDATE SET ${updateQuery}` + + ` WHEN NOT MATCHED THEN INSERT (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl) VALUES (${valuesPlaceholder.join(', ')});`; + + exports.sqlQuery(ctx, mergeSqlCommand, function(error, result) { + if (error) { + reject(error); + } else { + resolve(result); + } + }, undefined, undefined, values); + }); +} + +module.exports = { + sqlQuery, + addSqlParameter, + concatParams, + upsert +} \ No newline at end of file diff --git a/DocService/sources/postgreSqlBaseConnector.js b/DocService/sources/postgreSqlBaseConnector.js index d185ef0f..9a3b8a5a 100644 --- a/DocService/sources/postgreSqlBaseConnector.js +++ b/DocService/sources/postgreSqlBaseConnector.js @@ -35,7 +35,7 @@ var pg = require('pg'); var co = require('co'); var types = require('pg').types; -var sqlBase = require('./baseConnector'); +const connectorUtilities = require('./connectorUtilities'); const config = require('config'); var configSql = config.get('services.CoAuthoring.sql'); const cfgTableResult = config.get('services.CoAuthoring.sql.tableResult'); @@ -109,7 +109,7 @@ function getUpsertString(task, values) { let dateNow = new Date(); let cbInsert = task.callback; if (isSupportOnConflict && task.callback) { - let userCallback = new sqlBase.UserCallback(); + let userCallback = new connectorUtilities.UserCallback(); userCallback.fromValues(task.userIndex, task.callback); cbInsert = userCallback.toSQLInsert(); } @@ -130,7 +130,7 @@ function getUpsertString(task, values) { sqlCommand += ` ON CONFLICT (tenant, id) DO UPDATE SET last_open_date = ${p9}`; if (task.callback) { let p10 = addSqlParam(JSON.stringify(task.callback), values); - sqlCommand += `, callback = ${cfgTableResult}.callback || '${sqlBase.UserCallback.prototype.delimiter}{"userIndex":' `; + sqlCommand += `, callback = ${cfgTableResult}.callback || '${connectorUtilities.UserCallback.prototype.delimiter}{"userIndex":' `; sqlCommand += ` || (${cfgTableResult}.user_index + 1)::text || ',"callback":' || ${p10}::text || '}'`; } if (task.baseurl) { From 9059ab83009e70d36ce2c321d98e617f25341f46 Mon Sep 17 00:00:00 2001 From: Georgii Petrov Date: Fri, 23 Jun 2023 17:16:09 +0300 Subject: [PATCH 02/13] [ds] Oracle base connector pt.2 --- DocService/sources/baseConnector.js | 2 +- DocService/sources/oracleBaseConnector.js | 36 ++++++++++++++++------- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/DocService/sources/baseConnector.js b/DocService/sources/baseConnector.js index b0e0b441..f1a14222 100644 --- a/DocService/sources/baseConnector.js +++ b/DocService/sources/baseConnector.js @@ -283,7 +283,7 @@ exports.healthCheck = function (ctx) { let sql; switch (dbType) { case sqlDataBaseType.oracle: { - sql = 'SELECT 1 FROM DUAL;'; + sql = 'SELECT 1 FROM DUAL'; break; } default: { diff --git a/DocService/sources/oracleBaseConnector.js b/DocService/sources/oracleBaseConnector.js index b58182b3..9303131d 100644 --- a/DocService/sources/oracleBaseConnector.js +++ b/DocService/sources/oracleBaseConnector.js @@ -40,15 +40,18 @@ const configSql = config.get('services.CoAuthoring.sql'); const cfgTableResult = config.get('services.CoAuthoring.sql.tableResult'); const connectionConfiguration = { - user: 'pdbadmin', // configSql.get('dbUser') - password: 'admin', // configSql.get('dbPass') - connectString: '192.168.5.7:1521/xepdb1', // `${configSql.get('dbHost')}:${configSql.get('dbPort')}/${configSql.get('dbName')}` + user: configSql.get('dbUser'), + password: configSql.get('dbPass'), + connectString: `${configSql.get('dbHost')}:${configSql.get('dbPort')}/${configSql.get('dbName')}`, poolMin: 0, - poolMax: 10 // configSql.get('connectionlimit') + poolMax: configSql.get('connectionlimit') }; let pool = null; async function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes, opt_noLog, opt_values) { + // Query must not have any ';' in oracle connector. + const correctedSql = sqlCommand.replace(/;/g, ''); + try { if (!pool) { pool = await oracledb.createPool(connectionConfiguration); @@ -59,9 +62,10 @@ async function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes, opt_ const handler = (error, result) => { if (error) { if (!opt_noLog) { - ctx.logger.error('sqlQuery error sqlCommand: %s: %s', sqlCommand.slice(0, 50), error.stack); + ctx.logger.error('sqlQuery error sqlCommand: %s: %s', correctedSql.slice(0, 50), error.stack); } + connection.close(); callbackFunction?.(error); return; @@ -80,15 +84,27 @@ async function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes, opt_ } } - connection.close(); callbackFunction?.(error, output); }; if (opt_values) { - connection.execute(sqlCommand, opt_values, handler); + connection.execute(correctedSql, opt_values, handler); } else { - connection.execute(sqlCommand, handler); + connection.execute(correctedSql, handler); } + + // Transaction must be committed otherwise connector roll it back. + connection.execute('COMMIT', (error, result) => { + if (error) { + if (!opt_noLog) { + ctx.logger.error('sqlQuery error sqlCommand: %s: %s', correctedSql.slice(0, 50), error.stack); + } + + callbackFunction?.(error); + } + + connection.close(); + }); } catch (error) { if (!opt_noLog) { ctx.logger.error('sqlQuery error while pool manipulation: %s', error.stack); @@ -152,9 +168,9 @@ function upsert(ctx, task, opt_updateUserIndex) { const updateQuery = `last_open_date = ${addSqlParameter(dateNow, values)}${callback}${baseUrl}${userIndex}` const condition = `tenant = ${valuesPlaceholder[0]} AND id = ${valuesPlaceholder[1]}` - let mergeSqlCommand = `MERGE INTO ${cfgTableResult} USING DUAL searching_alias ON (${condition})` + let mergeSqlCommand = `MERGE INTO ${cfgTableResult} USING DUAL ON (${condition})` + ` WHEN MATCHED THEN UPDATE SET ${updateQuery}` - + ` WHEN NOT MATCHED THEN INSERT (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl) VALUES (${valuesPlaceholder.join(', ')});`; + + ` WHEN NOT MATCHED THEN INSERT (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl) VALUES (${valuesPlaceholder.join(', ')})`; exports.sqlQuery(ctx, mergeSqlCommand, function(error, result) { if (error) { From f43db03f83baa1ddb25eb52ea7bb52f0cd07ac88 Mon Sep 17 00:00:00 2001 From: Georgii Petrov Date: Thu, 29 Jun 2023 07:25:05 +0300 Subject: [PATCH 03/13] [ds] Oracle base connector pt.3 --- schema/oracle/createdb.sql | 40 +++++++++++++++++++++++++++++++++++++ schema/oracle/removedb.sql | 5 +++++ schema/oracle/removetbl.sql | 6 ++++++ 3 files changed, 51 insertions(+) create mode 100644 schema/oracle/createdb.sql create mode 100644 schema/oracle/removedb.sql create mode 100644 schema/oracle/removetbl.sql diff --git a/schema/oracle/createdb.sql b/schema/oracle/createdb.sql new file mode 100644 index 00000000..c500de32 --- /dev/null +++ b/schema/oracle/createdb.sql @@ -0,0 +1,40 @@ +-- You must be logged in as SYS(sysdba) user. +-- Here, "onlyoffice" is a PBD(service) name. +alter session set container = onlyoffice; + +-- In tables creation section "onlyoffice" is a user name. +-- ---------------------------- +-- Table structure for doc_changes +-- ---------------------------- + +CREATE TABLE onlyoffice.doc_changes ( + tenant NVARCHAR2(255) NOT NULL, + id NVARCHAR2(255) NOT NULL, + change_id NUMBER NOT NULL, + user_id NVARCHAR2(255) NOT NULL, + user_id_original NVARCHAR2(255) NOT NULL, + user_name NVARCHAR2(255) NOT NULL, + change_data NCLOB NOT NULL, + change_date TIMESTAMP NOT NULL, + CONSTRAINT doc_changes_unsigned_int CHECK (change_id between 0 and 4294967295) +); + +-- ---------------------------- +-- Table structure for task_result +-- ---------------------------- + +CREATE TABLE onlyoffice.task_result ( + tenant NVARCHAR2(255) NOT NULL, + id NVARCHAR2(255) NOT NULL, + status NUMBER NOT NULL, + status_info NUMBER NOT NULL, + created_at TIMESTAMP DEFAULT SYSDATE, -- check format + last_open_date TIMESTAMP NOT NULL, + user_index NUMBER DEFAULT 1 NOT NULL, + change_id NUMBER DEFAULT 0 NOT NULL, + callback NCLOB NOT NULL, + baseurl NCLOB NOT NULL, + password NCLOB NULL, + additional NCLOB NULL, + CONSTRAINT task_result_unsigned_int CHECK (user_index BETWEEN 0 AND 4294967295 AND change_id BETWEEN 0 AND 4294967295) +); diff --git a/schema/oracle/removedb.sql b/schema/oracle/removedb.sql new file mode 100644 index 00000000..ce295644 --- /dev/null +++ b/schema/oracle/removedb.sql @@ -0,0 +1,5 @@ +-- You must be logged in as SYS(sysdba) user. +-- Here, "onlyoffice" is a PBD(service) name. +alter session set container = onlyoffice; + +DROP USER onlyoffice CASCADE; \ No newline at end of file diff --git a/schema/oracle/removetbl.sql b/schema/oracle/removetbl.sql new file mode 100644 index 00000000..fc1cad26 --- /dev/null +++ b/schema/oracle/removetbl.sql @@ -0,0 +1,6 @@ +-- You must be logged in as SYS(sysdba) user. +-- Here, "onlyoffice" is a PBD(service) name. +alter session set container = onlyoffice; + +DROP TABLE onlyoffice.doc_changes CASCADE CONSTRAINTS PURGE; +DROP TABLE onlyoffice.task_result CASCADE CONSTRAINTS PURGE; \ No newline at end of file From 5c8b3b4859951efa34bd2420f68e01330dc7e8ad Mon Sep 17 00:00:00 2001 From: Georgii Petrov Date: Fri, 30 Jun 2023 14:00:46 +0300 Subject: [PATCH 04/13] [ds] Oracle base connector pt.4 --- DocService/sources/oracleBaseConnector.js | 96 +++++++++++++++-------- schema/oracle/createdb.sql | 2 +- schema/oracle/removetbl.sql | 2 +- 3 files changed, 64 insertions(+), 36 deletions(-) diff --git a/DocService/sources/oracleBaseConnector.js b/DocService/sources/oracleBaseConnector.js index 9303131d..f1e632f4 100644 --- a/DocService/sources/oracleBaseConnector.js +++ b/DocService/sources/oracleBaseConnector.js @@ -48,6 +48,38 @@ const connectionConfiguration = { }; let pool = null; +oracledb.fetchAsString = [ oracledb.NCLOB ]; +oracledb.autoCommit = true; + +function columnsToLowercase(rows) { + const formattedRows = []; + for (const row of rows) { + const newRow = {}; + for (const column in row) { + if (row.hasOwnProperty(column)) { + newRow[column.toLowerCase()] = row[column]; + } + } + + formattedRows.push(newRow); + } + + return formattedRows; +} + +function reconfigureParametersBinding(parameters) { + if (!parameters) { + return {}; + } + + const objectConfiguration = {}; + for (const index in parameters) { + objectConfiguration[`:${index}`] = parameters[index]; + } + + return objectConfiguration; +} + async function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes, opt_noLog, opt_values) { // Query must not have any ';' in oracle connector. const correctedSql = sqlCommand.replace(/;/g, ''); @@ -57,12 +89,12 @@ async function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes, opt_ pool = await oracledb.createPool(connectionConfiguration); } - const connection = await oracledb.getConnection(); + const connection = await pool.getConnection(); const handler = (error, result) => { if (error) { if (!opt_noLog) { - ctx.logger.error('sqlQuery error sqlCommand: %s: %s', correctedSql.slice(0, 50), error.stack); + ctx.logger.error('sqlQuery error sqlCommand: %s: %s', correctedSql, error.stack); } connection.close(); @@ -71,40 +103,23 @@ async function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes, opt_ return; } - let output = result; - if (!opt_noModifyRes) { - output = { rows: [], affectedRows: 0 }; + let output = { rows: [], affectedRows: 0 }; + if (result?.rowsAffected) { + output = { affectedRows: result.rowsAffected }; + } - if (result?.rowsAffected) { - output = { affectedRows: result.rowsAffected }; - } - - if (result?.rows) { - output = { rows: result.rows }; - } + if (result?.rows) { + output = !opt_noModifyRes ? columnsToLowercase(result.rows) : result.rows; } callbackFunction?.(error, output); }; - if (opt_values) { - connection.execute(correctedSql, opt_values, handler); - } else { - connection.execute(correctedSql, handler); - } + const bondedValues = reconfigureParametersBinding(opt_values); + const outputFormat = { outFormat: !opt_noModifyRes ? oracledb.OUT_FORMAT_OBJECT : oracledb.OUT_FORMAT_ARRAY }; + connection.execute(correctedSql, bondedValues, outputFormat, handler); - // Transaction must be committed otherwise connector roll it back. - connection.execute('COMMIT', (error, result) => { - if (error) { - if (!opt_noLog) { - ctx.logger.error('sqlQuery error sqlCommand: %s: %s', correctedSql.slice(0, 50), error.stack); - } - - callbackFunction?.(error); - } - - connection.close(); - }); + connection.close(); } catch (error) { if (!opt_noLog) { ctx.logger.error('sqlQuery error while pool manipulation: %s', error.stack); @@ -119,8 +134,20 @@ function addSqlParameter(parameter, accumulatedArray) { return `:${currentIndex}`; } -function concatParams(val1, val2) { - return `COALESCE(${val1}, '') || COALESCE(${val2}, '')`; +function concatParams(firstParameter, secondParameter) { + return `${firstParameter} || ${secondParameter} || ''`; +} + +function getTableColumns(ctx, tableName) { + return new Promise((resolve, reject) => { + sqlQuery(ctx, `SELECT LOWER(column_name) AS column_name FROM user_tab_columns WHERE table_name = '${tableName.toUpperCase()}'`, function (error, result) { + if (error) { + reject(error); + } else { + resolve(result); + } + }); + }); } function upsert(ctx, task, opt_updateUserIndex) { @@ -151,7 +178,7 @@ function upsert(ctx, task, opt_updateUserIndex) { let callback = ''; if (task.callback) { const parameter = addSqlParameter(JSON.stringify(task.callback), values); - callback = `, callback = CONCAT(callback , '${connectorUtilities.UserCallback.prototype.delimiter}{"userIndex":' , (user_index + 1) , ',"callback":', ${parameter}, '}')`; + callback = `, callback = callback || '${connectorUtilities.UserCallback.prototype.delimiter}{"userIndex":' || (user_index + 1) || ',"callback":' || ${parameter} || '}'`; } let baseUrl = ''; @@ -172,13 +199,13 @@ function upsert(ctx, task, opt_updateUserIndex) { + ` WHEN MATCHED THEN UPDATE SET ${updateQuery}` + ` WHEN NOT MATCHED THEN INSERT (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl) VALUES (${valuesPlaceholder.join(', ')})`; - exports.sqlQuery(ctx, mergeSqlCommand, function(error, result) { + sqlQuery(ctx, mergeSqlCommand, function(error, result) { if (error) { reject(error); } else { resolve(result); } - }, undefined, undefined, values); + }, false, false, values); }); } @@ -186,5 +213,6 @@ module.exports = { sqlQuery, addSqlParameter, concatParams, + getTableColumns, upsert } \ No newline at end of file diff --git a/schema/oracle/createdb.sql b/schema/oracle/createdb.sql index c500de32..f2370fe2 100644 --- a/schema/oracle/createdb.sql +++ b/schema/oracle/createdb.sql @@ -1,6 +1,6 @@ -- You must be logged in as SYS(sysdba) user. -- Here, "onlyoffice" is a PBD(service) name. -alter session set container = onlyoffice; +alter session set container = onlyoffice; --test name: xepdb1; -- In tables creation section "onlyoffice" is a user name. -- ---------------------------- diff --git a/schema/oracle/removetbl.sql b/schema/oracle/removetbl.sql index fc1cad26..ba79fbde 100644 --- a/schema/oracle/removetbl.sql +++ b/schema/oracle/removetbl.sql @@ -1,6 +1,6 @@ -- You must be logged in as SYS(sysdba) user. -- Here, "onlyoffice" is a PBD(service) name. -alter session set container = onlyoffice; +alter session set container = xepdb1; DROP TABLE onlyoffice.doc_changes CASCADE CONSTRAINTS PURGE; DROP TABLE onlyoffice.task_result CASCADE CONSTRAINTS PURGE; \ No newline at end of file From fff79d9413ea92d86bc61a60e4b79cd836c4370e Mon Sep 17 00:00:00 2001 From: Georgii Petrov Date: Tue, 4 Jul 2023 05:28:32 +0300 Subject: [PATCH 05/13] [ds] Oracle base connector pt.5 --- DocService/sources/baseConnector.js | 7 +- DocService/sources/oracleBaseConnector.js | 135 +++++++++++++++------- schema/oracle/createdb.sql | 8 +- schema/oracle/removetbl.sql | 2 +- 4 files changed, 103 insertions(+), 49 deletions(-) diff --git a/DocService/sources/baseConnector.js b/DocService/sources/baseConnector.js index f1a14222..31c24da1 100644 --- a/DocService/sources/baseConnector.js +++ b/DocService/sources/baseConnector.js @@ -113,7 +113,6 @@ exports.insertChangesPromise = function (ctx, objChanges, docId, index, user) { } else { return exports.insertChangesPromiseCompatibility(ctx, objChanges, docId, index, user); } - }; function _getDateTime2(oDate) { return oDate.toISOString().slice(0, 19).replace('T', ' '); @@ -127,10 +126,12 @@ function _insertChangesCallback (ctx, startIndex, objChanges, docId, index, user if (i === l) return; + const indexBytes = 4; + const timeBytes = 8; for (; i < l; ++i, ++index) { - //44 - length of "($1001,... $1007)," + //49 - length of "($1001,... $1008)," //4 is max utf8 bytes per symbol - lengthUtf8Row = 44 + 4 * (docId.length + user.id.length + user.idOriginal.length + user.username.length + objChanges[i].change.length) + 4 + 8; + lengthUtf8Row = 49 + 4 * (ctx.tenant.length + docId.length + user.id.length + user.idOriginal.length + user.username.length + objChanges[i].change.length) + indexBytes + timeBytes; if (lengthUtf8Row + lengthUtf8Current >= maxPacketSize && i > startIndex) { sqlCommand += ';'; (function(tmpStart, tmpIndex) { diff --git a/DocService/sources/oracleBaseConnector.js b/DocService/sources/oracleBaseConnector.js index f1e632f4..1ab4dc21 100644 --- a/DocService/sources/oracleBaseConnector.js +++ b/DocService/sources/oracleBaseConnector.js @@ -37,7 +37,8 @@ const config = require('config'); const connectorUtilities = require("./connectorUtilities"); const configSql = config.get('services.CoAuthoring.sql'); -const cfgTableResult = config.get('services.CoAuthoring.sql.tableResult'); +const cfgTableResult = configSql.get('tableResult'); +const cfgMaxPacketSize = configSql.get('max_allowed_packet'); const connectionConfiguration = { user: configSql.get('dbUser'), @@ -48,7 +49,7 @@ const connectionConfiguration = { }; let pool = null; -oracledb.fetchAsString = [ oracledb.NCLOB ]; +oracledb.fetchAsString = [ oracledb.NCLOB, oracledb.CLOB ]; oracledb.autoCommit = true; function columnsToLowercase(rows) { @@ -67,19 +68,6 @@ function columnsToLowercase(rows) { return formattedRows; } -function reconfigureParametersBinding(parameters) { - if (!parameters) { - return {}; - } - - const objectConfiguration = {}; - for (const index in parameters) { - objectConfiguration[`:${index}`] = parameters[index]; - } - - return objectConfiguration; -} - async function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes, opt_noLog, opt_values) { // Query must not have any ';' in oracle connector. const correctedSql = sqlCommand.replace(/;/g, ''); @@ -97,29 +85,32 @@ async function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes, opt_ ctx.logger.error('sqlQuery error sqlCommand: %s: %s', correctedSql, error.stack); } - connection.close(); callbackFunction?.(error); return; } - let output = { rows: [], affectedRows: 0 }; - if (result?.rowsAffected) { - output = { affectedRows: result.rowsAffected }; - } + connection.close(); - if (result?.rows) { - output = !opt_noModifyRes ? columnsToLowercase(result.rows) : result.rows; + let output = { rows: [], affectedRows: 0 }; + if (!opt_noModifyRes) { + if (result?.rowsAffected) { + output = { affectedRows: result.rowsAffected }; + } + + if (result?.rows) { + output = columnsToLowercase(result.rows); + } + } else { + output = result; } callbackFunction?.(error, output); }; - const bondedValues = reconfigureParametersBinding(opt_values); + const bondedValues = opt_values ?? []; const outputFormat = { outFormat: !opt_noModifyRes ? oracledb.OUT_FORMAT_OBJECT : oracledb.OUT_FORMAT_ARRAY }; connection.execute(correctedSql, bondedValues, outputFormat, handler); - - connection.close(); } catch (error) { if (!opt_noLog) { ctx.logger.error('sqlQuery error while pool manipulation: %s', error.stack); @@ -162,18 +153,11 @@ function upsert(ctx, task, opt_updateUserIndex) { } const dateNow = new Date(); + const values = []; - const valuesPlaceholder = [ - addSqlParameter(task.tenant, values), - addSqlParameter(task.key, values), - addSqlParameter(task.status, values), - addSqlParameter(task.statusInfo, values), - addSqlParameter(dateNow, values), - addSqlParameter(task.userIndex, values), - addSqlParameter(task.changeId, values), - addSqlParameter(cbInsert, values), - addSqlParameter(task.baseurl, values) - ]; + const tenant = addSqlParameter(task.tenant, values); + const id = addSqlParameter(task.key, values); + const lastOpenDate = addSqlParameter(dateNow, values); let callback = ''; if (task.callback) { @@ -192,12 +176,25 @@ function upsert(ctx, task, opt_updateUserIndex) { userIndex = ', user_index = user_index + 1'; } - const updateQuery = `last_open_date = ${addSqlParameter(dateNow, values)}${callback}${baseUrl}${userIndex}` - const condition = `tenant = ${valuesPlaceholder[0]} AND id = ${valuesPlaceholder[1]}` + const updateQuery = `last_open_date = ${lastOpenDate}${callback}${baseUrl}${userIndex}` + const condition = `tenant = ${tenant} AND id = ${id}` let mergeSqlCommand = `MERGE INTO ${cfgTableResult} USING DUAL ON (${condition})` - + ` WHEN MATCHED THEN UPDATE SET ${updateQuery}` - + ` WHEN NOT MATCHED THEN INSERT (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl) VALUES (${valuesPlaceholder.join(', ')})`; + + ` WHEN MATCHED THEN UPDATE SET ${updateQuery}`; + + const valuesPlaceholder = [ + addSqlParameter(task.tenant, values), + addSqlParameter(task.key, values), + addSqlParameter(task.status, values), + addSqlParameter(task.statusInfo, values), + addSqlParameter(dateNow, values), + addSqlParameter(task.userIndex, values), + addSqlParameter(task.changeId, values), + addSqlParameter(cbInsert, values), + addSqlParameter(task.baseurl, values) + ]; + + mergeSqlCommand += ` WHEN NOT MATCHED THEN INSERT (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl) VALUES (${valuesPlaceholder.join(', ')})`; sqlQuery(ctx, mergeSqlCommand, function(error, result) { if (error) { @@ -209,10 +206,66 @@ function upsert(ctx, task, opt_updateUserIndex) { }); } +async function insertChanges(ctx, tableChanges, startIndex, objChanges, docId, index, user, callback) { + if (startIndex === objChanges.length) { + return; + } + + let packetCapacityReached = false; + let currentIndex = startIndex; + let lengthUtf8Current = 'INSERT ALL SELECT 1 FROM DUAL'.length; + let insertAllSqlCommand = 'INSERT ALL '; + const values = []; + + const maxInsertionClauseLength = `INTO ${tableChanges} VALUES(:9991,:9992,:9993,:9994,:9995,:9996,:9997,:9998) `.length; + const indexBytes = 4; + const timeBytes = 8; + for (; currentIndex < objChanges.length; ++currentIndex, ++index) { + // 4 bytes is maximum for utf8 symbol. + const lengthUtf8Row = maxInsertionClauseLength + indexBytes + timeBytes + + 4 * (ctx.tenant.length + docId.length + user.id.length + user.idOriginal.length + user.username.length + objChanges[currentIndex].change.length); + + if (lengthUtf8Row + lengthUtf8Current >= cfgMaxPacketSize && currentIndex > startIndex) { + packetCapacityReached = true; + break; + } + + const valuesPlaceholder= [ + addSqlParameter(ctx.tenant, values), + addSqlParameter(docId, values), + addSqlParameter(index, values), + addSqlParameter(user.id, values), + addSqlParameter(user.idOriginal, values), + addSqlParameter(user.username, values), + addSqlParameter(objChanges[currentIndex].change, values), + addSqlParameter(objChanges[currentIndex].time, values) + ]; + + insertAllSqlCommand += `INTO ${tableChanges} VALUES(${valuesPlaceholder.join(',')}) `; + } + + insertAllSqlCommand += 'SELECT 1 FROM DUAL'; + + await sqlQuery(ctx, insertAllSqlCommand, function (error, result) { + if (error) { + callback(error, null, true); + + return; + } + + if (packetCapacityReached) { + insertChanges(ctx, tableChanges, currentIndex, objChanges, docId, index, user, callback); + } else { + callback(error, result, true); + } + }, false, false, values); +} + module.exports = { sqlQuery, addSqlParameter, concatParams, getTableColumns, - upsert + upsert, + insertChanges } \ No newline at end of file diff --git a/schema/oracle/createdb.sql b/schema/oracle/createdb.sql index f2370fe2..85263087 100644 --- a/schema/oracle/createdb.sql +++ b/schema/oracle/createdb.sql @@ -1,6 +1,6 @@ -- You must be logged in as SYS(sysdba) user. -- Here, "onlyoffice" is a PBD(service) name. -alter session set container = onlyoffice; --test name: xepdb1; +alter session set container = onlyoffice; -- In tables creation section "onlyoffice" is a user name. -- ---------------------------- @@ -28,12 +28,12 @@ CREATE TABLE onlyoffice.task_result ( id NVARCHAR2(255) NOT NULL, status NUMBER NOT NULL, status_info NUMBER NOT NULL, - created_at TIMESTAMP DEFAULT SYSDATE, -- check format + created_at TIMESTAMP DEFAULT SYSDATE NOT NULL, last_open_date TIMESTAMP NOT NULL, user_index NUMBER DEFAULT 1 NOT NULL, change_id NUMBER DEFAULT 0 NOT NULL, - callback NCLOB NOT NULL, - baseurl NCLOB NOT NULL, + callback NCLOB, -- codebase uses '' as default values here, but Oracle treat '' as NULL, so NULL permitted for this value. + baseurl NCLOB, -- codebase uses '' as default values here, but Oracle treat '' as NULL, so NULL permitted for this value. password NCLOB NULL, additional NCLOB NULL, CONSTRAINT task_result_unsigned_int CHECK (user_index BETWEEN 0 AND 4294967295 AND change_id BETWEEN 0 AND 4294967295) diff --git a/schema/oracle/removetbl.sql b/schema/oracle/removetbl.sql index ba79fbde..fc1cad26 100644 --- a/schema/oracle/removetbl.sql +++ b/schema/oracle/removetbl.sql @@ -1,6 +1,6 @@ -- You must be logged in as SYS(sysdba) user. -- Here, "onlyoffice" is a PBD(service) name. -alter session set container = xepdb1; +alter session set container = onlyoffice; DROP TABLE onlyoffice.doc_changes CASCADE CONSTRAINTS PURGE; DROP TABLE onlyoffice.task_result CASCADE CONSTRAINTS PURGE; \ No newline at end of file From 554c0c00128b3362b703dd1906e07bb42383d8b0 Mon Sep 17 00:00:00 2001 From: Georgii Petrov Date: Fri, 7 Jul 2023 10:38:40 +0300 Subject: [PATCH 06/13] [ds] Oracle base connector pt.6 --- DocService/sources/oracleBaseConnector.js | 24 +++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/DocService/sources/oracleBaseConnector.js b/DocService/sources/oracleBaseConnector.js index 1ab4dc21..cd33e0d7 100644 --- a/DocService/sources/oracleBaseConnector.js +++ b/DocService/sources/oracleBaseConnector.js @@ -48,6 +48,7 @@ const connectionConfiguration = { poolMax: configSql.get('connectionlimit') }; let pool = null; +let affectedRowsTotal = 0; oracledb.fetchAsString = [ oracledb.NCLOB, oracledb.CLOB ]; oracledb.autoCommit = true; @@ -195,13 +196,25 @@ function upsert(ctx, task, opt_updateUserIndex) { ]; mergeSqlCommand += ` WHEN NOT MATCHED THEN INSERT (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl) VALUES (${valuesPlaceholder.join(', ')})`; - + sqlQuery(ctx, mergeSqlCommand, function(error, result) { if (error) { reject(error); - } else { - resolve(result); + + return; } + + sqlQuery(ctx, `SELECT user_index FROM ${cfgTableResult} WHERE tenant = :0 AND id = :1`, function (selectError, selectResult) { + if (selectError) { + reject(selectError); + + return; + } + + const row = selectResult.pop(); + result.insertId = row?.['user_index'] ?? 0; + resolve(result); + }, false, false, [task.tenant, task.key]); }, false, false, values); }); } @@ -246,16 +259,19 @@ async function insertChanges(ctx, tableChanges, startIndex, objChanges, docId, i insertAllSqlCommand += 'SELECT 1 FROM DUAL'; - await sqlQuery(ctx, insertAllSqlCommand, function (error, result) { + sqlQuery(ctx, insertAllSqlCommand, function (error, result) { if (error) { callback(error, null, true); return; } + affectedRowsTotal += result.affectedRows; if (packetCapacityReached) { insertChanges(ctx, tableChanges, currentIndex, objChanges, docId, index, user, callback); } else { + result.affectedRows = affectedRowsTotal; + affectedRowsTotal = 0; callback(error, result, true); } }, false, false, values); From 3d66727cfa0841ddb7ed847d56a0d953bf080a8c Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Tue, 11 Jul 2023 09:58:48 +0300 Subject: [PATCH 07/13] [oracle] Test RETURNING clause --- DocService/sources/oracleBaseConnector.js | 29 +++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/DocService/sources/oracleBaseConnector.js b/DocService/sources/oracleBaseConnector.js index cd33e0d7..200141ac 100644 --- a/DocService/sources/oracleBaseConnector.js +++ b/DocService/sources/oracleBaseConnector.js @@ -155,6 +155,35 @@ function upsert(ctx, task, opt_updateUserIndex) { const dateNow = new Date(); + // const values = []; + // + // const lastOpenDate = addSqlParameter(dateNow, values); + // + // let callback = ''; + // if (task.callback) { + // const parameter = addSqlParameter(JSON.stringify(task.callback), values); + // callback = `, callback = callback || '${connectorUtilities.UserCallback.prototype.delimiter}{"userIndex":' || (user_index + 1) || ',"callback":' || ${parameter} || '}'`; + // } + // + // let baseUrl = ''; + // if (task.baseurl) { + // const parameter = addSqlParameter(task.baseurl, values); + // baseUrl = `, baseurl = ${parameter}`; + // } + // + // let userIndex = ''; + // if (opt_updateUserIndex) { + // userIndex = ', user_index = user_index + 1'; + // } + // + // const updateQuery = `last_open_date = ${lastOpenDate}${callback}${baseUrl}${userIndex}` + // const tenant = addSqlParameter(task.tenant, values); + // const id = addSqlParameter(task.key, values); + // const condition = `tenant = ${tenant} AND id = ${id}` + // + // const returning = addSqlParameter({ type: oracledb.NUMBER, dir: oracledb.BIND_OUT }, values); + // let mergeSqlCommand = `UPDATE ${cfgTableResult} SET ${updateQuery} WHERE ${condition} RETURNING user_index INTO ${returning}`; + const values = []; const tenant = addSqlParameter(task.tenant, values); const id = addSqlParameter(task.key, values); From b85dad459a5541e178f2993065e609bc30837a41 Mon Sep 17 00:00:00 2001 From: Georgii Petrov Date: Mon, 17 Jul 2023 16:55:03 +0300 Subject: [PATCH 08/13] [ds] Bugs fixing in Oracle base connector --- DocService/sources/oracleBaseConnector.js | 182 +++++++++++----------- schema/oracle/createdb.sql | 2 + 2 files changed, 92 insertions(+), 92 deletions(-) diff --git a/DocService/sources/oracleBaseConnector.js b/DocService/sources/oracleBaseConnector.js index 200141ac..480030d6 100644 --- a/DocService/sources/oracleBaseConnector.js +++ b/DocService/sources/oracleBaseConnector.js @@ -34,7 +34,7 @@ const oracledb = require('oracledb'); const config = require('config'); -const connectorUtilities = require("./connectorUtilities"); +const connectorUtilities = require('./connectorUtilities'); const configSql = config.get('services.CoAuthoring.sql'); const cfgTableResult = configSql.get('tableResult'); @@ -48,7 +48,6 @@ const connectionConfiguration = { poolMax: configSql.get('connectionlimit') }; let pool = null; -let affectedRowsTotal = 0; oracledb.fetchAsString = [ oracledb.NCLOB, oracledb.CLOB ]; oracledb.autoCommit = true; @@ -73,12 +72,13 @@ async function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes, opt_ // Query must not have any ';' in oracle connector. const correctedSql = sqlCommand.replace(/;/g, ''); + let connection = null; try { if (!pool) { pool = await oracledb.createPool(connectionConfiguration); } - const connection = await pool.getConnection(); + connection = await pool.getConnection(); const handler = (error, result) => { if (error) { @@ -91,8 +91,6 @@ async function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes, opt_ return; } - connection.close(); - let output = { rows: [], affectedRows: 0 }; if (!opt_noModifyRes) { if (result?.rowsAffected) { @@ -111,13 +109,17 @@ async function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes, opt_ const bondedValues = opt_values ?? []; const outputFormat = { outFormat: !opt_noModifyRes ? oracledb.OUT_FORMAT_OBJECT : oracledb.OUT_FORMAT_ARRAY }; - connection.execute(correctedSql, bondedValues, outputFormat, handler); + await connection.execute(correctedSql, bondedValues, outputFormat, handler); } catch (error) { if (!opt_noLog) { ctx.logger.error('sqlQuery error while pool manipulation: %s', error.stack); } callbackFunction?.(error); + } finally { + if (connection) { + connection.close(); + } } } @@ -142,6 +144,40 @@ function getTableColumns(ctx, tableName) { }); } +function makeUpdateSql(dateNow, task, values, opt_updateUserIndex) { + const lastOpenDate = addSqlParameter(dateNow, values); + + let callback = ''; + if (task.callback) { + const parameter = addSqlParameter(JSON.stringify(task.callback), values); + callback = `, callback = callback || '${connectorUtilities.UserCallback.prototype.delimiter}{"userIndex":' || (user_index + 1) || ',"callback":' || ${parameter} || '}'`; + } + + let baseUrl = ''; + if (task.baseurl) { + const parameter = addSqlParameter(task.baseurl, values); + baseUrl = `, baseurl = ${parameter}`; + } + + let userIndex = ''; + if (opt_updateUserIndex) { + userIndex = ', user_index = user_index + 1'; + } + + const updateQuery = `last_open_date = ${lastOpenDate}${callback}${baseUrl}${userIndex}` + const tenant = addSqlParameter(task.tenant, values); + const id = addSqlParameter(task.key, values); + const condition = `tenant = ${tenant} AND id = ${id}` + + const returning = addSqlParameter({ type: oracledb.NUMBER, dir: oracledb.BIND_OUT }, values); + + return `UPDATE ${cfgTableResult} SET ${updateQuery} WHERE ${condition} RETURNING user_index INTO ${returning}`; +} + +function getReturnedValue(returned) { + return returned?.outBinds?.pop()?.pop(); +} + function upsert(ctx, task, opt_updateUserIndex) { return new Promise((resolve, reject) => { task.completeDefaults(); @@ -155,100 +191,61 @@ function upsert(ctx, task, opt_updateUserIndex) { const dateNow = new Date(); - // const values = []; - // - // const lastOpenDate = addSqlParameter(dateNow, values); - // - // let callback = ''; - // if (task.callback) { - // const parameter = addSqlParameter(JSON.stringify(task.callback), values); - // callback = `, callback = callback || '${connectorUtilities.UserCallback.prototype.delimiter}{"userIndex":' || (user_index + 1) || ',"callback":' || ${parameter} || '}'`; - // } - // - // let baseUrl = ''; - // if (task.baseurl) { - // const parameter = addSqlParameter(task.baseurl, values); - // baseUrl = `, baseurl = ${parameter}`; - // } - // - // let userIndex = ''; - // if (opt_updateUserIndex) { - // userIndex = ', user_index = user_index + 1'; - // } - // - // const updateQuery = `last_open_date = ${lastOpenDate}${callback}${baseUrl}${userIndex}` - // const tenant = addSqlParameter(task.tenant, values); - // const id = addSqlParameter(task.key, values); - // const condition = `tenant = ${tenant} AND id = ${id}` - // - // const returning = addSqlParameter({ type: oracledb.NUMBER, dir: oracledb.BIND_OUT }, values); - // let mergeSqlCommand = `UPDATE ${cfgTableResult} SET ${updateQuery} WHERE ${condition} RETURNING user_index INTO ${returning}`; - - const values = []; - const tenant = addSqlParameter(task.tenant, values); - const id = addSqlParameter(task.key, values); - const lastOpenDate = addSqlParameter(dateNow, values); - - let callback = ''; - if (task.callback) { - const parameter = addSqlParameter(JSON.stringify(task.callback), values); - callback = `, callback = callback || '${connectorUtilities.UserCallback.prototype.delimiter}{"userIndex":' || (user_index + 1) || ',"callback":' || ${parameter} || '}'`; - } - - let baseUrl = ''; - if (task.baseurl) { - const parameter = addSqlParameter(task.baseurl, values); - baseUrl = `, baseurl = ${parameter}`; - } - - let userIndex = ''; - if (opt_updateUserIndex) { - userIndex = ', user_index = user_index + 1'; - } - - const updateQuery = `last_open_date = ${lastOpenDate}${callback}${baseUrl}${userIndex}` - const condition = `tenant = ${tenant} AND id = ${id}` - - let mergeSqlCommand = `MERGE INTO ${cfgTableResult} USING DUAL ON (${condition})` - + ` WHEN MATCHED THEN UPDATE SET ${updateQuery}`; - - const valuesPlaceholder = [ - addSqlParameter(task.tenant, values), - addSqlParameter(task.key, values), - addSqlParameter(task.status, values), - addSqlParameter(task.statusInfo, values), - addSqlParameter(dateNow, values), - addSqlParameter(task.userIndex, values), - addSqlParameter(task.changeId, values), - addSqlParameter(cbInsert, values), - addSqlParameter(task.baseurl, values) + const insertValues = []; + const insertValuesPlaceholder = [ + addSqlParameter(task.tenant, insertValues), + addSqlParameter(task.key, insertValues), + addSqlParameter(task.status, insertValues), + addSqlParameter(task.statusInfo, insertValues), + addSqlParameter(dateNow, insertValues), + addSqlParameter(task.userIndex, insertValues), + addSqlParameter(task.changeId, insertValues), + addSqlParameter(cbInsert, insertValues), + addSqlParameter(task.baseurl, insertValues) ]; - mergeSqlCommand += ` WHEN NOT MATCHED THEN INSERT (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl) VALUES (${valuesPlaceholder.join(', ')})`; - - sqlQuery(ctx, mergeSqlCommand, function(error, result) { - if (error) { - reject(error); + const returned = addSqlParameter({ type: oracledb.NUMBER, dir: oracledb.BIND_OUT }, insertValues); + let sqlInsertTry = `INSERT INTO ${cfgTableResult} (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl) ` + + `VALUES(${insertValuesPlaceholder.join(', ')}) RETURNING user_index INTO ${returned}`; + + sqlQuery(ctx, sqlInsertTry, function (insertError, insertResult) { + if (insertResult) { + const insertId = getReturnedValue(insertResult); + resolve({ affectedRows: insertResult.affectedRows, insertId }); return; } - sqlQuery(ctx, `SELECT user_index FROM ${cfgTableResult} WHERE tenant = :0 AND id = :1`, function (selectError, selectResult) { - if (selectError) { - reject(selectError); + if (insertError) { + if (insertError.code !== 'ORA-00001') { + reject(insertError); return; } - const row = selectResult.pop(); - result.insertId = row?.['user_index'] ?? 0; - resolve(result); - }, false, false, [task.tenant, task.key]); - }, false, false, values); + const values = []; + sqlQuery(ctx, makeUpdateSql(dateNow, task, values, opt_updateUserIndex), function (updateError, updateResult) { + if (updateError) { + reject(updateError); + + return; + } + + const insertId = getReturnedValue(updateResult); + resolve({ affectedRows: updateResult.affectedRows, insertId }); + }, true, false, values); + } + }, true, true, insertValues); }); } -async function insertChanges(ctx, tableChanges, startIndex, objChanges, docId, index, user, callback) { +function insertChanges(ctx, tableChanges, startIndex, objChanges, docId, index, user, callback) { + let affectedRowsTotal = 0; + + return insertChangesClosure.apply({ affectedRowsTotal }, arguments); +} + +async function insertChangesClosure(ctx, tableChanges, startIndex, objChanges, docId, index, user, callback) { if (startIndex === objChanges.length) { return; } @@ -284,23 +281,24 @@ async function insertChanges(ctx, tableChanges, startIndex, objChanges, docId, i ]; insertAllSqlCommand += `INTO ${tableChanges} VALUES(${valuesPlaceholder.join(',')}) `; + lengthUtf8Current += lengthUtf8Row; } insertAllSqlCommand += 'SELECT 1 FROM DUAL'; - sqlQuery(ctx, insertAllSqlCommand, function (error, result) { + sqlQuery(ctx, insertAllSqlCommand, (error, result) => { if (error) { callback(error, null, true); return; } - affectedRowsTotal += result.affectedRows; + this.affectedRowsTotal += result.affectedRows; if (packetCapacityReached) { - insertChanges(ctx, tableChanges, currentIndex, objChanges, docId, index, user, callback); + insertChanges.apply(this, [ctx, tableChanges, currentIndex, objChanges, docId, index, user, callback]); } else { - result.affectedRows = affectedRowsTotal; - affectedRowsTotal = 0; + result.affectedRows = this.affectedRowsTotal; + this.affectedRowsTotal = 0; callback(error, result, true); } }, false, false, values); diff --git a/schema/oracle/createdb.sql b/schema/oracle/createdb.sql index 85263087..0a5b6a76 100644 --- a/schema/oracle/createdb.sql +++ b/schema/oracle/createdb.sql @@ -16,6 +16,7 @@ CREATE TABLE onlyoffice.doc_changes ( user_name NVARCHAR2(255) NOT NULL, change_data NCLOB NOT NULL, change_date TIMESTAMP NOT NULL, + CONSTRAINT doc_changes_unique UNIQUE (tenant, id, change_id), CONSTRAINT doc_changes_unsigned_int CHECK (change_id between 0 and 4294967295) ); @@ -36,5 +37,6 @@ CREATE TABLE onlyoffice.task_result ( baseurl NCLOB, -- codebase uses '' as default values here, but Oracle treat '' as NULL, so NULL permitted for this value. password NCLOB NULL, additional NCLOB NULL, + CONSTRAINT task_result_unique UNIQUE (tenant, id), CONSTRAINT task_result_unsigned_int CHECK (user_index BETWEEN 0 AND 4294967295 AND change_id BETWEEN 0 AND 4294967295) ); From 971941798cda020786cf76f260062ab3a76859cd Mon Sep 17 00:00:00 2001 From: Georgii Petrov Date: Tue, 25 Jul 2023 11:03:50 +0300 Subject: [PATCH 09/13] Bug fix in upsert() function --- DocService/sources/oracleBaseConnector.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DocService/sources/oracleBaseConnector.js b/DocService/sources/oracleBaseConnector.js index 480030d6..dd657a94 100644 --- a/DocService/sources/oracleBaseConnector.js +++ b/DocService/sources/oracleBaseConnector.js @@ -211,7 +211,7 @@ function upsert(ctx, task, opt_updateUserIndex) { sqlQuery(ctx, sqlInsertTry, function (insertError, insertResult) { if (insertResult) { const insertId = getReturnedValue(insertResult); - resolve({ affectedRows: insertResult.affectedRows, insertId }); + resolve({ affectedRows: 1, insertId }); return; } @@ -232,7 +232,7 @@ function upsert(ctx, task, opt_updateUserIndex) { } const insertId = getReturnedValue(updateResult); - resolve({ affectedRows: updateResult.affectedRows, insertId }); + resolve({ affectedRows: 2, insertId }); }, true, false, values); } }, true, true, insertValues); From 5ff0c77c36853b3f6a4a80d69884a18807e4bb7b Mon Sep 17 00:00:00 2001 From: Georgii Petrov Date: Tue, 1 Aug 2023 16:05:55 +0300 Subject: [PATCH 10/13] [ds] insert() optimization --- DocService/sources/oracleBaseConnector.js | 263 +++++++++++----------- 1 file changed, 131 insertions(+), 132 deletions(-) diff --git a/DocService/sources/oracleBaseConnector.js b/DocService/sources/oracleBaseConnector.js index dd657a94..5dca8532 100644 --- a/DocService/sources/oracleBaseConnector.js +++ b/DocService/sources/oracleBaseConnector.js @@ -68,7 +68,14 @@ function columnsToLowercase(rows) { return formattedRows; } -async function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes, opt_noLog, opt_values) { +function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes = false, opt_noLog = false, opt_values = []) { + return executeQuery(ctx, sqlCommand, opt_values, opt_noModifyRes, opt_noLog).then( + result => callbackFunction?.(null, result), + error => callbackFunction?.(error) + ); +} + +async function executeQuery(ctx, sqlCommand, values = [], noModifyRes = false, noLog = false) { // Query must not have any ';' in oracle connector. const correctedSql = sqlCommand.replace(/;/g, ''); @@ -80,42 +87,57 @@ async function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes, opt_ connection = await pool.getConnection(); - const handler = (error, result) => { - if (error) { - if (!opt_noLog) { - ctx.logger.error('sqlQuery error sqlCommand: %s: %s', correctedSql, error.stack); - } + const bondedValues = values ?? []; + const outputFormat = { outFormat: !noModifyRes ? oracledb.OUT_FORMAT_OBJECT : oracledb.OUT_FORMAT_ARRAY }; + const result = await connection.execute(correctedSql, bondedValues, outputFormat); - callbackFunction?.(error); - - return; + let output = { rows: [], affectedRows: 0 }; + if (!noModifyRes) { + if (result?.rowsAffected) { + output = { affectedRows: result.rowsAffected }; } - let output = { rows: [], affectedRows: 0 }; - if (!opt_noModifyRes) { - if (result?.rowsAffected) { - output = { affectedRows: result.rowsAffected }; - } - - if (result?.rows) { - output = columnsToLowercase(result.rows); - } - } else { - output = result; + if (result?.rows) { + output = columnsToLowercase(result.rows); } - - callbackFunction?.(error, output); - }; - - const bondedValues = opt_values ?? []; - const outputFormat = { outFormat: !opt_noModifyRes ? oracledb.OUT_FORMAT_OBJECT : oracledb.OUT_FORMAT_ARRAY }; - await connection.execute(correctedSql, bondedValues, outputFormat, handler); - } catch (error) { - if (!opt_noLog) { - ctx.logger.error('sqlQuery error while pool manipulation: %s', error.stack); + } else { + output = result; } - callbackFunction?.(error); + return output; + } catch (error) { + if (!noLog) { + ctx.logger.error('sqlQuery error sqlCommand: %s: %s', correctedSql, error.stack); + // ctx.logger.error('sqlQuery error while pool manipulation: %s', error.stack); + + throw error; + } + } finally { + if (connection) { + connection.close(); + } + } +} + +async function executeBunch(ctx, sqlCommand, values = [], noLog = false) { + let connection = null; + try { + if (!pool) { + pool = await oracledb.createPool(connectionConfiguration); + } + + connection = await pool.getConnection(); + + const result = await connection.executeMany(sqlCommand, values); + + return { affectedRows: result?.rowsAffected ?? 0 }; + } catch (error) { + if (!noLog) { + ctx.logger.error('sqlQuery error sqlCommand: %s: %s', sqlCommand, error.stack); + // ctx.logger.error('sqlQuery error while pool manipulation: %s', error.stack); + + throw error; + } } finally { if (connection) { connection.close(); @@ -133,15 +155,7 @@ function concatParams(firstParameter, secondParameter) { } function getTableColumns(ctx, tableName) { - return new Promise((resolve, reject) => { - sqlQuery(ctx, `SELECT LOWER(column_name) AS column_name FROM user_tab_columns WHERE table_name = '${tableName.toUpperCase()}'`, function (error, result) { - if (error) { - reject(error); - } else { - resolve(result); - } - }); - }); + return executeQuery(ctx, `SELECT LOWER(column_name) AS column_name FROM user_tab_columns WHERE table_name = '${tableName.toUpperCase()}'`); } function makeUpdateSql(dateNow, task, values, opt_updateUserIndex) { @@ -178,90 +192,79 @@ function getReturnedValue(returned) { return returned?.outBinds?.pop()?.pop(); } -function upsert(ctx, task, opt_updateUserIndex) { - return new Promise((resolve, reject) => { - task.completeDefaults(); +async function upsert(ctx, task, opt_updateUserIndex) { + task.completeDefaults(); - let cbInsert = task.callback; - if (task.callback) { - const userCallback = new connectorUtilities.UserCallback(); - userCallback.fromValues(task.userIndex, task.callback); - cbInsert = userCallback.toSQLInsert(); + let cbInsert = task.callback; + if (task.callback) { + const userCallback = new connectorUtilities.UserCallback(); + userCallback.fromValues(task.userIndex, task.callback); + cbInsert = userCallback.toSQLInsert(); + } + + const dateNow = new Date(); + + const insertValues = []; + const insertValuesPlaceholder = [ + addSqlParameter(task.tenant, insertValues), + addSqlParameter(task.key, insertValues), + addSqlParameter(task.status, insertValues), + addSqlParameter(task.statusInfo, insertValues), + addSqlParameter(dateNow, insertValues), + addSqlParameter(task.userIndex, insertValues), + addSqlParameter(task.changeId, insertValues), + addSqlParameter(cbInsert, insertValues), + addSqlParameter(task.baseurl, insertValues) + ]; + + const returned = addSqlParameter({ type: oracledb.NUMBER, dir: oracledb.BIND_OUT }, insertValues); + let sqlInsertTry = `INSERT INTO ${cfgTableResult} (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl) ` + + `VALUES(${insertValuesPlaceholder.join(', ')}) RETURNING user_index INTO ${returned}`; + + try { + const insertResult = await executeQuery(ctx, sqlInsertTry, insertValues, true, true); + const insertId = getReturnedValue(insertResult); + + return { affectedRows: 1, insertId }; + } catch (insertError) { + if (insertError.code !== 'ORA-00001') { + throw insertError; } - const dateNow = new Date(); + const values = []; + const updateResult = await executeQuery(ctx, makeUpdateSql(dateNow, task, values, opt_updateUserIndex), values, true); + const insertId = getReturnedValue(updateResult); - const insertValues = []; - const insertValuesPlaceholder = [ - addSqlParameter(task.tenant, insertValues), - addSqlParameter(task.key, insertValues), - addSqlParameter(task.status, insertValues), - addSqlParameter(task.statusInfo, insertValues), - addSqlParameter(dateNow, insertValues), - addSqlParameter(task.userIndex, insertValues), - addSqlParameter(task.changeId, insertValues), - addSqlParameter(cbInsert, insertValues), - addSqlParameter(task.baseurl, insertValues) - ]; - - const returned = addSqlParameter({ type: oracledb.NUMBER, dir: oracledb.BIND_OUT }, insertValues); - let sqlInsertTry = `INSERT INTO ${cfgTableResult} (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl) ` - + `VALUES(${insertValuesPlaceholder.join(', ')}) RETURNING user_index INTO ${returned}`; - - sqlQuery(ctx, sqlInsertTry, function (insertError, insertResult) { - if (insertResult) { - const insertId = getReturnedValue(insertResult); - resolve({ affectedRows: 1, insertId }); - - return; - } - - if (insertError) { - if (insertError.code !== 'ORA-00001') { - reject(insertError); - - return; - } - - const values = []; - sqlQuery(ctx, makeUpdateSql(dateNow, task, values, opt_updateUserIndex), function (updateError, updateResult) { - if (updateError) { - reject(updateError); - - return; - } - - const insertId = getReturnedValue(updateResult); - resolve({ affectedRows: 2, insertId }); - }, true, false, values); - } - }, true, true, insertValues); - }); + return { affectedRows: 2, insertId }; + } } function insertChanges(ctx, tableChanges, startIndex, objChanges, docId, index, user, callback) { - let affectedRowsTotal = 0; - - return insertChangesClosure.apply({ affectedRowsTotal }, arguments); + insertChangesAsync(ctx, tableChanges, startIndex, objChanges, docId, index, user).then( + result => { ctx.logger.debug('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!TOTAL:', result.affectedRows); callback(null, result, true) }, + error => callback(error, null, true) + ); } -async function insertChangesClosure(ctx, tableChanges, startIndex, objChanges, docId, index, user, callback) { +async function insertChangesAsync(ctx, tableChanges, startIndex, objChanges, docId, index, user) { if (startIndex === objChanges.length) { - return; + return { affectedRows: 0 }; } + const parametersCount = 8; + const placeholderLength = ':9'.length; + // (parametersCount - 1) - separator symbols length. + const maxInsertStatementLength = `INSERT /*+ APPEND_VALUES*/INTO ${tableChanges} VALUES()`.length + placeholderLength * parametersCount + (parametersCount - 1); let packetCapacityReached = false; - let currentIndex = startIndex; - let lengthUtf8Current = 'INSERT ALL SELECT 1 FROM DUAL'.length; - let insertAllSqlCommand = 'INSERT ALL '; - const values = []; - const maxInsertionClauseLength = `INTO ${tableChanges} VALUES(:9991,:9992,:9993,:9994,:9995,:9996,:9997,:9998) `.length; + const values = []; const indexBytes = 4; const timeBytes = 8; + let lengthUtf8Current = 0; + let currentIndex = startIndex; for (; currentIndex < objChanges.length; ++currentIndex, ++index) { // 4 bytes is maximum for utf8 symbol. - const lengthUtf8Row = maxInsertionClauseLength + indexBytes + timeBytes + const lengthUtf8Row = maxInsertStatementLength + indexBytes + timeBytes + 4 * (ctx.tenant.length + docId.length + user.id.length + user.idOriginal.length + user.username.length + objChanges[currentIndex].change.length); if (lengthUtf8Row + lengthUtf8Current >= cfgMaxPacketSize && currentIndex > startIndex) { @@ -269,39 +272,35 @@ async function insertChangesClosure(ctx, tableChanges, startIndex, objChanges, d break; } - const valuesPlaceholder= [ - addSqlParameter(ctx.tenant, values), - addSqlParameter(docId, values), - addSqlParameter(index, values), - addSqlParameter(user.id, values), - addSqlParameter(user.idOriginal, values), - addSqlParameter(user.username, values), - addSqlParameter(objChanges[currentIndex].change, values), - addSqlParameter(objChanges[currentIndex].time, values) - ]; + const rowValues = { + '0': ctx.tenant, + '1': docId, + '2': index, + '3': user.id, + '4': user.idOriginal, + '5': user.username, + '6': objChanges[currentIndex].change, + '7': objChanges[currentIndex].time + }; - insertAllSqlCommand += `INTO ${tableChanges} VALUES(${valuesPlaceholder.join(',')}) `; + values.push(rowValues); lengthUtf8Current += lengthUtf8Row; } - insertAllSqlCommand += 'SELECT 1 FROM DUAL'; + const placeholder = []; + for (let i = 0; i < parametersCount; i++) { + placeholder.push(`:${i}`); + } - sqlQuery(ctx, insertAllSqlCommand, (error, result) => { - if (error) { - callback(error, null, true); + const sqlInsert = `INSERT /*+ APPEND_VALUES*/INTO ${tableChanges} VALUES(${placeholder.join(',')})` + const result = await executeBunch(ctx, sqlInsert, values); - return; - } + if (packetCapacityReached) { + const recursiveValue = await insertChangesAsync(ctx, tableChanges, currentIndex, objChanges, docId, index, user); + result.affectedRows += recursiveValue.affectedRows; + } - this.affectedRowsTotal += result.affectedRows; - if (packetCapacityReached) { - insertChanges.apply(this, [ctx, tableChanges, currentIndex, objChanges, docId, index, user, callback]); - } else { - result.affectedRows = this.affectedRowsTotal; - this.affectedRowsTotal = 0; - callback(error, result, true); - } - }, false, false, values); + return result; } module.exports = { From d73b26966bdd7a466d65408d0060039e5da739ed Mon Sep 17 00:00:00 2001 From: Georgii Petrov Date: Wed, 2 Aug 2023 14:27:11 +0300 Subject: [PATCH 11/13] [ds] Oracle connector fixes and improvements --- DocService/sources/oracleBaseConnector.js | 42 +++++++++++------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/DocService/sources/oracleBaseConnector.js b/DocService/sources/oracleBaseConnector.js index 5dca8532..19da8288 100644 --- a/DocService/sources/oracleBaseConnector.js +++ b/DocService/sources/oracleBaseConnector.js @@ -107,11 +107,10 @@ async function executeQuery(ctx, sqlCommand, values = [], noModifyRes = false, n return output; } catch (error) { if (!noLog) { - ctx.logger.error('sqlQuery error sqlCommand: %s: %s', correctedSql, error.stack); - // ctx.logger.error('sqlQuery error while pool manipulation: %s', error.stack); - - throw error; + ctx.logger.error(`sqlQuery() error while executing query: ${sqlCommand}\n${error.stack}`); } + + throw error; } finally { if (connection) { connection.close(); @@ -133,11 +132,10 @@ async function executeBunch(ctx, sqlCommand, values = [], noLog = false) { return { affectedRows: result?.rowsAffected ?? 0 }; } catch (error) { if (!noLog) { - ctx.logger.error('sqlQuery error sqlCommand: %s: %s', sqlCommand, error.stack); - // ctx.logger.error('sqlQuery error while pool manipulation: %s', error.stack); - - throw error; + ctx.logger.error(`sqlQuery() error while executing query: ${sqlCommand}\n${error.stack}`); } + + throw error; } finally { if (connection) { connection.close(); @@ -241,7 +239,7 @@ async function upsert(ctx, task, opt_updateUserIndex) { function insertChanges(ctx, tableChanges, startIndex, objChanges, docId, index, user, callback) { insertChangesAsync(ctx, tableChanges, startIndex, objChanges, docId, index, user).then( - result => { ctx.logger.debug('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!TOTAL:', result.affectedRows); callback(null, result, true) }, + result => callback(null, result, true), error => callback(error, null, true) ); } @@ -252,9 +250,9 @@ async function insertChangesAsync(ctx, tableChanges, startIndex, objChanges, doc } const parametersCount = 8; - const placeholderLength = ':9'.length; + const maxPlaceholderLength = ':99'.length; // (parametersCount - 1) - separator symbols length. - const maxInsertStatementLength = `INSERT /*+ APPEND_VALUES*/INTO ${tableChanges} VALUES()`.length + placeholderLength * parametersCount + (parametersCount - 1); + const maxInsertStatementLength = `INSERT /*+ APPEND_VALUES*/INTO ${tableChanges} VALUES()`.length + maxPlaceholderLength * parametersCount + (parametersCount - 1); let packetCapacityReached = false; const values = []; @@ -272,16 +270,18 @@ async function insertChangesAsync(ctx, tableChanges, startIndex, objChanges, doc break; } - const rowValues = { - '0': ctx.tenant, - '1': docId, - '2': index, - '3': user.id, - '4': user.idOriginal, - '5': user.username, - '6': objChanges[currentIndex].change, - '7': objChanges[currentIndex].time - }; + const parameters = [ + ctx.tenant, + docId, + index, + user.id, + user.idOriginal, + user.username, + objChanges[currentIndex].change, + objChanges[currentIndex].time + ]; + + const rowValues = { ...parameters }; values.push(rowValues); lengthUtf8Current += lengthUtf8Row; From e90826dcc314980a8d4c8fd7364a37f5359285a7 Mon Sep 17 00:00:00 2001 From: Georgii Petrov Date: Wed, 9 Aug 2023 05:57:07 +0300 Subject: [PATCH 12/13] [ds] Oracle connector fixes and improvements pt. 2 --- Common/config/default.json | 3 ++- DocService/sources/oracleBaseConnector.js | 14 ++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Common/config/default.json b/Common/config/default.json index 0ad3d348..21467a72 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -175,7 +175,8 @@ "connectionlimit": 10, "max_allowed_packet": 1048575, "pgPoolExtraOptions": {}, - "damengExtraOptions": {} + "damengExtraOptions": {}, + "oracleExtraOptions": {} }, "redis": { "name": "redis", diff --git a/DocService/sources/oracleBaseConnector.js b/DocService/sources/oracleBaseConnector.js index 19da8288..9747d292 100644 --- a/DocService/sources/oracleBaseConnector.js +++ b/DocService/sources/oracleBaseConnector.js @@ -47,6 +47,8 @@ const connectionConfiguration = { poolMin: 0, poolMax: configSql.get('connectionlimit') }; +const additionalOptions = configSql.get('oracleExtraOptions'); +const configuration = Object.assign({}, connectionConfiguration, additionalOptions); let pool = null; oracledb.fetchAsString = [ oracledb.NCLOB, oracledb.CLOB ]; @@ -82,7 +84,7 @@ async function executeQuery(ctx, sqlCommand, values = [], noModifyRes = false, n let connection = null; try { if (!pool) { - pool = await oracledb.createPool(connectionConfiguration); + pool = await oracledb.createPool(configuration); } connection = await pool.getConnection(); @@ -112,9 +114,7 @@ async function executeQuery(ctx, sqlCommand, values = [], noModifyRes = false, n throw error; } finally { - if (connection) { - connection.close(); - } + connection?.close(); } } @@ -122,7 +122,7 @@ async function executeBunch(ctx, sqlCommand, values = [], noLog = false) { let connection = null; try { if (!pool) { - pool = await oracledb.createPool(connectionConfiguration); + pool = await oracledb.createPool(configuration); } connection = await pool.getConnection(); @@ -137,9 +137,7 @@ async function executeBunch(ctx, sqlCommand, values = [], noLog = false) { throw error; } finally { - if (connection) { - connection.close(); - } + connection?.close(); } } From 7ab7a95ba62836dba5079292e44f1d93629d9ab8 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Wed, 9 Aug 2023 19:06:09 +0300 Subject: [PATCH 13/13] [sql] Change session to xepdb1 --- DocService/sources/oracleBaseConnector.js | 2 +- schema/oracle/createdb.sql | 2 +- schema/oracle/removedb.sql | 2 +- schema/oracle/removetbl.sql | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/DocService/sources/oracleBaseConnector.js b/DocService/sources/oracleBaseConnector.js index 9747d292..03fa7192 100644 --- a/DocService/sources/oracleBaseConnector.js +++ b/DocService/sources/oracleBaseConnector.js @@ -308,4 +308,4 @@ module.exports = { getTableColumns, upsert, insertChanges -} \ No newline at end of file +} diff --git a/schema/oracle/createdb.sql b/schema/oracle/createdb.sql index 0a5b6a76..89de67d8 100644 --- a/schema/oracle/createdb.sql +++ b/schema/oracle/createdb.sql @@ -1,6 +1,6 @@ -- You must be logged in as SYS(sysdba) user. -- Here, "onlyoffice" is a PBD(service) name. -alter session set container = onlyoffice; +alter session set container = xepdb1; -- In tables creation section "onlyoffice" is a user name. -- ---------------------------- diff --git a/schema/oracle/removedb.sql b/schema/oracle/removedb.sql index ce295644..e4e04ae2 100644 --- a/schema/oracle/removedb.sql +++ b/schema/oracle/removedb.sql @@ -1,5 +1,5 @@ -- You must be logged in as SYS(sysdba) user. -- Here, "onlyoffice" is a PBD(service) name. -alter session set container = onlyoffice; +alter session set container = xepdb1; DROP USER onlyoffice CASCADE; \ No newline at end of file diff --git a/schema/oracle/removetbl.sql b/schema/oracle/removetbl.sql index fc1cad26..ba79fbde 100644 --- a/schema/oracle/removetbl.sql +++ b/schema/oracle/removetbl.sql @@ -1,6 +1,6 @@ -- You must be logged in as SYS(sysdba) user. -- Here, "onlyoffice" is a PBD(service) name. -alter session set container = onlyoffice; +alter session set container = xepdb1; DROP TABLE onlyoffice.doc_changes CASCADE CONSTRAINTS PURGE; DROP TABLE onlyoffice.task_result CASCADE CONSTRAINTS PURGE; \ No newline at end of file