From 70e933cbf0e5aa42249bf11694f11ad0be3a5a2a Mon Sep 17 00:00:00 2001 From: konovalovsergey Date: Fri, 23 Sep 2016 17:14:17 +0300 Subject: [PATCH 1/2] ipfilter dns lookup --- Common/config/default.json | 5 ++++ Common/package.json | 2 ++ Common/sources/utils.js | 39 +++++++++++++++++++++++++++--- DocService/package.json | 1 - DocService/sources/DocsCoServer.js | 3 ++- DocService/sources/server.js | 8 ------ FileConverter/sources/converter.js | 3 ++- 7 files changed, 47 insertions(+), 14 deletions(-) diff --git a/Common/config/default.json b/Common/config/default.json index d613739f..81aa2238 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -39,6 +39,11 @@ "queueconverttask": "dc.converttask", "queueconvertresponse": "dc.convertresponse" }, + "dnscache": { + "enable" : true, + "ttl" : 300, + "cachesize" : 1000 + }, "services": { "CoAuthoring": { "server": { diff --git a/Common/package.json b/Common/package.json index cde73a51..9b56d39c 100644 --- a/Common/package.json +++ b/Common/package.json @@ -9,7 +9,9 @@ "aws-sdk": "^2.4.12", "co": "^4.6.0", "config": "^1.21.0", + "dnscache": "0.0.4", "escape-string-regexp": "^1.0.5", + "ipaddr.js": "^1.2.0", "log4js": "^0.6.38", "mime": "^1.3.4", "mkdirp": "^0.5.1", diff --git a/Common/sources/utils.js b/Common/sources/utils.js index d2a53305..d6ca4278 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -37,6 +37,13 @@ var request = require('request'); var co = require('co'); var URI = require("uri-js"); const escapeStringRegexp = require('escape-string-regexp'); +const ipaddr = require('ipaddr.js'); +var configDnsCache = config.get('dnscache'); +const dnscache = require('dnscache')({ + "enable": configDnsCache.get('enable'), + "ttl": configDnsCache.get('ttl'), + "cachesize": configDnsCache.get('cachesize'), + }); var constants = require('./constants'); var configIpFilter = config.get('services.CoAuthoring.ipfilter'); @@ -511,11 +518,25 @@ function* pipeFiles(from, to) { yield pipeStreams(fromStream, toStream, true); } exports.pipeFiles = co.wrap(pipeFiles); -function checkIpFilter(hostname) { +function checkIpFilter(ipString, opt_hostname) { var status = 0; - for (var i = 0; i < g_oIpFilterRules.length; ++i) { + var ip4; + var ip6; + if (ipaddr.isValid(ipString)) { + var ip = ipaddr.parse(ipString); + if ('ipv6' == ip.kind()) { + if (ip.isIPv4MappedAddress()) { + ip4 = ip.toIPv4Address().toString(); + } + ip6 = ip.toNormalizedString(); + } else { + ip4 = ip.toString(); + ip6 = ip.toIPv4MappedAddress().toNormalizedString(); + } + } + for (i = 0; i < g_oIpFilterRules.length; ++i) { var rule = g_oIpFilterRules[i]; - if (rule.exp.test(hostname)) { + if ((opt_hostname && rule.exp.test(opt_hostname)) || (ip4 && rule.exp.test(ip4)) || (ip6 && rule.exp.test(ip6))) { if (!rule.allow) { status = cfgIpFilterErrorCode; } @@ -525,3 +546,15 @@ function checkIpFilter(hostname) { return status; } exports.checkIpFilter = checkIpFilter; +function dnsLookup(hostname, options) { + return new Promise(function(resolve, reject) { + dnscache.lookup(hostname, options, function(err, addresses){ + if (err) { + reject(err); + } else { + resolve(addresses); + } + }); + }); +} +exports.dnsLookup = dnsLookup; \ No newline at end of file diff --git a/DocService/package.json b/DocService/package.json index cfb7d4e9..6b87daa7 100644 --- a/DocService/package.json +++ b/DocService/package.json @@ -12,7 +12,6 @@ "express": "^4.14.0", "fakeredis": "^1.0.3", "forwarded": "^0.1.0", - "ipaddr.js": "^1.2.0", "mime": "^1.3.4", "multiparty": "^4.1.2", "mysql": "^2.11.1", diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 65c37371..25f8ae0f 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -720,7 +720,8 @@ function* bindEvents(docId, callback, baseUrl, opt_userAction, opt_userData) { oCallbackUrl = parseUrl(callback); bChangeBase = c_oAscChangeBase.All; if (null !== oCallbackUrl) { - if (utils.checkIpFilter(oCallbackUrl.host) > 0) { + var hostIp = yield utils.dnsLookup(oCallbackUrl.host); + if (utils.checkIpFilter(hostIp, oCallbackUrl.host) > 0) { logger.error('checkIpFilter error: docId = %s;url = %s', docId, callback); //todo add new error type oCallbackUrl = null; diff --git a/DocService/sources/server.js b/DocService/sources/server.js index ffa0383c..e65a4417 100644 --- a/DocService/sources/server.js +++ b/DocService/sources/server.js @@ -105,7 +105,6 @@ if (cluster.isMaster) { const bodyParser = require("body-parser"); const mime = require('mime'); const forwarded = require('forwarded'); - const ipaddr = require('ipaddr.js'); const docsCoServer = require('./DocsCoServer'); const canvasService = require('./canvasservice'); const converterService = require('./converterservice'); @@ -167,13 +166,6 @@ if (cluster.isMaster) { if (cfgIpFilterEseForRequest) { var addresses = forwarded(req); var ipString = addresses[addresses.length - 1]; - //IPv6 -> IPv4 - if (ipaddr.IPv6.isValid(ipString)) { - var ip = ipaddr.IPv6.parse(ipString); - if (ip.isIPv4MappedAddress()) { - ipString = ip.toIPv4Address().toString(); - } - } status = utils.checkIpFilter(ipString); } if (status > 0) { diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index a712027e..4dfdb059 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -167,7 +167,8 @@ function* downloadFile(docId, uri, fileFrom) { var data = null; var downloadAttemptCount = 0; var urlParsed = url.parse(uri); - var filterStatus = utils.checkIpFilter(urlParsed.hostname); + var hostIp = yield utils.dnsLookup(urlParsed.hostname); + var filterStatus = utils.checkIpFilter(hostIp, urlParsed.hostname); if (0 == filterStatus) { while (!res && downloadAttemptCount++ < cfgDownloadAttemptMaxCount) { try { From b7efffcf3c5dfeb6ae21252701e4bb50f9581aea Mon Sep 17 00:00:00 2001 From: konovalovsergey Date: Mon, 3 Oct 2016 16:23:23 +0300 Subject: [PATCH 2/2] change several field to 4 byte int(like in mysql) because int8 returned as string --- schema/postgresql/createdb.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/schema/postgresql/createdb.sql b/schema/postgresql/createdb.sql index 2d05cab8..096d91d7 100644 --- a/schema/postgresql/createdb.sql +++ b/schema/postgresql/createdb.sql @@ -26,7 +26,7 @@ WITH (OIDS=FALSE); -- ---------------------------- CREATE TABLE IF NOT EXISTS "public"."doc_changes" ( "id" varchar(255) COLLATE "default" NOT NULL, -"change_id" int8 NOT NULL, +"change_id" int4 NOT NULL, "user_id" varchar(255) COLLATE "default" NOT NULL, "user_id_original" varchar(255) COLLATE "default" NOT NULL, "user_name" varchar(255) COLLATE "default" NOT NULL, @@ -42,17 +42,17 @@ WITH (OIDS=FALSE); CREATE TABLE IF NOT EXISTS "public"."task_result" ( "id" varchar(255) COLLATE "default" NOT NULL, "status" int2 NOT NULL, -"status_info" int8 NOT NULL, +"status_info" int4 NOT NULL, "last_open_date" timestamp without time zone NOT NULL, "title" varchar(255) COLLATE "default" NOT NULL, -"user_index" int8 NOT NULL DEFAULT 1, -"change_id" int8 NOT NULL DEFAULT 0, +"user_index" int4 NOT NULL DEFAULT 1, +"change_id" int4 NOT NULL DEFAULT 0, PRIMARY KEY ("id") ) WITH (OIDS=FALSE); --https://www.postgresql.org/docs/current/static/plpgsql-control-structures.html#PLPGSQL-UPSERT-EXAMPLE -CREATE OR REPLACE FUNCTION merge_db(_id varchar(255), _status int2, _status_info int8, _last_open_date timestamp without time zone, _title varchar(255), _user_index int8, _change_id int8, OUT isupdate char(5), OUT userindex int8) AS +CREATE OR REPLACE FUNCTION merge_db(_id varchar(255), _status int2, _status_info int4, _last_open_date timestamp without time zone, _title varchar(255), _user_index int4, _change_id int4, OUT isupdate char(5), OUT userindex int4) AS $$ DECLARE t_var "public"."task_result"."user_index"%TYPE;