From 9e15918810a1f4fca0848e2a4b8df334871b448e Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Wed, 22 Jun 2016 13:04:09 +0300
Subject: [PATCH 01/34] =?UTF-8?q?=D0=B8=D0=B7=20=D0=B4=D0=B0=D0=BD=D0=BD?=
 =?UTF-8?q?=D1=8B=D1=85=20=D0=B2=20callbackUrl=20=D1=83=D0=B1=D1=80=D0=B0?=
 =?UTF-8?q?=D0=BD=D1=8B=20action=20=D0=BD=D0=B0=20forcesave=20=D0=B8=20inf?=
 =?UTF-8?q?o,=20=D0=B4=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5=D0=BD=D0=B0=20?=
 =?UTF-8?q?=D0=BE=D1=88=D0=B8=D0=B1=D0=BA=D0=B0=20=D0=BD=D0=B0=20=D0=BF?=
 =?UTF-8?q?=D1=83=D1=81=D1=82=D0=BE=D0=B9=20callback,=20=D0=B4=D0=BE=D0=B1?=
 =?UTF-8?q?=D0=B0=D0=B2=D0=BB=D0=B5=D0=BD=D0=B0=20userdata=20=D0=B2=20info?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 Common/sources/commondefines.js     |  7 +++---
 DocService/sources/DocsCoServer.js  | 33 ++++++++++++-----------------
 DocService/sources/canvasservice.js |  2 +-
 3 files changed, 18 insertions(+), 24 deletions(-)

diff --git a/Common/sources/commondefines.js b/Common/sources/commondefines.js
index 36409551..8844a327 100644
--- a/Common/sources/commondefines.js
+++ b/Common/sources/commondefines.js
@@ -463,8 +463,8 @@ function OutputSfcData() {
   this['url'] = undefined;
   this['changesurl'] = undefined;
   this['changeshistory'] = undefined;
-  this['users'] = [];
-  this['actions'] = [];
+  this['users'] = undefined;
+  this['actions'] = undefined;
   this['mailMerge'] = undefined;
   this['userdata'] = undefined;
 }
@@ -670,8 +670,7 @@ var c_oAscEncodingsMap = {"437": 43, "720": 1, "737": 21, "775": 5, "850": 39, "
 var c_oAscCodePageUtf8 = 46;//65001
 var c_oAscUserAction = {
   Out: 0,
-  In: 1,
-  AllIn: 2
+  In: 1
 };
 var c_oAscServerCommandErrors = {
   NoError: 0,
diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js
index b4a9a5bc..cca15dca 100644
--- a/DocService/sources/DocsCoServer.js
+++ b/DocService/sources/DocsCoServer.js
@@ -531,7 +531,10 @@ function* sendServerRequest(docId, uri, postData) {
 function parseUrl(callbackUrl) {
   var result = null;
   try {
-    var parseObject = url.parse(decodeURIComponent(callbackUrl));
+    //делать decodeURIComponent не нужно http://expressjs.com/en/4x/api.html#app.settings.table
+    //по умолчанию express использует 'query parser' = 'extended', но даже в 'simple' версии делается decode
+    //percent-encoded characters within the query string will be assumed to use UTF-8 encoding
+    var parseObject = url.parse(callbackUrl);
     var isHttps = 'https:' === parseObject.protocol;
     var port = parseObject.port;
     if (!port) {
@@ -601,10 +604,10 @@ function* setForceSave(docId, lastSave, savePathDoc) {
  * @param callback
  * @param baseUrl
  */
-function* sendStatusDocument(docId, bChangeBase, userAction, callback, baseUrl) {
+function* sendStatusDocument(docId, bChangeBase, userAction, callback, baseUrl, opt_userData) {
   if (!callback) {
     var getRes = yield* getCallback(docId);
-    if(getRes) {
+    if (getRes) {
       callback = getRes.server;
     }
   }
@@ -634,21 +637,14 @@ function* sendStatusDocument(docId, bChangeBase, userAction, callback, baseUrl)
   var sendData = new commonDefines.OutputSfcData();
   sendData.setKey(docId);
   sendData.setStatus(status);
-  if(c_oAscServerStatus.Closed !== status){
+  if (c_oAscServerStatus.Closed !== status) {
     sendData.setUsers(participants);
-  } else {
-    sendData.setUsers(undefined);
   }
   if (userAction) {
-    var actions = [];
-    if (commonDefines.c_oAscUserAction.AllIn === userAction.type) {
-      for (var i = 0; i < participants.length; ++i) {
-        actions.push(new commonDefines.OutputAction(commonDefines.c_oAscUserAction.In, participants[i]));
-      }
-    } else {
-      actions.push(userAction);
-    }
-    sendData.setActions(actions);
+    sendData.setActions([userAction]);
+  }
+  if (opt_userData) {
+    sendData.setUserData(opt_userData);
   }
   var uri = callback.href;
   var replyData = null;
@@ -705,7 +701,7 @@ function dropUserFromDocument(docId, userId, description) {
 }
 
 // Подписка на эвенты:
-function* bindEvents(docId, callback, baseUrl, opt_userAction) {
+function* bindEvents(docId, callback, baseUrl, opt_userAction, opt_userData) {
   // Подписка на эвенты:
   // - если пользователей нет и изменений нет, то отсылаем статус "закрыто" и в базу не добавляем
   // - если пользователей нет, а изменения есть, то отсылаем статус "редактируем" без пользователей, но добавляем в базу
@@ -722,8 +718,7 @@ function* bindEvents(docId, callback, baseUrl, opt_userAction) {
     }
     bChangeBase = c_oAscChangeBase.All;
   }
-  var userAction = opt_userAction ? opt_userAction : new commonDefines.OutputAction(commonDefines.c_oAscUserAction.AllIn, null);
-  yield* sendStatusDocument(docId, bChangeBase, userAction, oCallbackUrl, baseUrl);
+  yield* sendStatusDocument(docId, bChangeBase, opt_userAction, oCallbackUrl, baseUrl, opt_userData);
   return commonDefines.c_oAscServerCommandErrors.NoError;
 }
 
@@ -2096,7 +2091,7 @@ exports.commandFromServer = function (req, res) {
         logger.debug('Start commandFromServer: docId = %s c = %s', docId, query.c);
         switch (query.c) {
           case 'info':
-            result = yield* bindEvents(docId, query.callback, utils.getBaseUrlByRequest(req));
+            result = yield* bindEvents(docId, query.callback, utils.getBaseUrlByRequest(req), undefined, query.userdata);
             break;
           case 'drop':
             if (query.userid) {
diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js
index 6d3624cf..8288477b 100644
--- a/DocService/sources/canvasservice.js
+++ b/DocService/sources/canvasservice.js
@@ -1 +1 @@
-/*
 * (c) Copyright Ascensio System SIA 2010-2016
 *
 * This program is a free software product. You can redistribute it and/or
 * modify it under the terms of the GNU Affero General Public License (AGPL)
 * version 3 as published by the Free Software Foundation. In accordance with
 * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect
 * that Ascensio System SIA expressly excludes the warranty of non-infringement
 * of any third-party rights.
 *
 * This program is distributed WITHOUT ANY WARRANTY; without even the implied
 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE. For
 * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
 *
 * You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia,
 * EU, LV-1021.
 *
 * The  interactive user interfaces in modified source and object code versions
 * of the Program must display Appropriate Legal Notices, as required under
 * Section 5 of the GNU AGPL version 3.
 *
 * Pursuant to Section 7(b) of the License you must retain the original Product
 * logo when distributing the program. Pursuant to Section 7(e) we decline to
 * grant you any rights under trademark law for use of our trademarks.
 *
 * All the Product's GUI elements, including illustrations and icon sets, as
 * well as technical writing content are licensed under the terms of the
 * Creative Commons Attribution-ShareAlike 4.0 International. See the License
 * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
 *
 */

var pathModule = require('path');
var urlModule = require('url');
var co = require('co');
var sqlBase = require('./baseConnector');
var docsCoServer = require('./DocsCoServer');
var taskResult = require('./taskresult');
var logger = require('./../../Common/sources/logger');
var utils = require('./../../Common/sources/utils');
var constants = require('./../../Common/sources/constants');
var commonDefines = require('./../../Common/sources/commondefines');
var storage = require('./../../Common/sources/storage-base');
var formatChecker = require('./../../Common/sources/formatchecker');
var statsDClient = require('./../../Common/sources/statsdclient');
var config = require('config');
var config_server = config.get('services.CoAuthoring.server');
var config_utils = config.get('services.CoAuthoring.utils');
var pubsubRedis = require('./pubsubRedis');


var cfgTypesUpload = config_utils.get('limits_image_types_upload');
var cfgTypesCopy = config_utils.get('limits_image_types_copy');
var cfgImageSize = config_server.get('limits_image_size');
var cfgImageDownloadTimeout = config_server.get('limits_image_download_timeout');
var cfgRedisPrefix = config.get('services.CoAuthoring.redis.prefix');

var SAVE_TYPE_PART_START = 0;
var SAVE_TYPE_PART = 1;
var SAVE_TYPE_COMPLETE = 2;
var SAVE_TYPE_COMPLETE_ALL = 3;

var clientStatsD = statsDClient.getClient();
var redisClient = pubsubRedis.getClientRedis();
var redisKeySaved = cfgRedisPrefix + constants.REDIS_KEY_SAVED;
var redisKeyShutdown = cfgRedisPrefix + constants.REDIS_KEY_SHUTDOWN;

function OutputDataWrap(type, data) {
  this['type'] = type;
  this['data'] = data;
}
OutputDataWrap.prototype = {
  fromObject: function(data) {
    this['type'] = data['type'];
    this['data'] = new OutputData();
    this['data'].fromObject(data['data']);
  },
  getType: function() {
    return this['type'];
  },
  setType: function(data) {
    this['type'] = data;
  },
  getData: function() {
    return this['data'];
  },
  setData: function(data) {
    this['data'] = data;
  }
};
function OutputData(type) {
  this['type'] = type;
  this['status'] = undefined;
  this['data'] = undefined;
}
OutputData.prototype = {
  fromObject: function(data) {
    this['type'] = data['type'];
    this['status'] = data['status'];
    this['data'] = data['data'];
  },
  getType: function() {
    return this['type'];
  },
  setType: function(data) {
    this['type'] = data;
  },
  getStatus: function() {
    return this['status'];
  },
  setStatus: function(data) {
    this['status'] = data;
  },
  getData: function() {
    return this['data'];
  },
  setData: function(data) {
    this['data'] = data;
  }
};

function* getOutputData(cmd, outputData, key, status, statusInfo, optConn, optAdditionalOutput) {
  var docId = cmd.getDocId();
  switch (status) {
    case taskResult.FileStatus.SaveVersion:
    case taskResult.FileStatus.UpdateVersion:
    case taskResult.FileStatus.Ok:
      if(taskResult.FileStatus.Ok == status) {
        outputData.setStatus('ok');
      } else if(taskResult.FileStatus.SaveVersion == status) {
        if (optConn && optConn.user.view) {
          outputData.setStatus('updateversion');
        } else {
          var updateMask = new taskResult.TaskResultData();
          updateMask.key = docId;
          updateMask.status = status;
          updateMask.statusInfo = statusInfo;
          var updateTask = new taskResult.TaskResultData();
          updateTask.status = taskResult.FileStatus.Ok;
          updateTask.statusInfo = constants.NO_ERROR;
          var updateIfRes = yield taskResult.updateIf(updateTask, updateMask);
          if (updateIfRes.affectedRows > 0) {
            outputData.setStatus('ok');
          } else {
            outputData.setStatus('updateversion');
          }
        }
      } else {
        outputData.setStatus('updateversion');
      }
      var command = cmd.getCommand();
      if ('open' != command && 'reopen' != command) {
        var strPath = key + '/' + cmd.getOutputPath();
        if (optConn) {
          var contentDisposition = cmd.getInline() ? constants.CONTENT_DISPOSITION_INLINE : constants.CONTENT_DISPOSITION_ATTACHMENT;
          outputData.setData(yield storage.getSignedUrl(optConn.baseUrl, strPath, null, cmd.getTitle(), contentDisposition));
        } else if (optAdditionalOutput) {
          optAdditionalOutput.needUrlKey = strPath;
          optAdditionalOutput.needUrlMethod = 2;
        }
      } else {
        if (optConn) {
          outputData.setData(yield storage.getSignedUrls(optConn.baseUrl, key));
        } else if (optAdditionalOutput) {
          optAdditionalOutput.needUrlKey = key;
          optAdditionalOutput.needUrlMethod = 0;
        }
      }
      break;
    case taskResult.FileStatus.NeedParams:
      outputData.setStatus('needparams');
      var settingsPath = key + '/' + 'settings.json';
      if (optConn) {
        outputData.setData(yield storage.getSignedUrl(optConn.baseUrl, settingsPath));
      } else if (optAdditionalOutput) {
        optAdditionalOutput.needUrlKey = settingsPath;
        optAdditionalOutput.needUrlMethod = 1;
      }
      break;
    case taskResult.FileStatus.NeedPassword:
      outputData.setStatus('needpassword');
      break;
    case taskResult.FileStatus.Err:
    case taskResult.FileStatus.ErrToReload:
      outputData.setStatus('err');
      outputData.setData(statusInfo);
      if (taskResult.FileStatus.ErrToReload == status) {
        yield cleanupCache(key);
      }
      break;
  }
}
function* addRandomKeyTaskCmd(cmd) {
  var task = yield* taskResult.addRandomKeyTask(cmd.getDocId());
  cmd.setSaveKey(task.key);
}
function* saveParts(cmd) {
  var result = false;
  var saveType = cmd.getSaveType();
  var filename;
  if (SAVE_TYPE_COMPLETE_ALL === saveType) {
    filename = 'Editor.bin';
  } else {
    filename = 'Editor' + (cmd.getSaveIndex() || '') + '.bin';
  }
  if (SAVE_TYPE_PART_START === saveType || SAVE_TYPE_COMPLETE_ALL === saveType) {
    yield* addRandomKeyTaskCmd(cmd);
  }
  if (cmd.getUrl()) {
    result = true;
  } else {
    var buffer = cmd.getData();
    yield storage.putObject(cmd.getSaveKey() + '/' + filename, buffer, buffer.length);
    //delete data to prevent serialize into json
    cmd.data = null;
    result = (SAVE_TYPE_COMPLETE_ALL === saveType || SAVE_TYPE_COMPLETE === saveType);
  }
  return result;
}
function getSaveTask(cmd) {
  cmd.setData(null);
  var queueData = new commonDefines.TaskQueueData();
  queueData.setCmd(cmd);
  queueData.setToFile(constants.OUTPUT_NAME + '.' + formatChecker.getStringFromFormat(cmd.getOutputFormat()));
  //todo paid
  //if (cmd.vkey) {
  //  bool
  //  bPaid;
  //  Signature.getVKeyParams(cmd.vkey, out bPaid);
  //  oTaskQueueData.m_bPaid = bPaid;
  //}
  return queueData;
}
function getUpdateResponse(cmd) {
  var updateTask = new taskResult.TaskResultData();
  updateTask.key = cmd.getSaveKey() ? cmd.getSaveKey() : cmd.getDocId();
  var statusInfo = cmd.getStatusInfo();
  if (constants.NO_ERROR == statusInfo) {
    updateTask.status = taskResult.FileStatus.Ok;
  } else if (constants.CONVERT_DOWNLOAD == statusInfo) {
    updateTask.status = taskResult.FileStatus.ErrToReload;
  } else if (constants.CONVERT_NEED_PARAMS == statusInfo) {
    updateTask.status = taskResult.FileStatus.NeedParams;
  } else if (constants.CONVERT_DRM == statusInfo) {
    updateTask.status = taskResult.FileStatus.NeedPassword;
  } else {
    updateTask.status = taskResult.FileStatus.Err;
  }
  updateTask.statusInfo = statusInfo;
  if (cmd.getTitle()) {
    updateTask.title = cmd.getTitle();
  }
  return updateTask;
}
var cleanupCache = co.wrap(function* (docId) {
  //todo redis ?
  var res = false;
  var removeRes = yield taskResult.remove(docId);
  if (removeRes.affectedRows > 0) {
    yield storage.deletePath(docId);
    res = true;
  }
  return res;
});

function commandOpenStartPromise(docId, cmd, opt_updateUserIndex) {
  var task = new taskResult.TaskResultData();
  task.key = docId;
  task.status = taskResult.FileStatus.WaitQueue;
  task.statusInfo = constants.NO_ERROR;
  if (cmd) {
    task.title = cmd.getTitle();
  } else {
    logger.warn("commandOpenStartPromise empty cmd: docId = %s", docId);
  }

  return taskResult.upsert(task, opt_updateUserIndex);
}
function* commandOpen(conn, cmd, outputData, opt_upsertRes) {
  var upsertRes;
  if (opt_upsertRes) {
    upsertRes = opt_upsertRes;
  } else {
    upsertRes = yield commandOpenStartPromise(cmd.getDocId(), cmd);
  }
  //if CLIENT_FOUND_ROWS don't specify 1 row is inserted , 2 row is updated, and 0 row is set to its current values
  //http://dev.mysql.com/doc/refman/5.7/en/insert-on-duplicate.html
  var bCreate = upsertRes.affectedRows == 1;
  if (!bCreate) {
    var selectRes = yield taskResult.select(cmd.getDocId());
    if (selectRes.length > 0) {
      var row = selectRes[0];
      yield* getOutputData(cmd, outputData, cmd.getDocId(), row.status, row.status_info, conn);
    }
  } else {
    //add task
    cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CANVAS);
    cmd.setEmbeddedFonts(false);
    var dataQueue = new commonDefines.TaskQueueData();
    dataQueue.setCmd(cmd);
    dataQueue.setToFile('Editor.bin');
    var priority = constants.QUEUE_PRIORITY_HIGH;
    var formatIn = formatChecker.getFormatFromString(cmd.getFormat());
    //decrease pdf, djvu, xps convert priority becase long open time
    if (constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === formatIn || constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_DJVU === formatIn ||
      constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_XPS === formatIn) {
      priority = constants.QUEUE_PRIORITY_LOW;
    }
    yield* docsCoServer.addTask(dataQueue, priority);
  }
}
function* commandReopen(cmd) {
  var task = new taskResult.TaskResultData();
  task.key = cmd.getDocId();
  task.status = taskResult.FileStatus.WaitQueue;
  task.statusInfo = constants.NO_ERROR;

  var upsertRes = yield taskResult.update(task);
  if (upsertRes.affectedRows > 0) {
    //add task
    cmd.setUrl(null);//url may expire
    cmd.setSaveKey(cmd.getDocId());
    cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CANVAS);
    cmd.setEmbeddedFonts(false);
    var dataQueue = new commonDefines.TaskQueueData();
    dataQueue.setCmd(cmd);
    dataQueue.setToFile('Editor.bin');
    dataQueue.setFromSettings(true);
    yield* docsCoServer.addTask(dataQueue, constants.QUEUE_PRIORITY_HIGH);
  }
}
function* commandSave(cmd, outputData) {
  var completeParts = yield* saveParts(cmd);
  if (completeParts) {
    var queueData = getSaveTask(cmd);
    yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
  }
  outputData.setStatus('ok');
  outputData.setData(cmd.getSaveKey());
}
function* commandSendMailMerge(cmd, outputData) {
  var completeParts = yield* saveParts(cmd);
  var isErr = false;
  if (completeParts) {
    isErr = true;
    var getRes = yield* docsCoServer.getCallback(cmd.getDocId());
    if (getRes) {
      var mailMergeSend = cmd.getMailMergeSend();
      mailMergeSend.setUrl(getRes.server.href);
      mailMergeSend.setBaseUrl(getRes.baseUrl);
      //меняем JsonKey и SaveKey, новый key нужет потому что за одну конвертацию делается часть, а json нужен всегда
      mailMergeSend.setJsonKey(cmd.getSaveKey());
      yield* addRandomKeyTaskCmd(cmd);
      var queueData = getSaveTask(cmd);
      yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
      isErr = false;
    }
  }
  if (isErr) {
    outputData.setStatus('err');
    outputData.setData(constants.UNKNOWN);
  } else {
    outputData.setStatus('ok');
    outputData.setData(cmd.getSaveKey());
  }
}
function* commandSfctByCmd(cmd) {
  yield* addRandomKeyTaskCmd(cmd);
  var queueData = getSaveTask(cmd);
  queueData.setFromChanges(true);
  yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
}
function* commandSfct(cmd, outputData) {
  yield* commandSfctByCmd(cmd);
  outputData.setStatus('ok');
}
function isDisplayedImage(strName) {
  var res = 0;
  if (strName) {
    //шаблон display[N]image.ext
    var findStr = constants.DISPLAY_PREFIX;
    var index = strName.indexOf(findStr);
    if (-1 != index) {
      if (index + findStr.length < strName.length) {
        var displayN = parseInt(strName[index + findStr.length]);
        if (1 <= displayN && displayN <= 6) {
          var imageIndex = index + findStr.length + 1;
          if (imageIndex == strName.indexOf("image", imageIndex))
            res = displayN;
        }
      }
    }
  }
  return res;
}
function* commandImgurls(conn, cmd, outputData) {
  var supportedFormats;
  var urls;
  var errorCode = constants.NO_ERROR;
  var isImgUrl = 'imgurl' == cmd.getCommand();
  if (isImgUrl) {
    urls = [cmd.getData()];
    supportedFormats = cfgTypesUpload || 'jpg';
  } else {
    urls = cmd.getData();
    supportedFormats = cfgTypesCopy || 'jpg';
  }
  //todo Promise.all()
  var displayedImageMap = {};//to make one imageIndex for ole object urls
  var imageCount = 0;
  var outputUrls = [];
  for (var i = 0; i < urls.length; ++i) {
    var urlSource = urls[i];
    var urlParsed;
    var data = undefined;
    if (urlSource.startsWith('data:')) {
      var delimiterIndex = urlSource.indexOf(',');
      if (-1 != delimiterIndex && (urlSource.length - (delimiterIndex + 1)) * 0.75 <= cfgImageSize) {
        data = new Buffer(urlSource.substring(delimiterIndex + 1), 'base64');
      }
    } else if(urlSource) {
      //todo stream
      data = yield utils.downloadUrlPromise(urlSource, cfgImageDownloadTimeout * 1000, cfgImageSize);
      urlParsed = urlModule.parse(urlSource);
    }
    var outputUrl = {url: 'error', path: 'error'};
    if (data) {
      var format = formatChecker.getImageFormat(data);
      var formatStr;
      if (constants.AVS_OFFICESTUDIO_FILE_UNKNOWN == format && urlParsed) {
        //bin, txt occur in ole object case
        var ext = pathModule.extname(urlParsed.pathname);
        if ('.bin' == ext || '.txt' == ext) {
          formatStr = ext.substring(1);
        }
      } else {
        formatStr = formatChecker.getStringFromFormat(format);
      }
      if (formatStr && -1 !== supportedFormats.indexOf(formatStr)) {
        var userid = cmd.getUserId();
        var imageIndex = cmd.getSaveIndex() + imageCount;
        imageCount++;
        var strLocalPath = 'media/' + utils.crc32(userid).toString(16) + '_';
        if (urlParsed) {
          var urlBasename = pathModule.basename(urlParsed.pathname);
          var displayN = isDisplayedImage(urlBasename);
          if (displayN > 0) {
            var displayedImageName = urlBasename.substring(0, urlBasename.length - formatStr.length - 1);
            var tempIndex = displayedImageMap[displayedImageName];
            if (null != tempIndex) {
              imageIndex = tempIndex;
              imageCount--;
            } else {
              displayedImageMap[displayedImageName] = imageIndex;
            }
            strLocalPath += constants.DISPLAY_PREFIX + displayN;
          }
        }
        strLocalPath += 'image' + imageIndex + '.' + formatStr;
        var strPath = cmd.getDocId() + '/' + strLocalPath;
        yield storage.putObject(strPath, data, data.length);
        var imgUrl = yield storage.getSignedUrl(conn.baseUrl, strPath);
        outputUrl = {url: imgUrl, path: strLocalPath};
      }
    }
    if (isImgUrl && ('error' === outputUrl.url || 'error' === outputUrl.path)) {
      errorCode = constants.UPLOAD_EXTENSION;
      break;
    }
    outputUrls.push(outputUrl);
  }
  if (constants.NO_ERROR !== errorCode) {
    outputData.setStatus('err');
    outputData.setData(errorCode);
  } else {
    outputData.setStatus('ok');
    outputData.setData(outputUrls);
  }
}
function* commandPathUrl(conn, cmd, outputData) {
  var contentDisposition = cmd.getInline() ? constants.CONTENT_DISPOSITION_INLINE :
    constants.CONTENT_DISPOSITION_ATTACHMENT;
  var strPath = cmd.getDocId() + '/' + cmd.getData();
  var url = yield storage.getSignedUrl(conn.baseUrl, strPath, null, cmd.getTitle(), contentDisposition);
  var errorCode = constants.NO_ERROR;
  if (constants.NO_ERROR !== errorCode) {
    outputData.setStatus('err');
    outputData.setData(errorCode);
  } else {
    outputData.setStatus('ok');
    outputData.setData(url);
  }
}
function* commandSaveFromOrigin(cmd, outputData) {
  yield* addRandomKeyTaskCmd(cmd);
  var queueData = getSaveTask(cmd);
  queueData.setFromOrigin(true);
  yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
  outputData.setStatus('ok');
  outputData.setData(cmd.getSaveKey());
}
function* commandSfcCallback(cmd, isSfcm) {
  var docId = cmd.getDocId();
  logger.debug('Start commandSfcCallback: docId = %s', docId);
  var saveKey = cmd.getSaveKey();
  var statusInfo = cmd.getStatusInfo();
  var isError = constants.NO_ERROR != statusInfo && constants.CONVERT_CORRUPTED != statusInfo;
  var savePathDoc = saveKey + '/' + cmd.getOutputPath();
  var savePathChanges = saveKey + '/changes.zip';
  var savePathHistory = saveKey + '/changesHistory.json';
  var getRes = yield* docsCoServer.getCallback(docId);
  if (getRes) {
    logger.debug('Callback commandSfcCallback: docId = %s callback = %s', docId, getRes.server.href);
    var outputSfc = new commonDefines.OutputSfcData();
    outputSfc.setKey(docId);
    if (cmd.getUserId()) {
      outputSfc.setUsers([cmd.getUserId()]);
    }
    if (isSfcm) {
      outputSfc.setActions(undefined);
      outputSfc.setUserData(cmd.getUserData());
    } else {
      //use UserId case UserActionId miss in gc convertion
      var userActionId = cmd.getUserActionId() || cmd.getUserId();
      if (userActionId) {
        outputSfc.setActions([new commonDefines.OutputAction(commonDefines.c_oAscUserAction.Out, userActionId)]);
      }
    }
    if (!isError) {
      try {
        var data = yield storage.getObject(savePathHistory);
        outputSfc.setChangeHistory(data.toString('utf-8'));
        outputSfc.setUrl(yield storage.getSignedUrl(getRes.baseUrl, savePathDoc));
        outputSfc.setChangeUrl(yield storage.getSignedUrl(getRes.baseUrl, savePathChanges));
      } catch (e) {
        logger.error('Error commandSfcCallback: docId = %s\r\n%s', docId, e.stack);
      }
      if (outputSfc.getUrl() && outputSfc.getUsers().length > 0) {
        if (isSfcm) {
          outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MustSaveForce);
        } else {
          outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MustSave);
        }
      } else {
        isError = true;
      }
    }
    if (isError) {
      if (isSfcm) {
        outputSfc.setStatus(docsCoServer.c_oAscServerStatus.CorruptedForce);
      } else {
        outputSfc.setStatus(docsCoServer.c_oAscServerStatus.Corrupted);
      }
    }
    var uri = getRes.server.href;
    var postData = JSON.stringify(outputSfc);
    if (isSfcm) {
      var lastSave = cmd.getLastSave();
      if (lastSave && !isError) {
        yield* docsCoServer.setForceSave(docId, lastSave, savePathDoc);
      }
      try {
        yield* docsCoServer.sendServerRequest(docId, uri, postData);
      } catch (err) {
        logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack);
      }
    } else {
      //if anybody in document stop save
      var hasEditors = yield* docsCoServer.hasEditors(docId);
      logger.debug('hasEditors commandSfcCallback: docId = %s hasEditors = %d', docId, hasEditors);
      if (!hasEditors) {
        var updateMask = new taskResult.TaskResultData();
        updateMask.key = docId;
        updateMask.status = taskResult.FileStatus.SaveVersion;
        updateMask.statusInfo = cmd.getData();
        var updateIfTask = new taskResult.TaskResultData();
        updateIfTask.status = taskResult.FileStatus.UpdateVersion;
        updateIfTask.statusInfo = constants.NO_ERROR;
        var updateIfRes = yield taskResult.updateIf(updateIfTask, updateMask);
        if (updateIfRes.affectedRows > 0) {
          var replyStr = null;
          try {
            replyStr = yield* docsCoServer.sendServerRequest(docId, uri, postData);
          } catch (err) {
            replyStr = null;
            logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack);
          }
          var requestRes = false;
          var replyData = docsCoServer.parseReplyData(docId, replyStr);
          if (replyData && commonDefines.c_oAscServerCommandErrors.NoError == replyData.error) {
            //в случае comunity server придет запрос в CommandService проверяем результат
            var multi = redisClient.multi([
              ['get', redisKeySaved + docId],
              ['del', redisKeySaved + docId]
            ]);
            var execRes = yield utils.promiseRedis(multi, multi.exec);
            var savedVal = execRes[0];
            requestRes = (null == savedVal || '1' === savedVal);
          }
          if (requestRes) {
            yield docsCoServer.cleanDocumentOnExitPromise(docId, true);
          } else {
            var updateTask = new taskResult.TaskResultData();
            updateTask.key = docId;
            updateTask.status = taskResult.FileStatus.Ok;
            updateTask.statusInfo = constants.NO_ERROR;
            yield taskResult.update(updateTask);
          }
        }
      }
    }
  }
  if (docsCoServer.getIsShutdown() && !isSfcm) {
    yield utils.promiseRedis(redisClient, redisClient.srem, redisKeyShutdown, docId);
  }
  logger.debug('End commandSfcCallback: docId = %s', docId);
}
function* commandSendMMCallback(cmd) {
  var docId = cmd.getDocId();
  logger.debug('Start commandSendMMCallback: docId = %s', docId);
  var saveKey = cmd.getSaveKey();
  var statusInfo = cmd.getStatusInfo();
  var outputSfc = new commonDefines.OutputSfcData();
  outputSfc.setKey(docId);
  if (constants.NO_ERROR == statusInfo) {
    outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MailMerge);
  } else {
    outputSfc.setStatus(docsCoServer.c_oAscServerStatus.Corrupted);
  }
  var mailMergeSendData = cmd.getMailMergeSend();
  var outputMailMerge = new commonDefines.OutputMailMerge(mailMergeSendData);
  outputSfc.setMailMerge(outputMailMerge);
  outputSfc.setUsers([mailMergeSendData.getUserId()]);
  var data = yield storage.getObject(saveKey + '/' + cmd.getOutputPath());
  var xml = data.toString('utf8');
  var files = xml.match(/[< ]file.*?\/>/g);
  var recordRemain = (mailMergeSendData.getRecordTo() - mailMergeSendData.getRecordFrom() + 1);
  var recordIndexStart = mailMergeSendData.getRecordCount() - recordRemain;
  for (var i = 0; i < files.length; ++i) {
    var file = files[i];
    var fieldRes = /field=["'](.*?)["']/.exec(file);
    outputMailMerge.setTo(fieldRes[1]);
    outputMailMerge.setRecordIndex(recordIndexStart + i);
    var pathRes = /path=["'](.*?)["']/.exec(file);
    var signedUrl = yield storage.getSignedUrl(mailMergeSendData.getBaseUrl(), saveKey + '/' + pathRes[1]);
    outputSfc.setUrl(signedUrl);
    var uri = mailMergeSendData.getUrl();
    var postData = JSON.stringify(outputSfc);
    try {
      yield* docsCoServer.sendServerRequest(docId, uri, postData);
    } catch (err) {
      logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack);
    }
  }
  var newRecordFrom = mailMergeSendData.getRecordFrom() + Math.max(files.length, 1);
  if (newRecordFrom <= mailMergeSendData.getRecordTo()) {
    mailMergeSendData.setRecordFrom(newRecordFrom);
    yield* addRandomKeyTaskCmd(cmd);
    var queueData = getSaveTask(cmd);
    yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
  } else {
    logger.debug('End MailMerge: docId = %s', docId);
  }
  logger.debug('End commandSendMMCallback: docId = %s', docId);
}

exports.openDocument = function(conn, cmd, opt_upsertRes) {
  return co(function* () {
    var outputData;
    var docId = conn ? conn.docId : 'null';
    try {
      var startDate = null;
      if(clientStatsD) {
        startDate = new Date();
      }
      logger.debug('Start command: docId = %s %s', docId, JSON.stringify(cmd));
      outputData = new OutputData(cmd.getCommand());
      switch (cmd.getCommand()) {
        case 'open':
          //yield utils.sleep(5000);
          yield* commandOpen(conn, cmd, outputData, opt_upsertRes);
          break;
        case 'reopen':
          yield* commandReopen(cmd);
          break;
        case 'imgurl':
        case 'imgurls':
          yield* commandImgurls(conn, cmd, outputData);
          break;
        case 'pathurl':
          yield* commandPathUrl(conn, cmd, outputData);
          break;
        default:
          outputData.setStatus('err');
          outputData.setData(constants.UNKNOWN);
          break;
      }
      if(clientStatsD) {
        clientStatsD.timing('coauth.openDocument.' + cmd.getCommand(), new Date() - startDate);
      }
    }
    catch (e) {
      logger.error('Error openDocument: docId = %s\r\n%s', docId, e.stack);
      if (!outputData) {
        outputData = new OutputData();
      }
      outputData.setStatus('err');
      outputData.setData(constants.UNKNOWN);
    }
    finally {
      if (outputData && outputData.getStatus()) {
        logger.debug('Response command: docId = %s %s', docId, JSON.stringify(outputData));
        docsCoServer.sendData(conn, new OutputDataWrap('documentOpen', outputData));
      }
      logger.debug('End command: docId = %s', docId);
    }
  });
};
exports.downloadAs = function(req, res) {
  return co(function* () {
    var docId = 'null';
    try {
      var startDate = null;
      if(clientStatsD) {
        startDate = new Date();
      }
      var strCmd = req.query['cmd'];
      var cmd = new commonDefines.InputCommand(JSON.parse(strCmd));
      docId = cmd.getDocId();
      logger.debug('Start downloadAs: docId = %s %s', docId, strCmd);
      cmd.setData(req.body);
      var outputData = new OutputData(cmd.getCommand());
      switch (cmd.getCommand()) {
        case 'save':
          yield* commandSave(cmd, outputData);
          break;
        case 'savefromorigin':
          yield* commandSaveFromOrigin(cmd, outputData);
          break;
        case 'sendmm':
          yield* commandSendMailMerge(cmd, outputData);
          break;
        case 'sfct':
          yield* commandSfct(cmd, outputData);
          break;
        default:
          outputData.setStatus('err');
          outputData.setData(constants.UNKNOWN);
          break;
      }
      var strRes = JSON.stringify(outputData);
      res.send(strRes);
      logger.debug('End downloadAs: docId = %s %s', docId, strRes);
      if(clientStatsD) {
        clientStatsD.timing('coauth.downloadAs.' + cmd.getCommand(), new Date() - startDate);
      }
    }
    catch (e) {
      logger.error('Error downloadAs: docId = %s\r\n%s', docId, e.stack);
      res.sendStatus(400);
    }
  });
};
exports.saveFromChanges = function(docId, statusInfo, optFormat, opt_userId, opt_queue) {
  return co(function* () {
    try {
      var startDate = null;
      if(clientStatsD) {
        startDate = new Date();
      }
      logger.debug('Start saveFromChanges: docId = %s', docId);
      var task = new taskResult.TaskResultData();
      task.key = docId;
      //делаем select, потому что за время timeout информация могла измениться
      var selectRes = yield taskResult.select(docId);
      var row = selectRes.length > 0 ? selectRes[0] : null;
      if (row && row.status == taskResult.FileStatus.SaveVersion && row.status_info == statusInfo) {
        if (null == optFormat) {
          optFormat = constants.AVS_OFFICESTUDIO_FILE_OTHER_TEAMLAB_INNER;
        }
        var cmd = new commonDefines.InputCommand();
        cmd.setCommand('sfc');
        cmd.setDocId(docId);
        cmd.setOutputFormat(optFormat);
        cmd.setData(statusInfo);
        cmd.setUserActionId(opt_userId);
        yield* addRandomKeyTaskCmd(cmd);
        var queueData = getSaveTask(cmd);
        queueData.setFromChanges(true);
        yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_NORMAL, opt_queue);
        if (docsCoServer.getIsShutdown()) {
          yield utils.promiseRedis(redisClient, redisClient.sadd, redisKeyShutdown, docId);
        }
        logger.debug('AddTask saveFromChanges: docId = %s', docId);
      } else {
        if (row) {
          logger.debug('saveFromChanges status mismatch: docId = %s; row: %d; %d; expected: %d', docId, row.status, row.status_info, statusInfo);
        }
      }
      if (clientStatsD) {
        clientStatsD.timing('coauth.saveFromChanges', new Date() - startDate);
      }
    }
    catch (e) {
      logger.error('Error saveFromChanges: docId = %s\r\n%s', docId, e.stack);
    }
  });
};
exports.receiveTask = function(data, dataRaw) {
  return co(function* () {
    var docId = 'null';
    try {
      var task = new commonDefines.TaskQueueData(JSON.parse(data));
      if (task) {
        var cmd = task.getCmd();
        docId = cmd.getDocId();
        logger.debug('Start receiveTask: docId = %s %s', docId, data);
        var updateTask = getUpdateResponse(cmd);
        var updateRes = yield taskResult.update(updateTask);
        if (updateRes.affectedRows > 0) {
          var outputData = new OutputData(cmd.getCommand());
          var command = cmd.getCommand();
          var additionalOutput = {needUrlKey: null, needUrlMethod: null};
          if ('open' == command || 'reopen' == command) {
            //yield utils.sleep(5000);
            yield* getOutputData(cmd, outputData, cmd.getDocId(), updateTask.status,
              updateTask.statusInfo, null, additionalOutput);
          } else if ('save' == command || 'savefromorigin' == command || 'sfct' == command) {
            yield* getOutputData(cmd, outputData, cmd.getSaveKey(), updateTask.status,
              updateTask.statusInfo, null, additionalOutput);
          } else if ('sfcm' == command) {
            yield* commandSfcCallback(cmd, true);
          } else if ('sfc' == command) {
            yield* commandSfcCallback(cmd, false);
          } else if ('sendmm' == command) {
            yield* commandSendMMCallback(cmd);
          } else if ('conv' == command) {
            //nothing
          }
          if (outputData.getStatus()) {
            logger.debug('Send receiveTask: docId = %s %s', docId, JSON.stringify(outputData));
            var output = new OutputDataWrap('documentOpen', outputData);
            yield* docsCoServer.publish({
              type: commonDefines.c_oPublishType.receiveTask, cmd: cmd, output: output,
              needUrlKey: additionalOutput.needUrlKey, needUrlMethod: additionalOutput.needUrlMethod
            });
          }
        }
        yield* docsCoServer.removeResponse(dataRaw);
        logger.debug('End receiveTask: docId = %s', docId);
      }
    } catch (err) {
      logger.debug('Error receiveTask: docId = %s\r\n%s', docId, err.stack);
    }
  });
};

exports.cleanupCache = cleanupCache;
exports.commandSfctByCmd = commandSfctByCmd;
exports.commandOpenStartPromise = commandOpenStartPromise;
exports.OutputDataWrap = OutputDataWrap;
exports.OutputData = OutputData;
\ No newline at end of file
+/*
 * (c) Copyright Ascensio System SIA 2010-2016
 *
 * This program is a free software product. You can redistribute it and/or
 * modify it under the terms of the GNU Affero General Public License (AGPL)
 * version 3 as published by the Free Software Foundation. In accordance with
 * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect
 * that Ascensio System SIA expressly excludes the warranty of non-infringement
 * of any third-party rights.
 *
 * This program is distributed WITHOUT ANY WARRANTY; without even the implied
 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE. For
 * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
 *
 * You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia,
 * EU, LV-1021.
 *
 * The  interactive user interfaces in modified source and object code versions
 * of the Program must display Appropriate Legal Notices, as required under
 * Section 5 of the GNU AGPL version 3.
 *
 * Pursuant to Section 7(b) of the License you must retain the original Product
 * logo when distributing the program. Pursuant to Section 7(e) we decline to
 * grant you any rights under trademark law for use of our trademarks.
 *
 * All the Product's GUI elements, including illustrations and icon sets, as
 * well as technical writing content are licensed under the terms of the
 * Creative Commons Attribution-ShareAlike 4.0 International. See the License
 * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
 *
 */

var pathModule = require('path');
var urlModule = require('url');
var co = require('co');
var sqlBase = require('./baseConnector');
var docsCoServer = require('./DocsCoServer');
var taskResult = require('./taskresult');
var logger = require('./../../Common/sources/logger');
var utils = require('./../../Common/sources/utils');
var constants = require('./../../Common/sources/constants');
var commonDefines = require('./../../Common/sources/commondefines');
var storage = require('./../../Common/sources/storage-base');
var formatChecker = require('./../../Common/sources/formatchecker');
var statsDClient = require('./../../Common/sources/statsdclient');
var config = require('config');
var config_server = config.get('services.CoAuthoring.server');
var config_utils = config.get('services.CoAuthoring.utils');
var pubsubRedis = require('./pubsubRedis');


var cfgTypesUpload = config_utils.get('limits_image_types_upload');
var cfgTypesCopy = config_utils.get('limits_image_types_copy');
var cfgImageSize = config_server.get('limits_image_size');
var cfgImageDownloadTimeout = config_server.get('limits_image_download_timeout');
var cfgRedisPrefix = config.get('services.CoAuthoring.redis.prefix');

var SAVE_TYPE_PART_START = 0;
var SAVE_TYPE_PART = 1;
var SAVE_TYPE_COMPLETE = 2;
var SAVE_TYPE_COMPLETE_ALL = 3;

var clientStatsD = statsDClient.getClient();
var redisClient = pubsubRedis.getClientRedis();
var redisKeySaved = cfgRedisPrefix + constants.REDIS_KEY_SAVED;
var redisKeyShutdown = cfgRedisPrefix + constants.REDIS_KEY_SHUTDOWN;

function OutputDataWrap(type, data) {
  this['type'] = type;
  this['data'] = data;
}
OutputDataWrap.prototype = {
  fromObject: function(data) {
    this['type'] = data['type'];
    this['data'] = new OutputData();
    this['data'].fromObject(data['data']);
  },
  getType: function() {
    return this['type'];
  },
  setType: function(data) {
    this['type'] = data;
  },
  getData: function() {
    return this['data'];
  },
  setData: function(data) {
    this['data'] = data;
  }
};
function OutputData(type) {
  this['type'] = type;
  this['status'] = undefined;
  this['data'] = undefined;
}
OutputData.prototype = {
  fromObject: function(data) {
    this['type'] = data['type'];
    this['status'] = data['status'];
    this['data'] = data['data'];
  },
  getType: function() {
    return this['type'];
  },
  setType: function(data) {
    this['type'] = data;
  },
  getStatus: function() {
    return this['status'];
  },
  setStatus: function(data) {
    this['status'] = data;
  },
  getData: function() {
    return this['data'];
  },
  setData: function(data) {
    this['data'] = data;
  }
};

function* getOutputData(cmd, outputData, key, status, statusInfo, optConn, optAdditionalOutput) {
  var docId = cmd.getDocId();
  switch (status) {
    case taskResult.FileStatus.SaveVersion:
    case taskResult.FileStatus.UpdateVersion:
    case taskResult.FileStatus.Ok:
      if(taskResult.FileStatus.Ok == status) {
        outputData.setStatus('ok');
      } else if(taskResult.FileStatus.SaveVersion == status) {
        if (optConn && optConn.user.view) {
          outputData.setStatus('updateversion');
        } else {
          var updateMask = new taskResult.TaskResultData();
          updateMask.key = docId;
          updateMask.status = status;
          updateMask.statusInfo = statusInfo;
          var updateTask = new taskResult.TaskResultData();
          updateTask.status = taskResult.FileStatus.Ok;
          updateTask.statusInfo = constants.NO_ERROR;
          var updateIfRes = yield taskResult.updateIf(updateTask, updateMask);
          if (updateIfRes.affectedRows > 0) {
            outputData.setStatus('ok');
          } else {
            outputData.setStatus('updateversion');
          }
        }
      } else {
        outputData.setStatus('updateversion');
      }
      var command = cmd.getCommand();
      if ('open' != command && 'reopen' != command) {
        var strPath = key + '/' + cmd.getOutputPath();
        if (optConn) {
          var contentDisposition = cmd.getInline() ? constants.CONTENT_DISPOSITION_INLINE : constants.CONTENT_DISPOSITION_ATTACHMENT;
          outputData.setData(yield storage.getSignedUrl(optConn.baseUrl, strPath, null, cmd.getTitle(), contentDisposition));
        } else if (optAdditionalOutput) {
          optAdditionalOutput.needUrlKey = strPath;
          optAdditionalOutput.needUrlMethod = 2;
        }
      } else {
        if (optConn) {
          outputData.setData(yield storage.getSignedUrls(optConn.baseUrl, key));
        } else if (optAdditionalOutput) {
          optAdditionalOutput.needUrlKey = key;
          optAdditionalOutput.needUrlMethod = 0;
        }
      }
      break;
    case taskResult.FileStatus.NeedParams:
      outputData.setStatus('needparams');
      var settingsPath = key + '/' + 'settings.json';
      if (optConn) {
        outputData.setData(yield storage.getSignedUrl(optConn.baseUrl, settingsPath));
      } else if (optAdditionalOutput) {
        optAdditionalOutput.needUrlKey = settingsPath;
        optAdditionalOutput.needUrlMethod = 1;
      }
      break;
    case taskResult.FileStatus.NeedPassword:
      outputData.setStatus('needpassword');
      break;
    case taskResult.FileStatus.Err:
    case taskResult.FileStatus.ErrToReload:
      outputData.setStatus('err');
      outputData.setData(statusInfo);
      if (taskResult.FileStatus.ErrToReload == status) {
        yield cleanupCache(key);
      }
      break;
  }
}
function* addRandomKeyTaskCmd(cmd) {
  var task = yield* taskResult.addRandomKeyTask(cmd.getDocId());
  cmd.setSaveKey(task.key);
}
function* saveParts(cmd) {
  var result = false;
  var saveType = cmd.getSaveType();
  var filename;
  if (SAVE_TYPE_COMPLETE_ALL === saveType) {
    filename = 'Editor.bin';
  } else {
    filename = 'Editor' + (cmd.getSaveIndex() || '') + '.bin';
  }
  if (SAVE_TYPE_PART_START === saveType || SAVE_TYPE_COMPLETE_ALL === saveType) {
    yield* addRandomKeyTaskCmd(cmd);
  }
  if (cmd.getUrl()) {
    result = true;
  } else {
    var buffer = cmd.getData();
    yield storage.putObject(cmd.getSaveKey() + '/' + filename, buffer, buffer.length);
    //delete data to prevent serialize into json
    cmd.data = null;
    result = (SAVE_TYPE_COMPLETE_ALL === saveType || SAVE_TYPE_COMPLETE === saveType);
  }
  return result;
}
function getSaveTask(cmd) {
  cmd.setData(null);
  var queueData = new commonDefines.TaskQueueData();
  queueData.setCmd(cmd);
  queueData.setToFile(constants.OUTPUT_NAME + '.' + formatChecker.getStringFromFormat(cmd.getOutputFormat()));
  //todo paid
  //if (cmd.vkey) {
  //  bool
  //  bPaid;
  //  Signature.getVKeyParams(cmd.vkey, out bPaid);
  //  oTaskQueueData.m_bPaid = bPaid;
  //}
  return queueData;
}
function getUpdateResponse(cmd) {
  var updateTask = new taskResult.TaskResultData();
  updateTask.key = cmd.getSaveKey() ? cmd.getSaveKey() : cmd.getDocId();
  var statusInfo = cmd.getStatusInfo();
  if (constants.NO_ERROR == statusInfo) {
    updateTask.status = taskResult.FileStatus.Ok;
  } else if (constants.CONVERT_DOWNLOAD == statusInfo) {
    updateTask.status = taskResult.FileStatus.ErrToReload;
  } else if (constants.CONVERT_NEED_PARAMS == statusInfo) {
    updateTask.status = taskResult.FileStatus.NeedParams;
  } else if (constants.CONVERT_DRM == statusInfo) {
    updateTask.status = taskResult.FileStatus.NeedPassword;
  } else {
    updateTask.status = taskResult.FileStatus.Err;
  }
  updateTask.statusInfo = statusInfo;
  if (cmd.getTitle()) {
    updateTask.title = cmd.getTitle();
  }
  return updateTask;
}
var cleanupCache = co.wrap(function* (docId) {
  //todo redis ?
  var res = false;
  var removeRes = yield taskResult.remove(docId);
  if (removeRes.affectedRows > 0) {
    yield storage.deletePath(docId);
    res = true;
  }
  return res;
});

function commandOpenStartPromise(docId, cmd, opt_updateUserIndex) {
  var task = new taskResult.TaskResultData();
  task.key = docId;
  task.status = taskResult.FileStatus.WaitQueue;
  task.statusInfo = constants.NO_ERROR;
  if (cmd) {
    task.title = cmd.getTitle();
  } else {
    logger.warn("commandOpenStartPromise empty cmd: docId = %s", docId);
  }

  return taskResult.upsert(task, opt_updateUserIndex);
}
function* commandOpen(conn, cmd, outputData, opt_upsertRes) {
  var upsertRes;
  if (opt_upsertRes) {
    upsertRes = opt_upsertRes;
  } else {
    upsertRes = yield commandOpenStartPromise(cmd.getDocId(), cmd);
  }
  //if CLIENT_FOUND_ROWS don't specify 1 row is inserted , 2 row is updated, and 0 row is set to its current values
  //http://dev.mysql.com/doc/refman/5.7/en/insert-on-duplicate.html
  var bCreate = upsertRes.affectedRows == 1;
  if (!bCreate) {
    var selectRes = yield taskResult.select(cmd.getDocId());
    if (selectRes.length > 0) {
      var row = selectRes[0];
      yield* getOutputData(cmd, outputData, cmd.getDocId(), row.status, row.status_info, conn);
    }
  } else {
    //add task
    cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CANVAS);
    cmd.setEmbeddedFonts(false);
    var dataQueue = new commonDefines.TaskQueueData();
    dataQueue.setCmd(cmd);
    dataQueue.setToFile('Editor.bin');
    var priority = constants.QUEUE_PRIORITY_HIGH;
    var formatIn = formatChecker.getFormatFromString(cmd.getFormat());
    //decrease pdf, djvu, xps convert priority becase long open time
    if (constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === formatIn || constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_DJVU === formatIn ||
      constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_XPS === formatIn) {
      priority = constants.QUEUE_PRIORITY_LOW;
    }
    yield* docsCoServer.addTask(dataQueue, priority);
  }
}
function* commandReopen(cmd) {
  var task = new taskResult.TaskResultData();
  task.key = cmd.getDocId();
  task.status = taskResult.FileStatus.WaitQueue;
  task.statusInfo = constants.NO_ERROR;

  var upsertRes = yield taskResult.update(task);
  if (upsertRes.affectedRows > 0) {
    //add task
    cmd.setUrl(null);//url may expire
    cmd.setSaveKey(cmd.getDocId());
    cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CANVAS);
    cmd.setEmbeddedFonts(false);
    var dataQueue = new commonDefines.TaskQueueData();
    dataQueue.setCmd(cmd);
    dataQueue.setToFile('Editor.bin');
    dataQueue.setFromSettings(true);
    yield* docsCoServer.addTask(dataQueue, constants.QUEUE_PRIORITY_HIGH);
  }
}
function* commandSave(cmd, outputData) {
  var completeParts = yield* saveParts(cmd);
  if (completeParts) {
    var queueData = getSaveTask(cmd);
    yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
  }
  outputData.setStatus('ok');
  outputData.setData(cmd.getSaveKey());
}
function* commandSendMailMerge(cmd, outputData) {
  var completeParts = yield* saveParts(cmd);
  var isErr = false;
  if (completeParts) {
    isErr = true;
    var getRes = yield* docsCoServer.getCallback(cmd.getDocId());
    if (getRes) {
      var mailMergeSend = cmd.getMailMergeSend();
      mailMergeSend.setUrl(getRes.server.href);
      mailMergeSend.setBaseUrl(getRes.baseUrl);
      //меняем JsonKey и SaveKey, новый key нужет потому что за одну конвертацию делается часть, а json нужен всегда
      mailMergeSend.setJsonKey(cmd.getSaveKey());
      yield* addRandomKeyTaskCmd(cmd);
      var queueData = getSaveTask(cmd);
      yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
      isErr = false;
    }
  }
  if (isErr) {
    outputData.setStatus('err');
    outputData.setData(constants.UNKNOWN);
  } else {
    outputData.setStatus('ok');
    outputData.setData(cmd.getSaveKey());
  }
}
function* commandSfctByCmd(cmd) {
  yield* addRandomKeyTaskCmd(cmd);
  var queueData = getSaveTask(cmd);
  queueData.setFromChanges(true);
  yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
}
function* commandSfct(cmd, outputData) {
  yield* commandSfctByCmd(cmd);
  outputData.setStatus('ok');
}
function isDisplayedImage(strName) {
  var res = 0;
  if (strName) {
    //шаблон display[N]image.ext
    var findStr = constants.DISPLAY_PREFIX;
    var index = strName.indexOf(findStr);
    if (-1 != index) {
      if (index + findStr.length < strName.length) {
        var displayN = parseInt(strName[index + findStr.length]);
        if (1 <= displayN && displayN <= 6) {
          var imageIndex = index + findStr.length + 1;
          if (imageIndex == strName.indexOf("image", imageIndex))
            res = displayN;
        }
      }
    }
  }
  return res;
}
function* commandImgurls(conn, cmd, outputData) {
  var supportedFormats;
  var urls;
  var errorCode = constants.NO_ERROR;
  var isImgUrl = 'imgurl' == cmd.getCommand();
  if (isImgUrl) {
    urls = [cmd.getData()];
    supportedFormats = cfgTypesUpload || 'jpg';
  } else {
    urls = cmd.getData();
    supportedFormats = cfgTypesCopy || 'jpg';
  }
  //todo Promise.all()
  var displayedImageMap = {};//to make one imageIndex for ole object urls
  var imageCount = 0;
  var outputUrls = [];
  for (var i = 0; i < urls.length; ++i) {
    var urlSource = urls[i];
    var urlParsed;
    var data = undefined;
    if (urlSource.startsWith('data:')) {
      var delimiterIndex = urlSource.indexOf(',');
      if (-1 != delimiterIndex && (urlSource.length - (delimiterIndex + 1)) * 0.75 <= cfgImageSize) {
        data = new Buffer(urlSource.substring(delimiterIndex + 1), 'base64');
      }
    } else if(urlSource) {
      //todo stream
      data = yield utils.downloadUrlPromise(urlSource, cfgImageDownloadTimeout * 1000, cfgImageSize);
      urlParsed = urlModule.parse(urlSource);
    }
    var outputUrl = {url: 'error', path: 'error'};
    if (data) {
      var format = formatChecker.getImageFormat(data);
      var formatStr;
      if (constants.AVS_OFFICESTUDIO_FILE_UNKNOWN == format && urlParsed) {
        //bin, txt occur in ole object case
        var ext = pathModule.extname(urlParsed.pathname);
        if ('.bin' == ext || '.txt' == ext) {
          formatStr = ext.substring(1);
        }
      } else {
        formatStr = formatChecker.getStringFromFormat(format);
      }
      if (formatStr && -1 !== supportedFormats.indexOf(formatStr)) {
        var userid = cmd.getUserId();
        var imageIndex = cmd.getSaveIndex() + imageCount;
        imageCount++;
        var strLocalPath = 'media/' + utils.crc32(userid).toString(16) + '_';
        if (urlParsed) {
          var urlBasename = pathModule.basename(urlParsed.pathname);
          var displayN = isDisplayedImage(urlBasename);
          if (displayN > 0) {
            var displayedImageName = urlBasename.substring(0, urlBasename.length - formatStr.length - 1);
            var tempIndex = displayedImageMap[displayedImageName];
            if (null != tempIndex) {
              imageIndex = tempIndex;
              imageCount--;
            } else {
              displayedImageMap[displayedImageName] = imageIndex;
            }
            strLocalPath += constants.DISPLAY_PREFIX + displayN;
          }
        }
        strLocalPath += 'image' + imageIndex + '.' + formatStr;
        var strPath = cmd.getDocId() + '/' + strLocalPath;
        yield storage.putObject(strPath, data, data.length);
        var imgUrl = yield storage.getSignedUrl(conn.baseUrl, strPath);
        outputUrl = {url: imgUrl, path: strLocalPath};
      }
    }
    if (isImgUrl && ('error' === outputUrl.url || 'error' === outputUrl.path)) {
      errorCode = constants.UPLOAD_EXTENSION;
      break;
    }
    outputUrls.push(outputUrl);
  }
  if (constants.NO_ERROR !== errorCode) {
    outputData.setStatus('err');
    outputData.setData(errorCode);
  } else {
    outputData.setStatus('ok');
    outputData.setData(outputUrls);
  }
}
function* commandPathUrl(conn, cmd, outputData) {
  var contentDisposition = cmd.getInline() ? constants.CONTENT_DISPOSITION_INLINE :
    constants.CONTENT_DISPOSITION_ATTACHMENT;
  var strPath = cmd.getDocId() + '/' + cmd.getData();
  var url = yield storage.getSignedUrl(conn.baseUrl, strPath, null, cmd.getTitle(), contentDisposition);
  var errorCode = constants.NO_ERROR;
  if (constants.NO_ERROR !== errorCode) {
    outputData.setStatus('err');
    outputData.setData(errorCode);
  } else {
    outputData.setStatus('ok');
    outputData.setData(url);
  }
}
function* commandSaveFromOrigin(cmd, outputData) {
  yield* addRandomKeyTaskCmd(cmd);
  var queueData = getSaveTask(cmd);
  queueData.setFromOrigin(true);
  yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
  outputData.setStatus('ok');
  outputData.setData(cmd.getSaveKey());
}
function* commandSfcCallback(cmd, isSfcm) {
  var docId = cmd.getDocId();
  logger.debug('Start commandSfcCallback: docId = %s', docId);
  var saveKey = cmd.getSaveKey();
  var statusInfo = cmd.getStatusInfo();
  var isError = constants.NO_ERROR != statusInfo && constants.CONVERT_CORRUPTED != statusInfo;
  var savePathDoc = saveKey + '/' + cmd.getOutputPath();
  var savePathChanges = saveKey + '/changes.zip';
  var savePathHistory = saveKey + '/changesHistory.json';
  var getRes = yield* docsCoServer.getCallback(docId);
  if (getRes) {
    logger.debug('Callback commandSfcCallback: docId = %s callback = %s', docId, getRes.server.href);
    var outputSfc = new commonDefines.OutputSfcData();
    outputSfc.setKey(docId);
    var users = [];
    if (cmd.getUserId()) {
      users.push(cmd.getUserId());
    }
    outputSfc.setUsers(users);
    if (!isSfcm) {
      var actions = [];
      //use UserId case UserActionId miss in gc convertion
      var userActionId = cmd.getUserActionId() || cmd.getUserId();
      if (userActionId) {
        actions.push(new commonDefines.OutputAction(commonDefines.c_oAscUserAction.Out, userActionId));
      }
      outputSfc.setActions(actions);
    }
    outputSfc.setUserData(cmd.getUserData());
    if (!isError) {
      try {
        var data = yield storage.getObject(savePathHistory);
        outputSfc.setChangeHistory(data.toString('utf-8'));
        outputSfc.setUrl(yield storage.getSignedUrl(getRes.baseUrl, savePathDoc));
        outputSfc.setChangeUrl(yield storage.getSignedUrl(getRes.baseUrl, savePathChanges));
      } catch (e) {
        logger.error('Error commandSfcCallback: docId = %s\r\n%s', docId, e.stack);
      }
      if (outputSfc.getUrl() && outputSfc.getUsers().length > 0) {
        if (isSfcm) {
          outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MustSaveForce);
        } else {
          outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MustSave);
        }
      } else {
        isError = true;
      }
    }
    if (isError) {
      if (isSfcm) {
        outputSfc.setStatus(docsCoServer.c_oAscServerStatus.CorruptedForce);
      } else {
        outputSfc.setStatus(docsCoServer.c_oAscServerStatus.Corrupted);
      }
    }
    var uri = getRes.server.href;
    var postData = JSON.stringify(outputSfc);
    if (isSfcm) {
      var lastSave = cmd.getLastSave();
      if (lastSave && !isError) {
        yield* docsCoServer.setForceSave(docId, lastSave, savePathDoc);
      }
      try {
        yield* docsCoServer.sendServerRequest(docId, uri, postData);
      } catch (err) {
        logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack);
      }
    } else {
      //if anybody in document stop save
      var hasEditors = yield* docsCoServer.hasEditors(docId);
      logger.debug('hasEditors commandSfcCallback: docId = %s hasEditors = %d', docId, hasEditors);
      if (!hasEditors) {
        var updateMask = new taskResult.TaskResultData();
        updateMask.key = docId;
        updateMask.status = taskResult.FileStatus.SaveVersion;
        updateMask.statusInfo = cmd.getData();
        var updateIfTask = new taskResult.TaskResultData();
        updateIfTask.status = taskResult.FileStatus.UpdateVersion;
        updateIfTask.statusInfo = constants.NO_ERROR;
        var updateIfRes = yield taskResult.updateIf(updateIfTask, updateMask);
        if (updateIfRes.affectedRows > 0) {
          var replyStr = null;
          try {
            replyStr = yield* docsCoServer.sendServerRequest(docId, uri, postData);
          } catch (err) {
            replyStr = null;
            logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack);
          }
          var requestRes = false;
          var replyData = docsCoServer.parseReplyData(docId, replyStr);
          if (replyData && commonDefines.c_oAscServerCommandErrors.NoError == replyData.error) {
            //в случае comunity server придет запрос в CommandService проверяем результат
            var multi = redisClient.multi([
              ['get', redisKeySaved + docId],
              ['del', redisKeySaved + docId]
            ]);
            var execRes = yield utils.promiseRedis(multi, multi.exec);
            var savedVal = execRes[0];
            requestRes = (null == savedVal || '1' === savedVal);
          }
          if (requestRes) {
            yield docsCoServer.cleanDocumentOnExitPromise(docId, true);
          } else {
            var updateTask = new taskResult.TaskResultData();
            updateTask.key = docId;
            updateTask.status = taskResult.FileStatus.Ok;
            updateTask.statusInfo = constants.NO_ERROR;
            yield taskResult.update(updateTask);
          }
        }
      }
    }
  }
  if (docsCoServer.getIsShutdown() && !isSfcm) {
    yield utils.promiseRedis(redisClient, redisClient.srem, redisKeyShutdown, docId);
  }
  logger.debug('End commandSfcCallback: docId = %s', docId);
}
function* commandSendMMCallback(cmd) {
  var docId = cmd.getDocId();
  logger.debug('Start commandSendMMCallback: docId = %s', docId);
  var saveKey = cmd.getSaveKey();
  var statusInfo = cmd.getStatusInfo();
  var outputSfc = new commonDefines.OutputSfcData();
  outputSfc.setKey(docId);
  if (constants.NO_ERROR == statusInfo) {
    outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MailMerge);
  } else {
    outputSfc.setStatus(docsCoServer.c_oAscServerStatus.Corrupted);
  }
  var mailMergeSendData = cmd.getMailMergeSend();
  var outputMailMerge = new commonDefines.OutputMailMerge(mailMergeSendData);
  outputSfc.setMailMerge(outputMailMerge);
  outputSfc.setUsers([mailMergeSendData.getUserId()]);
  var data = yield storage.getObject(saveKey + '/' + cmd.getOutputPath());
  var xml = data.toString('utf8');
  var files = xml.match(/[< ]file.*?\/>/g);
  var recordRemain = (mailMergeSendData.getRecordTo() - mailMergeSendData.getRecordFrom() + 1);
  var recordIndexStart = mailMergeSendData.getRecordCount() - recordRemain;
  for (var i = 0; i < files.length; ++i) {
    var file = files[i];
    var fieldRes = /field=["'](.*?)["']/.exec(file);
    outputMailMerge.setTo(fieldRes[1]);
    outputMailMerge.setRecordIndex(recordIndexStart + i);
    var pathRes = /path=["'](.*?)["']/.exec(file);
    var signedUrl = yield storage.getSignedUrl(mailMergeSendData.getBaseUrl(), saveKey + '/' + pathRes[1]);
    outputSfc.setUrl(signedUrl);
    var uri = mailMergeSendData.getUrl();
    var postData = JSON.stringify(outputSfc);
    try {
      yield* docsCoServer.sendServerRequest(docId, uri, postData);
    } catch (err) {
      logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack);
    }
  }
  var newRecordFrom = mailMergeSendData.getRecordFrom() + Math.max(files.length, 1);
  if (newRecordFrom <= mailMergeSendData.getRecordTo()) {
    mailMergeSendData.setRecordFrom(newRecordFrom);
    yield* addRandomKeyTaskCmd(cmd);
    var queueData = getSaveTask(cmd);
    yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
  } else {
    logger.debug('End MailMerge: docId = %s', docId);
  }
  logger.debug('End commandSendMMCallback: docId = %s', docId);
}

exports.openDocument = function(conn, cmd, opt_upsertRes) {
  return co(function* () {
    var outputData;
    var docId = conn ? conn.docId : 'null';
    try {
      var startDate = null;
      if(clientStatsD) {
        startDate = new Date();
      }
      logger.debug('Start command: docId = %s %s', docId, JSON.stringify(cmd));
      outputData = new OutputData(cmd.getCommand());
      switch (cmd.getCommand()) {
        case 'open':
          //yield utils.sleep(5000);
          yield* commandOpen(conn, cmd, outputData, opt_upsertRes);
          break;
        case 'reopen':
          yield* commandReopen(cmd);
          break;
        case 'imgurl':
        case 'imgurls':
          yield* commandImgurls(conn, cmd, outputData);
          break;
        case 'pathurl':
          yield* commandPathUrl(conn, cmd, outputData);
          break;
        default:
          outputData.setStatus('err');
          outputData.setData(constants.UNKNOWN);
          break;
      }
      if(clientStatsD) {
        clientStatsD.timing('coauth.openDocument.' + cmd.getCommand(), new Date() - startDate);
      }
    }
    catch (e) {
      logger.error('Error openDocument: docId = %s\r\n%s', docId, e.stack);
      if (!outputData) {
        outputData = new OutputData();
      }
      outputData.setStatus('err');
      outputData.setData(constants.UNKNOWN);
    }
    finally {
      if (outputData && outputData.getStatus()) {
        logger.debug('Response command: docId = %s %s', docId, JSON.stringify(outputData));
        docsCoServer.sendData(conn, new OutputDataWrap('documentOpen', outputData));
      }
      logger.debug('End command: docId = %s', docId);
    }
  });
};
exports.downloadAs = function(req, res) {
  return co(function* () {
    var docId = 'null';
    try {
      var startDate = null;
      if(clientStatsD) {
        startDate = new Date();
      }
      var strCmd = req.query['cmd'];
      var cmd = new commonDefines.InputCommand(JSON.parse(strCmd));
      docId = cmd.getDocId();
      logger.debug('Start downloadAs: docId = %s %s', docId, strCmd);
      cmd.setData(req.body);
      var outputData = new OutputData(cmd.getCommand());
      switch (cmd.getCommand()) {
        case 'save':
          yield* commandSave(cmd, outputData);
          break;
        case 'savefromorigin':
          yield* commandSaveFromOrigin(cmd, outputData);
          break;
        case 'sendmm':
          yield* commandSendMailMerge(cmd, outputData);
          break;
        case 'sfct':
          yield* commandSfct(cmd, outputData);
          break;
        default:
          outputData.setStatus('err');
          outputData.setData(constants.UNKNOWN);
          break;
      }
      var strRes = JSON.stringify(outputData);
      res.send(strRes);
      logger.debug('End downloadAs: docId = %s %s', docId, strRes);
      if(clientStatsD) {
        clientStatsD.timing('coauth.downloadAs.' + cmd.getCommand(), new Date() - startDate);
      }
    }
    catch (e) {
      logger.error('Error downloadAs: docId = %s\r\n%s', docId, e.stack);
      res.sendStatus(400);
    }
  });
};
exports.saveFromChanges = function(docId, statusInfo, optFormat, opt_userId, opt_queue) {
  return co(function* () {
    try {
      var startDate = null;
      if(clientStatsD) {
        startDate = new Date();
      }
      logger.debug('Start saveFromChanges: docId = %s', docId);
      var task = new taskResult.TaskResultData();
      task.key = docId;
      //делаем select, потому что за время timeout информация могла измениться
      var selectRes = yield taskResult.select(docId);
      var row = selectRes.length > 0 ? selectRes[0] : null;
      if (row && row.status == taskResult.FileStatus.SaveVersion && row.status_info == statusInfo) {
        if (null == optFormat) {
          optFormat = constants.AVS_OFFICESTUDIO_FILE_OTHER_TEAMLAB_INNER;
        }
        var cmd = new commonDefines.InputCommand();
        cmd.setCommand('sfc');
        cmd.setDocId(docId);
        cmd.setOutputFormat(optFormat);
        cmd.setData(statusInfo);
        cmd.setUserActionId(opt_userId);
        yield* addRandomKeyTaskCmd(cmd);
        var queueData = getSaveTask(cmd);
        queueData.setFromChanges(true);
        yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_NORMAL, opt_queue);
        if (docsCoServer.getIsShutdown()) {
          yield utils.promiseRedis(redisClient, redisClient.sadd, redisKeyShutdown, docId);
        }
        logger.debug('AddTask saveFromChanges: docId = %s', docId);
      } else {
        if (row) {
          logger.debug('saveFromChanges status mismatch: docId = %s; row: %d; %d; expected: %d', docId, row.status, row.status_info, statusInfo);
        }
      }
      if (clientStatsD) {
        clientStatsD.timing('coauth.saveFromChanges', new Date() - startDate);
      }
    }
    catch (e) {
      logger.error('Error saveFromChanges: docId = %s\r\n%s', docId, e.stack);
    }
  });
};
exports.receiveTask = function(data, dataRaw) {
  return co(function* () {
    var docId = 'null';
    try {
      var task = new commonDefines.TaskQueueData(JSON.parse(data));
      if (task) {
        var cmd = task.getCmd();
        docId = cmd.getDocId();
        logger.debug('Start receiveTask: docId = %s %s', docId, data);
        var updateTask = getUpdateResponse(cmd);
        var updateRes = yield taskResult.update(updateTask);
        if (updateRes.affectedRows > 0) {
          var outputData = new OutputData(cmd.getCommand());
          var command = cmd.getCommand();
          var additionalOutput = {needUrlKey: null, needUrlMethod: null};
          if ('open' == command || 'reopen' == command) {
            //yield utils.sleep(5000);
            yield* getOutputData(cmd, outputData, cmd.getDocId(), updateTask.status,
              updateTask.statusInfo, null, additionalOutput);
          } else if ('save' == command || 'savefromorigin' == command || 'sfct' == command) {
            yield* getOutputData(cmd, outputData, cmd.getSaveKey(), updateTask.status,
              updateTask.statusInfo, null, additionalOutput);
          } else if ('sfcm' == command) {
            yield* commandSfcCallback(cmd, true);
          } else if ('sfc' == command) {
            yield* commandSfcCallback(cmd, false);
          } else if ('sendmm' == command) {
            yield* commandSendMMCallback(cmd);
          } else if ('conv' == command) {
            //nothing
          }
          if (outputData.getStatus()) {
            logger.debug('Send receiveTask: docId = %s %s', docId, JSON.stringify(outputData));
            var output = new OutputDataWrap('documentOpen', outputData);
            yield* docsCoServer.publish({
              type: commonDefines.c_oPublishType.receiveTask, cmd: cmd, output: output,
              needUrlKey: additionalOutput.needUrlKey, needUrlMethod: additionalOutput.needUrlMethod
            });
          }
        }
        yield* docsCoServer.removeResponse(dataRaw);
        logger.debug('End receiveTask: docId = %s', docId);
      }
    } catch (err) {
      logger.debug('Error receiveTask: docId = %s\r\n%s', docId, err.stack);
    }
  });
};

exports.cleanupCache = cleanupCache;
exports.commandSfctByCmd = commandSfctByCmd;
exports.commandOpenStartPromise = commandOpenStartPromise;
exports.OutputDataWrap = OutputDataWrap;
exports.OutputData = OutputData;
\ No newline at end of file

From 26ca3affcb212874eb86c786223f7dec1ae6b0eb Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Thu, 23 Jun 2016 13:39:13 +0300
Subject: [PATCH 02/34] healthcheck error

---
 DocService/sources/converterservice.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/DocService/sources/converterservice.js b/DocService/sources/converterservice.js
index 2f8bb1cd..7390248a 100644
--- a/DocService/sources/converterservice.js
+++ b/DocService/sources/converterservice.js
@@ -150,12 +150,13 @@ function convertHealthCheck(req, res) {
       var docId = task.key;
       //put test file to storage
       var data = yield utils.readFile(cfgHealthCheckFilePath);
-      yield storage.putObject(docId + '/origin', data, data.length);
+      var format = 'docx';
+      yield storage.putObject(docId + '/origin.' + format, data, data.length);
       //convert
       var cmd = new commonDefines.InputCommand();
       cmd.setCommand('conv');
       cmd.setSaveKey(docId);
-      cmd.setFormat('docx');
+      cmd.setFormat(format);
       cmd.setDocId(docId);
       cmd.setTitle('Editor.bin');
       cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CANVAS);

From 81ead010613197223420b6e448b1a5c484fd29a7 Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Tue, 19 Jul 2016 14:20:06 +0300
Subject: [PATCH 03/34] log

---
 FileConverter/sources/converter.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js
index 0d061a5c..0097c0dc 100644
--- a/FileConverter/sources/converter.js
+++ b/FileConverter/sources/converter.js
@@ -173,7 +173,7 @@ function* downloadFile(docId, uri, fileFrom) {
       res = true;
     } catch (err) {
       res = false;
-      logger.error('error downloadFile:url=%s;attempt=%d;(id=%s)\r\n%s', uri, downloadAttemptCount, docId, err.stack);
+      logger.error('error downloadFile:url=%s;attempt=%d;code:%s;connect:%s;(id=%s)\r\n%s', uri, downloadAttemptCount, err.code, err.connect, docId, err.stack);
       //not continue attempts if timeout
       if (err.code === 'ETIMEDOUT' || err.code === 'EMSGSIZE') {
         break;

From abeaafb8195a49886a7fdfe12db50144a7f1f459 Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Tue, 19 Jul 2016 15:18:36 +0300
Subject: [PATCH 04/34] check whether the key in the database in
 Commandservice.ashx?c=info

---
 DocService/sources/DocsCoServer.js | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js
index 9cebba2b..9bb33cb8 100644
--- a/DocService/sources/DocsCoServer.js
+++ b/DocService/sources/DocsCoServer.js
@@ -706,20 +706,22 @@ function* bindEvents(docId, callback, baseUrl, opt_userAction, opt_userData) {
   // - если пользователей нет и изменений нет, то отсылаем статус "закрыто" и в базу не добавляем
   // - если пользователей нет, а изменения есть, то отсылаем статус "редактируем" без пользователей, но добавляем в базу
   // - если есть пользователи, то просто добавляем в базу
-  var bChangeBase = c_oAscChangeBase.Delete;
-  var getRes = yield* getCallback(docId);
+  var bChangeBase;
   var oCallbackUrl;
+  var getRes = yield* getCallback(docId);
   if (getRes) {
     oCallbackUrl = getRes.server;
+    bChangeBase = c_oAscChangeBase.Delete;
   } else {
     oCallbackUrl = parseUrl(callback);
-    if (null === oCallbackUrl) {
-      return commonDefines.c_oAscServerCommandErrors.ParseError;
-    }
     bChangeBase = c_oAscChangeBase.All;
   }
-  yield* sendStatusDocument(docId, bChangeBase, opt_userAction, oCallbackUrl, baseUrl, opt_userData);
-  return commonDefines.c_oAscServerCommandErrors.NoError;
+  if (null === oCallbackUrl) {
+    return commonDefines.c_oAscServerCommandErrors.ParseError;
+  } else {
+    yield* sendStatusDocument(docId, bChangeBase, opt_userAction, oCallbackUrl, baseUrl, opt_userData);
+    return commonDefines.c_oAscServerCommandErrors.NoError;
+  }
 }
 
 function* cleanDocumentOnExit(docId, deleteChanges) {
@@ -2091,7 +2093,13 @@ exports.commandFromServer = function (req, res) {
         logger.debug('Start commandFromServer: docId = %s c = %s', docId, query.c);
         switch (query.c) {
           case 'info':
-            result = yield* bindEvents(docId, query.callback, utils.getBaseUrlByRequest(req), undefined, query.userdata);
+            //If no files in the database means they have not been edited.
+            var selectRes = yield taskResult.select(docId);
+            if (selectRes.length > 0) {
+              result = yield* bindEvents(docId, query.callback, utils.getBaseUrlByRequest(req), undefined, query.userdata);
+            } else {
+              result = commonDefines.c_oAscServerCommandErrors.DocumentIdError;
+            }
             break;
           case 'drop':
             if (query.userid) {

From 3c45122cfcbdee7266c92e36cac87310902ed3ef Mon Sep 17 00:00:00 2001
From: "Alexander.Trofimov" <Alexander.Trofimov@onlyoffice.com>
Date: Thu, 21 Jul 2016 16:41:52 +0300
Subject: [PATCH 05/34] delete old production config

---
 Common/config/production-windows.json | 22 ----------------------
 1 file changed, 22 deletions(-)
 delete mode 100644 Common/config/production-windows.json

diff --git a/Common/config/production-windows.json b/Common/config/production-windows.json
deleted file mode 100644
index df7529a7..00000000
--- a/Common/config/production-windows.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "services": {
-    "CoAuthoring": {
-      "utils": {
-        "utils_common_fontdir": "C:\\Windows\\Fonts",
-      }
-    }
-  },
-  "FileConverter": {
-    "converter": {
-      "fontDir": "C:\\Windows\\Fonts",
-      "presentationThemesDir": "../../../OfficeWeb/sdk/PowerPoint/themes",
-      "filePath": "../Bin/x2t.exe"
-    }
-  },
-  "license": {
-    "license_file": "./../../license.lic"
-  },
-  "FileStorage": {
-    "directory": "../App_Data"
-  }
-}
\ No newline at end of file

From d3c3c2f97e3813ffb6063f9ec3a0e6260d6fca49 Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Fri, 22 Jul 2016 13:48:58 +0300
Subject: [PATCH 06/34] don't check db or locks if connection restore in view
 mode

---
 DocService/sources/DocsCoServer.js | 113 +++++++++++++++--------------
 1 file changed, 60 insertions(+), 53 deletions(-)

diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js
index 9bb33cb8..b392807b 100644
--- a/DocService/sources/DocsCoServer.js
+++ b/DocService/sources/DocsCoServer.js
@@ -1232,6 +1232,16 @@ exports.install = function(server, callbackFunction) {
     return resultLock;
   }
 
+  function* authRestore(conn, sessionId) {
+    conn.sessionId = sessionId;//restore old
+    //Kill previous connections
+    connections = _.reject(connections, function(el) {
+      return el.sessionId === sessionId;//Delete this connection
+    });
+
+    yield* endAuth(conn, true);
+  }
+
   function* auth(conn, data) {
     // Проверка версий
     if (data.version !== asc_coAuthV) {
@@ -1302,71 +1312,68 @@ exports.install = function(server, callbackFunction) {
       if (bIsRestore) {
         logger.info("restored old session: docId = %s id = %s", docId, data.sessionId);
 
-        // Останавливаем сборку (вдруг она началась)
-        // Когда переподсоединение, нам нужна проверка на сборку файла
-        try {
-          var result = yield sqlBase.checkStatusFilePromise(docId);
+        if (!conn.user.view) {
+          // Останавливаем сборку (вдруг она началась)
+          // Когда переподсоединение, нам нужна проверка на сборку файла
+          try {
+            var result = yield sqlBase.checkStatusFilePromise(docId);
 
-          var status = result && result.length > 0 ? result[0]['status'] : null;
-          if (taskResult.FileStatus.Ok === status) {
-            // Все хорошо, статус обновлять не нужно
-          } else if (taskResult.FileStatus.SaveVersion === status) {
-            // Обновим статус файла (идет сборка, нужно ее остановить)
-            var updateMask = new taskResult.TaskResultData();
-            updateMask.key = docId;
-            updateMask.status = status;
-            updateMask.statusInfo = result[0]['status_info'];
-            var updateTask = new taskResult.TaskResultData();
-            updateTask.status = taskResult.FileStatus.Ok;
-            updateTask.statusInfo = constants.NO_ERROR;
-            var updateIfRes = yield taskResult.updateIf(updateTask, updateMask);
-            if (!(updateIfRes.affectedRows > 0)) {
+            var status = result && result.length > 0 ? result[0]['status'] : null;
+            if (taskResult.FileStatus.Ok === status) {
+              // Все хорошо, статус обновлять не нужно
+            } else if (taskResult.FileStatus.SaveVersion === status) {
+              // Обновим статус файла (идет сборка, нужно ее остановить)
+              var updateMask = new taskResult.TaskResultData();
+              updateMask.key = docId;
+              updateMask.status = status;
+              updateMask.statusInfo = result[0]['status_info'];
+              var updateTask = new taskResult.TaskResultData();
+              updateTask.status = taskResult.FileStatus.Ok;
+              updateTask.statusInfo = constants.NO_ERROR;
+              var updateIfRes = yield taskResult.updateIf(updateTask, updateMask);
+              if (!(updateIfRes.affectedRows > 0)) {
+                // error version
+                sendFileError(conn, 'Update Version error');
+                return;
+              }
+            } else if (taskResult.FileStatus.UpdateVersion === status) {
               // error version
               sendFileError(conn, 'Update Version error');
               return;
+            } else {
+              // Other error
+              sendFileError(conn, 'Other error');
+              return;
             }
-          } else if (taskResult.FileStatus.UpdateVersion === status) {
-            // error version
-            sendFileError(conn, 'Update Version error');
-            return;
-          } else {
-            // Other error
-            sendFileError(conn, 'Other error');
-            return;
-          }
 
-          var objChangesDocument = yield* getDocumentChanges(docId);
-          var bIsSuccessRestore = true;
-          if (objChangesDocument && 0 < objChangesDocument.arrChanges.length) {
-            var change = objChangesDocument.arrChanges[objChangesDocument.getLength() - 1];
-            if (change['change']) {
-              if (change['user'] !== curUserId) {
-                bIsSuccessRestore = 0 === (((data['lastOtherSaveTime'] - change['time']) / 1000) >> 0);
+            var objChangesDocument = yield* getDocumentChanges(docId);
+            var bIsSuccessRestore = true;
+            if (objChangesDocument && 0 < objChangesDocument.arrChanges.length) {
+              var change = objChangesDocument.arrChanges[objChangesDocument.getLength() - 1];
+              if (change['change']) {
+                if (change['user'] !== curUserId) {
+                  bIsSuccessRestore = 0 === (((data['lastOtherSaveTime'] - change['time']) / 1000) >> 0);
+                }
               }
             }
-          }
 
-          if (bIsSuccessRestore) {
-            conn.sessionId = data.sessionId;//restore old
-
-            // Проверяем lock-и
-            var arrayBlocks = data['block'];
-            var getLockRes = yield* getLock(conn, data, true);
-            if (arrayBlocks && (0 === arrayBlocks.length || getLockRes)) {
-              //Kill previous connections
-              connections = _.reject(connections, function(el) {
-                return el.sessionId === data.sessionId;//Delete this connection
-              });
-
-              yield* endAuth(conn, true);
+            if (bIsSuccessRestore) {
+              // Проверяем lock-и
+              var arrayBlocks = data['block'];
+              var getLockRes = yield* getLock(conn, data, true);
+              if (arrayBlocks && (0 === arrayBlocks.length || getLockRes)) {
+                yield* authRestore(conn, data.sessionId);
+              } else {
+                sendFileError(conn, 'Restore error. Locks not checked.');
+              }
             } else {
-              sendFileError(conn, 'Restore error. Locks not checked.');
+              sendFileError(conn, 'Restore error. Document modified.');
             }
-          } else {
-            sendFileError(conn, 'Restore error. Document modified.');
+          } catch (err) {
+            sendFileError(conn, 'DataBase error\r\n' + err.stack);
           }
-        } catch (err) {
-          sendFileError(conn, 'DataBase error\r\n' + err.stack);
+        } else {
+          yield* authRestore(conn, data.sessionId);
         }
       } else {
         conn.sessionId = conn.id;

From d77dcb9fb36ab42c4e4c913306481d729f02057c Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Fri, 22 Jul 2016 14:03:53 +0300
Subject: [PATCH 07/34] pipe instead readFileSync to prevent memory overflow

---
 Common/sources/utils.js            | 24 ++++++++++++++++++++++--
 FileConverter/sources/converter.js | 15 ++-------------
 2 files changed, 24 insertions(+), 15 deletions(-)

diff --git a/Common/sources/utils.js b/Common/sources/utils.js
index 500e91e2..6b6f23f0 100644
--- a/Common/sources/utils.js
+++ b/Common/sources/utils.js
@@ -337,7 +337,7 @@ exports.fillXmlResponse = function(res, uri, error) {
   res.setHeader('Content-Length', body.length);
   res.send(body);
 };
-exports.promiseCreateWriteStream = function(strPath, optOptions) {
+function promiseCreateWriteStream(strPath, optOptions) {
   return new Promise(function(resolve, reject) {
     var file = fs.createWriteStream(strPath, optOptions);
     var errorCallback = function(e) {
@@ -350,7 +350,8 @@ exports.promiseCreateWriteStream = function(strPath, optOptions) {
     });
   });
 };
-exports.promiseCreateReadStream = function(strPath) {
+exports.promiseCreateWriteStream = promiseCreateWriteStream;
+function promiseCreateReadStream(strPath) {
   return new Promise(function(resolve, reject) {
     var file = fs.createReadStream(strPath);
     var errorCallback = function(e) {
@@ -363,6 +364,7 @@ exports.promiseCreateReadStream = function(strPath) {
     });
   });
 };
+exports.promiseCreateReadStream = promiseCreateReadStream;
 exports.compareStringByLength = function(x, y) {
   if (x && y) {
     if (x.length == y.length) {
@@ -480,3 +482,21 @@ function changeOnlyOfficeUrl(inputUrl, strPath, optFilename) {
   return inputUrl + constants.ONLY_OFFICE_URL_PARAM + '=' + constants.OUTPUT_NAME + path.extname(optFilename || strPath);
 }
 exports.changeOnlyOfficeUrl = changeOnlyOfficeUrl;
+function pipeStreams(from, to, isEnd) {
+  return new Promise(function(resolve, reject) {
+    from.pipe(to, {end: isEnd});
+    from.on('end', function() {
+      resolve();
+    });
+    from.on('error', function(e) {
+      reject(e);
+    });
+  });
+}
+exports.pipeStreams = pipeStreams;
+function* pipeFiles(from, to) {
+  var fromStream = yield promiseCreateReadStream(from);
+  var toStream = yield promiseCreateWriteStream(to);
+  yield pipeStreams(fromStream, toStream, true);
+}
+exports.pipeFiles = co.wrap(pipeFiles);
diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js
index 0097c0dc..f1b57d48 100644
--- a/FileConverter/sources/converter.js
+++ b/FileConverter/sources/converter.js
@@ -231,17 +231,6 @@ function* downloadFileFromStorage(id, strPath, dir) {
     fs.writeFileSync(path.join(dir, fileRel), data);
   }
 }
-function pipeFile(fsFrom, fsTo) {
-  return new Promise(function(resolve, reject) {
-    fsFrom.pipe(fsTo, {end: false});
-    fsFrom.on('end', function() {
-      resolve();
-    });
-    fsFrom.on('error', function(e) {
-      reject(e);
-    });
-  });
-}
 function* processDownloadFromStorage(dataConvert, cmd, task, tempDirs) {
   if (task.getFromOrigin() || task.getFromSettings()) {
     dataConvert.fileFrom = path.join(tempDirs.source, 'origin.' + cmd.getFormat());
@@ -262,7 +251,7 @@ function* processDownloadFromStorage(dataConvert, cmd, task, tempDirs) {
           fsFullFile = yield utils.promiseCreateWriteStream(dataConvert.fileFrom);
         }
         var fsCurFile = yield utils.promiseCreateReadStream(file);
-        yield pipeFile(fsCurFile, fsFullFile);
+        yield utils.pipeStreams(fsCurFile, fsFullFile, false);
       }
     }
     if (fsFullFile) {
@@ -473,7 +462,7 @@ function* ExecuteTask(task) {
   if (constants.NO_ERROR === error) {
     if(constants.AVS_OFFICESTUDIO_FILE_OTHER_HTMLZIP === dataConvert.formatTo && cmd.getSaveKey() && !dataConvert.mailMergeSend) {
       //todo заглушка.вся конвертация на клиенте, но нет простого механизма сохранения на клиенте
-      fs.writeFileSync(dataConvert.fileTo, fs.readFileSync(dataConvert.fileFrom));
+      yield utils.pipeFiles(dataConvert.fileFrom, dataConvert.fileTo);
     } else {
       var paramsFile = path.join(tempDirs.temp, 'params.xml');
       dataConvert.serialize(paramsFile);

From 3a4f046e5c13a1f692ed107616d7ae3684edb748 Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Tue, 26 Jul 2016 12:03:33 +0300
Subject: [PATCH 08/34] IRI to URI

---
 Common/package.json     |  3 ++-
 Common/sources/utils.js | 15 +++++----------
 2 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/Common/package.json b/Common/package.json
index 89303880..0da38fc9 100644
--- a/Common/package.json
+++ b/Common/package.json
@@ -13,6 +13,7 @@
     "mime": "^1.3.4",
     "mkdirp": "^0.5.1",
     "node-statsd": "^0.1.1",
-    "request": "^2.69.0"
+    "request": "^2.69.0",
+    "uri-js": "^2.1.1"
   }
 }
diff --git a/Common/sources/utils.js b/Common/sources/utils.js
index 6b6f23f0..8e6df01e 100644
--- a/Common/sources/utils.js
+++ b/Common/sources/utils.js
@@ -35,6 +35,7 @@ var path = require('path');
 var url = require('url');
 var request = require('request');
 var co = require('co');
+var URI = require("uri-js");
 var constants = require('./constants');
 
 var ANDROID_SAFE_FILENAME = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ._-+,@£$€!½§~\'=()[]{}0123456789';
@@ -192,11 +193,8 @@ exports.getContentDisposition = getContentDisposition;
 exports.getContentDispositionS3 = getContentDispositionS3;
 function downloadUrlPromise(uri, optTimeout, optLimit) {
   return new Promise(function (resolve, reject) {
-    //todo может стоит делать url.parse, а потом с каждой частью отдельно работать
-    //для ссылок с руссикими буквами приходит 404
-    if (!containsAllAsciiNP(uri)) {
-      uri = encodeURI(uri);
-    }
+    //IRI to URI
+    uri = URI.serialize(URI.parse(uri));
     var urlParsed = url.parse(uri);
     //if you expect binary data, you should set encoding: null
     var options = {uri: urlParsed, encoding: null, timeout: optTimeout};
@@ -227,11 +225,8 @@ function downloadUrlPromise(uri, optTimeout, optLimit) {
 }
 function postRequestPromise(uri, postData, optTimeout) {
   return new Promise(function(resolve, reject) {
-    //todo может стоит делать url.parse, а потом с каждой частью отдельно работать
-    //для ссылок с руссикими буквами приходит 404
-    if (!containsAllAsciiNP(uri)) {
-      uri = encodeURI(uri);
-    }
+    //IRI to URI
+    uri = URI.serialize(URI.parse(uri));
     var urlParsed = url.parse(uri);
     var options = {uri: urlParsed, body: postData, encoding: 'utf8', headers: {'Content-Type': 'application/json'}, timeout: optTimeout};
     if (urlParsed.protocol === 'https:') {

From a814943215b8f91bd8dfa59b0ebf8321f3368e93 Mon Sep 17 00:00:00 2001
From: Alexey Golubev <alexey.golubev@onlyoffice.com>
Date: Tue, 26 Jul 2016 13:51:38 +0300
Subject: [PATCH 09/34] Fixed the download error

Fixed the download error by a http link that redirected to a httpS link.
---
 Common/sources/utils.js | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/Common/sources/utils.js b/Common/sources/utils.js
index 8e6df01e..ae7bf5a7 100644
--- a/Common/sources/utils.js
+++ b/Common/sources/utils.js
@@ -198,11 +198,11 @@ function downloadUrlPromise(uri, optTimeout, optLimit) {
     var urlParsed = url.parse(uri);
     //if you expect binary data, you should set encoding: null
     var options = {uri: urlParsed, encoding: null, timeout: optTimeout};
-    if (urlParsed.protocol === 'https:') {
-      //TODO: Check how to correct handle a ssl link
-      urlParsed.rejectUnauthorized = false;
-      options.rejectUnauthorized = false;
-    }
+
+    //TODO: Check how to correct handle a ssl link
+    urlParsed.rejectUnauthorized = false;
+    options.rejectUnauthorized = false;
+
     request.get(options, function (err, response, body) {
       if (err) {
         reject(err);
@@ -229,11 +229,11 @@ function postRequestPromise(uri, postData, optTimeout) {
     uri = URI.serialize(URI.parse(uri));
     var urlParsed = url.parse(uri);
     var options = {uri: urlParsed, body: postData, encoding: 'utf8', headers: {'Content-Type': 'application/json'}, timeout: optTimeout};
-    if (urlParsed.protocol === 'https:') {
-      //TODO: Check how to correct handle a ssl link
-      urlParsed.rejectUnauthorized = false;
-      options.rejectUnauthorized = false;
-    }
+
+    //TODO: Check how to correct handle a ssl link
+    urlParsed.rejectUnauthorized = false;
+    options.rejectUnauthorized = false;
+
     request.post(options, function(err, response, body) {
       if (err) {
         reject(err);

From 4a060fa0c8497cc434444e83dff35e94e082ec2c Mon Sep 17 00:00:00 2001
From: Alexander Trofimov <Alexander.Trofimov@onlyoffice.com>
Date: Thu, 28 Jul 2016 16:37:25 +0300
Subject: [PATCH 10/34] Update Readme.md

---
 Readme.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Readme.md b/Readme.md
index 58a4e871..21d1dcc0 100644
--- a/Readme.md
+++ b/Readme.md
@@ -1,5 +1,5 @@
 
-[![License](https://img.shields.io/badge/License-GNU%20AGPL%20V3-green.svg?style=flat)](http://www.gnu.org/licenses/agpl-3.0.ru.html) ![Release](https://img.shields.io/badge/Release-v4.0.1-blue.svg?style=flat)
+[![License](https://img.shields.io/badge/License-GNU%20AGPL%20V3-green.svg?style=flat)](http://www.gnu.org/licenses/agpl-3.0.ru.html) ![Release](https://img.shields.io/badge/Release-v4.1.0-blue.svg?style=flat)
 
 ## Server
 The backend server software layer which is the part of [ONLYOFFICE Document Server][2] and is the base for all other components.

From 414ccdfabe67bc0c45ecd73ed251e5541c3e0495 Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Tue, 2 Aug 2016 18:50:58 +0300
Subject: [PATCH 11/34] for bug 30749 add MailMerge field 'recordErrorCount' in
 callback

---
 Common/sources/commondefines.js     | 16 ++++++++++++++++
 DocService/sources/canvasservice.js |  2 +-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/Common/sources/commondefines.js b/Common/sources/commondefines.js
index 8844a327..a1cd31d8 100644
--- a/Common/sources/commondefines.js
+++ b/Common/sources/commondefines.js
@@ -285,6 +285,7 @@ function CMailMergeSendData(obj) {
     this['recordFrom'] = obj['recordFrom'];
     this['recordTo'] = obj['recordTo'];
     this['recordCount'] = obj['recordCount'];
+    this['recordErrorCount'] = obj['recordErrorCount'];
     this['userId'] = obj['userId'];
     this['url'] = obj['url'];
     this['baseUrl'] = obj['baseUrl'];
@@ -299,6 +300,7 @@ function CMailMergeSendData(obj) {
     this['recordFrom'] = null;
     this['recordTo'] = null;
     this['recordCount'] = null;
+    this['recordErrorCount'] = null;
     this['userId'] = null;
     this['url'] = null;
     this['baseUrl'] = null;
@@ -359,6 +361,12 @@ CMailMergeSendData.prototype.getRecordCount = function() {
 CMailMergeSendData.prototype.setRecordCount = function(v) {
   this['recordCount'] = v;
 };
+CMailMergeSendData.prototype.getRecordErrorCount = function() {
+  return this['recordErrorCount']
+};
+CMailMergeSendData.prototype.setRecordErrorCount = function(v) {
+  this['recordErrorCount'] = v;
+};
 CMailMergeSendData.prototype.getUserId = function() {
   return this['userId']
 };
@@ -544,6 +552,7 @@ function OutputMailMerge(mailMergeSendData) {
         break;
     }
     this['recordCount'] = mailMergeSendData.getRecordCount();
+    this['recordErrorCount'] = mailMergeSendData.getRecordErrorCount();
     this['to'] = null;
     this['recordIndex'] = null;
   } else {
@@ -555,6 +564,7 @@ function OutputMailMerge(mailMergeSendData) {
     this['type'] = null;
     this['recordCount'] = null;
     this['recordIndex'] = null;
+    this['recordErrorCount'] = null;
   }
 }
 OutputMailMerge.prototype.getRecordIndex = function() {
@@ -563,6 +573,12 @@ OutputMailMerge.prototype.getRecordIndex = function() {
 OutputMailMerge.prototype.setRecordIndex = function(data) {
   return this['recordIndex'] = data;
 };
+OutputMailMerge.prototype.getRecordErrorCount = function() {
+  return this['recordErrorCount'];
+};
+OutputMailMerge.prototype.setRecordErrorCount = function(data) {
+  return this['recordErrorCount'] = data;
+};
 OutputMailMerge.prototype.getTo = function() {
   return this['to'];
 };
diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js
index 8288477b..25124945 100644
--- a/DocService/sources/canvasservice.js
+++ b/DocService/sources/canvasservice.js
@@ -1 +1 @@
-/*
 * (c) Copyright Ascensio System SIA 2010-2016
 *
 * This program is a free software product. You can redistribute it and/or
 * modify it under the terms of the GNU Affero General Public License (AGPL)
 * version 3 as published by the Free Software Foundation. In accordance with
 * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect
 * that Ascensio System SIA expressly excludes the warranty of non-infringement
 * of any third-party rights.
 *
 * This program is distributed WITHOUT ANY WARRANTY; without even the implied
 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE. For
 * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
 *
 * You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia,
 * EU, LV-1021.
 *
 * The  interactive user interfaces in modified source and object code versions
 * of the Program must display Appropriate Legal Notices, as required under
 * Section 5 of the GNU AGPL version 3.
 *
 * Pursuant to Section 7(b) of the License you must retain the original Product
 * logo when distributing the program. Pursuant to Section 7(e) we decline to
 * grant you any rights under trademark law for use of our trademarks.
 *
 * All the Product's GUI elements, including illustrations and icon sets, as
 * well as technical writing content are licensed under the terms of the
 * Creative Commons Attribution-ShareAlike 4.0 International. See the License
 * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
 *
 */

var pathModule = require('path');
var urlModule = require('url');
var co = require('co');
var sqlBase = require('./baseConnector');
var docsCoServer = require('./DocsCoServer');
var taskResult = require('./taskresult');
var logger = require('./../../Common/sources/logger');
var utils = require('./../../Common/sources/utils');
var constants = require('./../../Common/sources/constants');
var commonDefines = require('./../../Common/sources/commondefines');
var storage = require('./../../Common/sources/storage-base');
var formatChecker = require('./../../Common/sources/formatchecker');
var statsDClient = require('./../../Common/sources/statsdclient');
var config = require('config');
var config_server = config.get('services.CoAuthoring.server');
var config_utils = config.get('services.CoAuthoring.utils');
var pubsubRedis = require('./pubsubRedis');


var cfgTypesUpload = config_utils.get('limits_image_types_upload');
var cfgTypesCopy = config_utils.get('limits_image_types_copy');
var cfgImageSize = config_server.get('limits_image_size');
var cfgImageDownloadTimeout = config_server.get('limits_image_download_timeout');
var cfgRedisPrefix = config.get('services.CoAuthoring.redis.prefix');

var SAVE_TYPE_PART_START = 0;
var SAVE_TYPE_PART = 1;
var SAVE_TYPE_COMPLETE = 2;
var SAVE_TYPE_COMPLETE_ALL = 3;

var clientStatsD = statsDClient.getClient();
var redisClient = pubsubRedis.getClientRedis();
var redisKeySaved = cfgRedisPrefix + constants.REDIS_KEY_SAVED;
var redisKeyShutdown = cfgRedisPrefix + constants.REDIS_KEY_SHUTDOWN;

function OutputDataWrap(type, data) {
  this['type'] = type;
  this['data'] = data;
}
OutputDataWrap.prototype = {
  fromObject: function(data) {
    this['type'] = data['type'];
    this['data'] = new OutputData();
    this['data'].fromObject(data['data']);
  },
  getType: function() {
    return this['type'];
  },
  setType: function(data) {
    this['type'] = data;
  },
  getData: function() {
    return this['data'];
  },
  setData: function(data) {
    this['data'] = data;
  }
};
function OutputData(type) {
  this['type'] = type;
  this['status'] = undefined;
  this['data'] = undefined;
}
OutputData.prototype = {
  fromObject: function(data) {
    this['type'] = data['type'];
    this['status'] = data['status'];
    this['data'] = data['data'];
  },
  getType: function() {
    return this['type'];
  },
  setType: function(data) {
    this['type'] = data;
  },
  getStatus: function() {
    return this['status'];
  },
  setStatus: function(data) {
    this['status'] = data;
  },
  getData: function() {
    return this['data'];
  },
  setData: function(data) {
    this['data'] = data;
  }
};

function* getOutputData(cmd, outputData, key, status, statusInfo, optConn, optAdditionalOutput) {
  var docId = cmd.getDocId();
  switch (status) {
    case taskResult.FileStatus.SaveVersion:
    case taskResult.FileStatus.UpdateVersion:
    case taskResult.FileStatus.Ok:
      if(taskResult.FileStatus.Ok == status) {
        outputData.setStatus('ok');
      } else if(taskResult.FileStatus.SaveVersion == status) {
        if (optConn && optConn.user.view) {
          outputData.setStatus('updateversion');
        } else {
          var updateMask = new taskResult.TaskResultData();
          updateMask.key = docId;
          updateMask.status = status;
          updateMask.statusInfo = statusInfo;
          var updateTask = new taskResult.TaskResultData();
          updateTask.status = taskResult.FileStatus.Ok;
          updateTask.statusInfo = constants.NO_ERROR;
          var updateIfRes = yield taskResult.updateIf(updateTask, updateMask);
          if (updateIfRes.affectedRows > 0) {
            outputData.setStatus('ok');
          } else {
            outputData.setStatus('updateversion');
          }
        }
      } else {
        outputData.setStatus('updateversion');
      }
      var command = cmd.getCommand();
      if ('open' != command && 'reopen' != command) {
        var strPath = key + '/' + cmd.getOutputPath();
        if (optConn) {
          var contentDisposition = cmd.getInline() ? constants.CONTENT_DISPOSITION_INLINE : constants.CONTENT_DISPOSITION_ATTACHMENT;
          outputData.setData(yield storage.getSignedUrl(optConn.baseUrl, strPath, null, cmd.getTitle(), contentDisposition));
        } else if (optAdditionalOutput) {
          optAdditionalOutput.needUrlKey = strPath;
          optAdditionalOutput.needUrlMethod = 2;
        }
      } else {
        if (optConn) {
          outputData.setData(yield storage.getSignedUrls(optConn.baseUrl, key));
        } else if (optAdditionalOutput) {
          optAdditionalOutput.needUrlKey = key;
          optAdditionalOutput.needUrlMethod = 0;
        }
      }
      break;
    case taskResult.FileStatus.NeedParams:
      outputData.setStatus('needparams');
      var settingsPath = key + '/' + 'settings.json';
      if (optConn) {
        outputData.setData(yield storage.getSignedUrl(optConn.baseUrl, settingsPath));
      } else if (optAdditionalOutput) {
        optAdditionalOutput.needUrlKey = settingsPath;
        optAdditionalOutput.needUrlMethod = 1;
      }
      break;
    case taskResult.FileStatus.NeedPassword:
      outputData.setStatus('needpassword');
      break;
    case taskResult.FileStatus.Err:
    case taskResult.FileStatus.ErrToReload:
      outputData.setStatus('err');
      outputData.setData(statusInfo);
      if (taskResult.FileStatus.ErrToReload == status) {
        yield cleanupCache(key);
      }
      break;
  }
}
function* addRandomKeyTaskCmd(cmd) {
  var task = yield* taskResult.addRandomKeyTask(cmd.getDocId());
  cmd.setSaveKey(task.key);
}
function* saveParts(cmd) {
  var result = false;
  var saveType = cmd.getSaveType();
  var filename;
  if (SAVE_TYPE_COMPLETE_ALL === saveType) {
    filename = 'Editor.bin';
  } else {
    filename = 'Editor' + (cmd.getSaveIndex() || '') + '.bin';
  }
  if (SAVE_TYPE_PART_START === saveType || SAVE_TYPE_COMPLETE_ALL === saveType) {
    yield* addRandomKeyTaskCmd(cmd);
  }
  if (cmd.getUrl()) {
    result = true;
  } else {
    var buffer = cmd.getData();
    yield storage.putObject(cmd.getSaveKey() + '/' + filename, buffer, buffer.length);
    //delete data to prevent serialize into json
    cmd.data = null;
    result = (SAVE_TYPE_COMPLETE_ALL === saveType || SAVE_TYPE_COMPLETE === saveType);
  }
  return result;
}
function getSaveTask(cmd) {
  cmd.setData(null);
  var queueData = new commonDefines.TaskQueueData();
  queueData.setCmd(cmd);
  queueData.setToFile(constants.OUTPUT_NAME + '.' + formatChecker.getStringFromFormat(cmd.getOutputFormat()));
  //todo paid
  //if (cmd.vkey) {
  //  bool
  //  bPaid;
  //  Signature.getVKeyParams(cmd.vkey, out bPaid);
  //  oTaskQueueData.m_bPaid = bPaid;
  //}
  return queueData;
}
function getUpdateResponse(cmd) {
  var updateTask = new taskResult.TaskResultData();
  updateTask.key = cmd.getSaveKey() ? cmd.getSaveKey() : cmd.getDocId();
  var statusInfo = cmd.getStatusInfo();
  if (constants.NO_ERROR == statusInfo) {
    updateTask.status = taskResult.FileStatus.Ok;
  } else if (constants.CONVERT_DOWNLOAD == statusInfo) {
    updateTask.status = taskResult.FileStatus.ErrToReload;
  } else if (constants.CONVERT_NEED_PARAMS == statusInfo) {
    updateTask.status = taskResult.FileStatus.NeedParams;
  } else if (constants.CONVERT_DRM == statusInfo) {
    updateTask.status = taskResult.FileStatus.NeedPassword;
  } else {
    updateTask.status = taskResult.FileStatus.Err;
  }
  updateTask.statusInfo = statusInfo;
  if (cmd.getTitle()) {
    updateTask.title = cmd.getTitle();
  }
  return updateTask;
}
var cleanupCache = co.wrap(function* (docId) {
  //todo redis ?
  var res = false;
  var removeRes = yield taskResult.remove(docId);
  if (removeRes.affectedRows > 0) {
    yield storage.deletePath(docId);
    res = true;
  }
  return res;
});

function commandOpenStartPromise(docId, cmd, opt_updateUserIndex) {
  var task = new taskResult.TaskResultData();
  task.key = docId;
  task.status = taskResult.FileStatus.WaitQueue;
  task.statusInfo = constants.NO_ERROR;
  if (cmd) {
    task.title = cmd.getTitle();
  } else {
    logger.warn("commandOpenStartPromise empty cmd: docId = %s", docId);
  }

  return taskResult.upsert(task, opt_updateUserIndex);
}
function* commandOpen(conn, cmd, outputData, opt_upsertRes) {
  var upsertRes;
  if (opt_upsertRes) {
    upsertRes = opt_upsertRes;
  } else {
    upsertRes = yield commandOpenStartPromise(cmd.getDocId(), cmd);
  }
  //if CLIENT_FOUND_ROWS don't specify 1 row is inserted , 2 row is updated, and 0 row is set to its current values
  //http://dev.mysql.com/doc/refman/5.7/en/insert-on-duplicate.html
  var bCreate = upsertRes.affectedRows == 1;
  if (!bCreate) {
    var selectRes = yield taskResult.select(cmd.getDocId());
    if (selectRes.length > 0) {
      var row = selectRes[0];
      yield* getOutputData(cmd, outputData, cmd.getDocId(), row.status, row.status_info, conn);
    }
  } else {
    //add task
    cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CANVAS);
    cmd.setEmbeddedFonts(false);
    var dataQueue = new commonDefines.TaskQueueData();
    dataQueue.setCmd(cmd);
    dataQueue.setToFile('Editor.bin');
    var priority = constants.QUEUE_PRIORITY_HIGH;
    var formatIn = formatChecker.getFormatFromString(cmd.getFormat());
    //decrease pdf, djvu, xps convert priority becase long open time
    if (constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === formatIn || constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_DJVU === formatIn ||
      constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_XPS === formatIn) {
      priority = constants.QUEUE_PRIORITY_LOW;
    }
    yield* docsCoServer.addTask(dataQueue, priority);
  }
}
function* commandReopen(cmd) {
  var task = new taskResult.TaskResultData();
  task.key = cmd.getDocId();
  task.status = taskResult.FileStatus.WaitQueue;
  task.statusInfo = constants.NO_ERROR;

  var upsertRes = yield taskResult.update(task);
  if (upsertRes.affectedRows > 0) {
    //add task
    cmd.setUrl(null);//url may expire
    cmd.setSaveKey(cmd.getDocId());
    cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CANVAS);
    cmd.setEmbeddedFonts(false);
    var dataQueue = new commonDefines.TaskQueueData();
    dataQueue.setCmd(cmd);
    dataQueue.setToFile('Editor.bin');
    dataQueue.setFromSettings(true);
    yield* docsCoServer.addTask(dataQueue, constants.QUEUE_PRIORITY_HIGH);
  }
}
function* commandSave(cmd, outputData) {
  var completeParts = yield* saveParts(cmd);
  if (completeParts) {
    var queueData = getSaveTask(cmd);
    yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
  }
  outputData.setStatus('ok');
  outputData.setData(cmd.getSaveKey());
}
function* commandSendMailMerge(cmd, outputData) {
  var completeParts = yield* saveParts(cmd);
  var isErr = false;
  if (completeParts) {
    isErr = true;
    var getRes = yield* docsCoServer.getCallback(cmd.getDocId());
    if (getRes) {
      var mailMergeSend = cmd.getMailMergeSend();
      mailMergeSend.setUrl(getRes.server.href);
      mailMergeSend.setBaseUrl(getRes.baseUrl);
      //меняем JsonKey и SaveKey, новый key нужет потому что за одну конвертацию делается часть, а json нужен всегда
      mailMergeSend.setJsonKey(cmd.getSaveKey());
      yield* addRandomKeyTaskCmd(cmd);
      var queueData = getSaveTask(cmd);
      yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
      isErr = false;
    }
  }
  if (isErr) {
    outputData.setStatus('err');
    outputData.setData(constants.UNKNOWN);
  } else {
    outputData.setStatus('ok');
    outputData.setData(cmd.getSaveKey());
  }
}
function* commandSfctByCmd(cmd) {
  yield* addRandomKeyTaskCmd(cmd);
  var queueData = getSaveTask(cmd);
  queueData.setFromChanges(true);
  yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
}
function* commandSfct(cmd, outputData) {
  yield* commandSfctByCmd(cmd);
  outputData.setStatus('ok');
}
function isDisplayedImage(strName) {
  var res = 0;
  if (strName) {
    //шаблон display[N]image.ext
    var findStr = constants.DISPLAY_PREFIX;
    var index = strName.indexOf(findStr);
    if (-1 != index) {
      if (index + findStr.length < strName.length) {
        var displayN = parseInt(strName[index + findStr.length]);
        if (1 <= displayN && displayN <= 6) {
          var imageIndex = index + findStr.length + 1;
          if (imageIndex == strName.indexOf("image", imageIndex))
            res = displayN;
        }
      }
    }
  }
  return res;
}
function* commandImgurls(conn, cmd, outputData) {
  var supportedFormats;
  var urls;
  var errorCode = constants.NO_ERROR;
  var isImgUrl = 'imgurl' == cmd.getCommand();
  if (isImgUrl) {
    urls = [cmd.getData()];
    supportedFormats = cfgTypesUpload || 'jpg';
  } else {
    urls = cmd.getData();
    supportedFormats = cfgTypesCopy || 'jpg';
  }
  //todo Promise.all()
  var displayedImageMap = {};//to make one imageIndex for ole object urls
  var imageCount = 0;
  var outputUrls = [];
  for (var i = 0; i < urls.length; ++i) {
    var urlSource = urls[i];
    var urlParsed;
    var data = undefined;
    if (urlSource.startsWith('data:')) {
      var delimiterIndex = urlSource.indexOf(',');
      if (-1 != delimiterIndex && (urlSource.length - (delimiterIndex + 1)) * 0.75 <= cfgImageSize) {
        data = new Buffer(urlSource.substring(delimiterIndex + 1), 'base64');
      }
    } else if(urlSource) {
      //todo stream
      data = yield utils.downloadUrlPromise(urlSource, cfgImageDownloadTimeout * 1000, cfgImageSize);
      urlParsed = urlModule.parse(urlSource);
    }
    var outputUrl = {url: 'error', path: 'error'};
    if (data) {
      var format = formatChecker.getImageFormat(data);
      var formatStr;
      if (constants.AVS_OFFICESTUDIO_FILE_UNKNOWN == format && urlParsed) {
        //bin, txt occur in ole object case
        var ext = pathModule.extname(urlParsed.pathname);
        if ('.bin' == ext || '.txt' == ext) {
          formatStr = ext.substring(1);
        }
      } else {
        formatStr = formatChecker.getStringFromFormat(format);
      }
      if (formatStr && -1 !== supportedFormats.indexOf(formatStr)) {
        var userid = cmd.getUserId();
        var imageIndex = cmd.getSaveIndex() + imageCount;
        imageCount++;
        var strLocalPath = 'media/' + utils.crc32(userid).toString(16) + '_';
        if (urlParsed) {
          var urlBasename = pathModule.basename(urlParsed.pathname);
          var displayN = isDisplayedImage(urlBasename);
          if (displayN > 0) {
            var displayedImageName = urlBasename.substring(0, urlBasename.length - formatStr.length - 1);
            var tempIndex = displayedImageMap[displayedImageName];
            if (null != tempIndex) {
              imageIndex = tempIndex;
              imageCount--;
            } else {
              displayedImageMap[displayedImageName] = imageIndex;
            }
            strLocalPath += constants.DISPLAY_PREFIX + displayN;
          }
        }
        strLocalPath += 'image' + imageIndex + '.' + formatStr;
        var strPath = cmd.getDocId() + '/' + strLocalPath;
        yield storage.putObject(strPath, data, data.length);
        var imgUrl = yield storage.getSignedUrl(conn.baseUrl, strPath);
        outputUrl = {url: imgUrl, path: strLocalPath};
      }
    }
    if (isImgUrl && ('error' === outputUrl.url || 'error' === outputUrl.path)) {
      errorCode = constants.UPLOAD_EXTENSION;
      break;
    }
    outputUrls.push(outputUrl);
  }
  if (constants.NO_ERROR !== errorCode) {
    outputData.setStatus('err');
    outputData.setData(errorCode);
  } else {
    outputData.setStatus('ok');
    outputData.setData(outputUrls);
  }
}
function* commandPathUrl(conn, cmd, outputData) {
  var contentDisposition = cmd.getInline() ? constants.CONTENT_DISPOSITION_INLINE :
    constants.CONTENT_DISPOSITION_ATTACHMENT;
  var strPath = cmd.getDocId() + '/' + cmd.getData();
  var url = yield storage.getSignedUrl(conn.baseUrl, strPath, null, cmd.getTitle(), contentDisposition);
  var errorCode = constants.NO_ERROR;
  if (constants.NO_ERROR !== errorCode) {
    outputData.setStatus('err');
    outputData.setData(errorCode);
  } else {
    outputData.setStatus('ok');
    outputData.setData(url);
  }
}
function* commandSaveFromOrigin(cmd, outputData) {
  yield* addRandomKeyTaskCmd(cmd);
  var queueData = getSaveTask(cmd);
  queueData.setFromOrigin(true);
  yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
  outputData.setStatus('ok');
  outputData.setData(cmd.getSaveKey());
}
function* commandSfcCallback(cmd, isSfcm) {
  var docId = cmd.getDocId();
  logger.debug('Start commandSfcCallback: docId = %s', docId);
  var saveKey = cmd.getSaveKey();
  var statusInfo = cmd.getStatusInfo();
  var isError = constants.NO_ERROR != statusInfo && constants.CONVERT_CORRUPTED != statusInfo;
  var savePathDoc = saveKey + '/' + cmd.getOutputPath();
  var savePathChanges = saveKey + '/changes.zip';
  var savePathHistory = saveKey + '/changesHistory.json';
  var getRes = yield* docsCoServer.getCallback(docId);
  if (getRes) {
    logger.debug('Callback commandSfcCallback: docId = %s callback = %s', docId, getRes.server.href);
    var outputSfc = new commonDefines.OutputSfcData();
    outputSfc.setKey(docId);
    var users = [];
    if (cmd.getUserId()) {
      users.push(cmd.getUserId());
    }
    outputSfc.setUsers(users);
    if (!isSfcm) {
      var actions = [];
      //use UserId case UserActionId miss in gc convertion
      var userActionId = cmd.getUserActionId() || cmd.getUserId();
      if (userActionId) {
        actions.push(new commonDefines.OutputAction(commonDefines.c_oAscUserAction.Out, userActionId));
      }
      outputSfc.setActions(actions);
    }
    outputSfc.setUserData(cmd.getUserData());
    if (!isError) {
      try {
        var data = yield storage.getObject(savePathHistory);
        outputSfc.setChangeHistory(data.toString('utf-8'));
        outputSfc.setUrl(yield storage.getSignedUrl(getRes.baseUrl, savePathDoc));
        outputSfc.setChangeUrl(yield storage.getSignedUrl(getRes.baseUrl, savePathChanges));
      } catch (e) {
        logger.error('Error commandSfcCallback: docId = %s\r\n%s', docId, e.stack);
      }
      if (outputSfc.getUrl() && outputSfc.getUsers().length > 0) {
        if (isSfcm) {
          outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MustSaveForce);
        } else {
          outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MustSave);
        }
      } else {
        isError = true;
      }
    }
    if (isError) {
      if (isSfcm) {
        outputSfc.setStatus(docsCoServer.c_oAscServerStatus.CorruptedForce);
      } else {
        outputSfc.setStatus(docsCoServer.c_oAscServerStatus.Corrupted);
      }
    }
    var uri = getRes.server.href;
    var postData = JSON.stringify(outputSfc);
    if (isSfcm) {
      var lastSave = cmd.getLastSave();
      if (lastSave && !isError) {
        yield* docsCoServer.setForceSave(docId, lastSave, savePathDoc);
      }
      try {
        yield* docsCoServer.sendServerRequest(docId, uri, postData);
      } catch (err) {
        logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack);
      }
    } else {
      //if anybody in document stop save
      var hasEditors = yield* docsCoServer.hasEditors(docId);
      logger.debug('hasEditors commandSfcCallback: docId = %s hasEditors = %d', docId, hasEditors);
      if (!hasEditors) {
        var updateMask = new taskResult.TaskResultData();
        updateMask.key = docId;
        updateMask.status = taskResult.FileStatus.SaveVersion;
        updateMask.statusInfo = cmd.getData();
        var updateIfTask = new taskResult.TaskResultData();
        updateIfTask.status = taskResult.FileStatus.UpdateVersion;
        updateIfTask.statusInfo = constants.NO_ERROR;
        var updateIfRes = yield taskResult.updateIf(updateIfTask, updateMask);
        if (updateIfRes.affectedRows > 0) {
          var replyStr = null;
          try {
            replyStr = yield* docsCoServer.sendServerRequest(docId, uri, postData);
          } catch (err) {
            replyStr = null;
            logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack);
          }
          var requestRes = false;
          var replyData = docsCoServer.parseReplyData(docId, replyStr);
          if (replyData && commonDefines.c_oAscServerCommandErrors.NoError == replyData.error) {
            //в случае comunity server придет запрос в CommandService проверяем результат
            var multi = redisClient.multi([
              ['get', redisKeySaved + docId],
              ['del', redisKeySaved + docId]
            ]);
            var execRes = yield utils.promiseRedis(multi, multi.exec);
            var savedVal = execRes[0];
            requestRes = (null == savedVal || '1' === savedVal);
          }
          if (requestRes) {
            yield docsCoServer.cleanDocumentOnExitPromise(docId, true);
          } else {
            var updateTask = new taskResult.TaskResultData();
            updateTask.key = docId;
            updateTask.status = taskResult.FileStatus.Ok;
            updateTask.statusInfo = constants.NO_ERROR;
            yield taskResult.update(updateTask);
          }
        }
      }
    }
  }
  if (docsCoServer.getIsShutdown() && !isSfcm) {
    yield utils.promiseRedis(redisClient, redisClient.srem, redisKeyShutdown, docId);
  }
  logger.debug('End commandSfcCallback: docId = %s', docId);
}
function* commandSendMMCallback(cmd) {
  var docId = cmd.getDocId();
  logger.debug('Start commandSendMMCallback: docId = %s', docId);
  var saveKey = cmd.getSaveKey();
  var statusInfo = cmd.getStatusInfo();
  var outputSfc = new commonDefines.OutputSfcData();
  outputSfc.setKey(docId);
  if (constants.NO_ERROR == statusInfo) {
    outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MailMerge);
  } else {
    outputSfc.setStatus(docsCoServer.c_oAscServerStatus.Corrupted);
  }
  var mailMergeSendData = cmd.getMailMergeSend();
  var outputMailMerge = new commonDefines.OutputMailMerge(mailMergeSendData);
  outputSfc.setMailMerge(outputMailMerge);
  outputSfc.setUsers([mailMergeSendData.getUserId()]);
  var data = yield storage.getObject(saveKey + '/' + cmd.getOutputPath());
  var xml = data.toString('utf8');
  var files = xml.match(/[< ]file.*?\/>/g);
  var recordRemain = (mailMergeSendData.getRecordTo() - mailMergeSendData.getRecordFrom() + 1);
  var recordIndexStart = mailMergeSendData.getRecordCount() - recordRemain;
  for (var i = 0; i < files.length; ++i) {
    var file = files[i];
    var fieldRes = /field=["'](.*?)["']/.exec(file);
    outputMailMerge.setTo(fieldRes[1]);
    outputMailMerge.setRecordIndex(recordIndexStart + i);
    var pathRes = /path=["'](.*?)["']/.exec(file);
    var signedUrl = yield storage.getSignedUrl(mailMergeSendData.getBaseUrl(), saveKey + '/' + pathRes[1]);
    outputSfc.setUrl(signedUrl);
    var uri = mailMergeSendData.getUrl();
    var postData = JSON.stringify(outputSfc);
    try {
      yield* docsCoServer.sendServerRequest(docId, uri, postData);
    } catch (err) {
      logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack);
    }
  }
  var newRecordFrom = mailMergeSendData.getRecordFrom() + Math.max(files.length, 1);
  if (newRecordFrom <= mailMergeSendData.getRecordTo()) {
    mailMergeSendData.setRecordFrom(newRecordFrom);
    yield* addRandomKeyTaskCmd(cmd);
    var queueData = getSaveTask(cmd);
    yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
  } else {
    logger.debug('End MailMerge: docId = %s', docId);
  }
  logger.debug('End commandSendMMCallback: docId = %s', docId);
}

exports.openDocument = function(conn, cmd, opt_upsertRes) {
  return co(function* () {
    var outputData;
    var docId = conn ? conn.docId : 'null';
    try {
      var startDate = null;
      if(clientStatsD) {
        startDate = new Date();
      }
      logger.debug('Start command: docId = %s %s', docId, JSON.stringify(cmd));
      outputData = new OutputData(cmd.getCommand());
      switch (cmd.getCommand()) {
        case 'open':
          //yield utils.sleep(5000);
          yield* commandOpen(conn, cmd, outputData, opt_upsertRes);
          break;
        case 'reopen':
          yield* commandReopen(cmd);
          break;
        case 'imgurl':
        case 'imgurls':
          yield* commandImgurls(conn, cmd, outputData);
          break;
        case 'pathurl':
          yield* commandPathUrl(conn, cmd, outputData);
          break;
        default:
          outputData.setStatus('err');
          outputData.setData(constants.UNKNOWN);
          break;
      }
      if(clientStatsD) {
        clientStatsD.timing('coauth.openDocument.' + cmd.getCommand(), new Date() - startDate);
      }
    }
    catch (e) {
      logger.error('Error openDocument: docId = %s\r\n%s', docId, e.stack);
      if (!outputData) {
        outputData = new OutputData();
      }
      outputData.setStatus('err');
      outputData.setData(constants.UNKNOWN);
    }
    finally {
      if (outputData && outputData.getStatus()) {
        logger.debug('Response command: docId = %s %s', docId, JSON.stringify(outputData));
        docsCoServer.sendData(conn, new OutputDataWrap('documentOpen', outputData));
      }
      logger.debug('End command: docId = %s', docId);
    }
  });
};
exports.downloadAs = function(req, res) {
  return co(function* () {
    var docId = 'null';
    try {
      var startDate = null;
      if(clientStatsD) {
        startDate = new Date();
      }
      var strCmd = req.query['cmd'];
      var cmd = new commonDefines.InputCommand(JSON.parse(strCmd));
      docId = cmd.getDocId();
      logger.debug('Start downloadAs: docId = %s %s', docId, strCmd);
      cmd.setData(req.body);
      var outputData = new OutputData(cmd.getCommand());
      switch (cmd.getCommand()) {
        case 'save':
          yield* commandSave(cmd, outputData);
          break;
        case 'savefromorigin':
          yield* commandSaveFromOrigin(cmd, outputData);
          break;
        case 'sendmm':
          yield* commandSendMailMerge(cmd, outputData);
          break;
        case 'sfct':
          yield* commandSfct(cmd, outputData);
          break;
        default:
          outputData.setStatus('err');
          outputData.setData(constants.UNKNOWN);
          break;
      }
      var strRes = JSON.stringify(outputData);
      res.send(strRes);
      logger.debug('End downloadAs: docId = %s %s', docId, strRes);
      if(clientStatsD) {
        clientStatsD.timing('coauth.downloadAs.' + cmd.getCommand(), new Date() - startDate);
      }
    }
    catch (e) {
      logger.error('Error downloadAs: docId = %s\r\n%s', docId, e.stack);
      res.sendStatus(400);
    }
  });
};
exports.saveFromChanges = function(docId, statusInfo, optFormat, opt_userId, opt_queue) {
  return co(function* () {
    try {
      var startDate = null;
      if(clientStatsD) {
        startDate = new Date();
      }
      logger.debug('Start saveFromChanges: docId = %s', docId);
      var task = new taskResult.TaskResultData();
      task.key = docId;
      //делаем select, потому что за время timeout информация могла измениться
      var selectRes = yield taskResult.select(docId);
      var row = selectRes.length > 0 ? selectRes[0] : null;
      if (row && row.status == taskResult.FileStatus.SaveVersion && row.status_info == statusInfo) {
        if (null == optFormat) {
          optFormat = constants.AVS_OFFICESTUDIO_FILE_OTHER_TEAMLAB_INNER;
        }
        var cmd = new commonDefines.InputCommand();
        cmd.setCommand('sfc');
        cmd.setDocId(docId);
        cmd.setOutputFormat(optFormat);
        cmd.setData(statusInfo);
        cmd.setUserActionId(opt_userId);
        yield* addRandomKeyTaskCmd(cmd);
        var queueData = getSaveTask(cmd);
        queueData.setFromChanges(true);
        yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_NORMAL, opt_queue);
        if (docsCoServer.getIsShutdown()) {
          yield utils.promiseRedis(redisClient, redisClient.sadd, redisKeyShutdown, docId);
        }
        logger.debug('AddTask saveFromChanges: docId = %s', docId);
      } else {
        if (row) {
          logger.debug('saveFromChanges status mismatch: docId = %s; row: %d; %d; expected: %d', docId, row.status, row.status_info, statusInfo);
        }
      }
      if (clientStatsD) {
        clientStatsD.timing('coauth.saveFromChanges', new Date() - startDate);
      }
    }
    catch (e) {
      logger.error('Error saveFromChanges: docId = %s\r\n%s', docId, e.stack);
    }
  });
};
exports.receiveTask = function(data, dataRaw) {
  return co(function* () {
    var docId = 'null';
    try {
      var task = new commonDefines.TaskQueueData(JSON.parse(data));
      if (task) {
        var cmd = task.getCmd();
        docId = cmd.getDocId();
        logger.debug('Start receiveTask: docId = %s %s', docId, data);
        var updateTask = getUpdateResponse(cmd);
        var updateRes = yield taskResult.update(updateTask);
        if (updateRes.affectedRows > 0) {
          var outputData = new OutputData(cmd.getCommand());
          var command = cmd.getCommand();
          var additionalOutput = {needUrlKey: null, needUrlMethod: null};
          if ('open' == command || 'reopen' == command) {
            //yield utils.sleep(5000);
            yield* getOutputData(cmd, outputData, cmd.getDocId(), updateTask.status,
              updateTask.statusInfo, null, additionalOutput);
          } else if ('save' == command || 'savefromorigin' == command || 'sfct' == command) {
            yield* getOutputData(cmd, outputData, cmd.getSaveKey(), updateTask.status,
              updateTask.statusInfo, null, additionalOutput);
          } else if ('sfcm' == command) {
            yield* commandSfcCallback(cmd, true);
          } else if ('sfc' == command) {
            yield* commandSfcCallback(cmd, false);
          } else if ('sendmm' == command) {
            yield* commandSendMMCallback(cmd);
          } else if ('conv' == command) {
            //nothing
          }
          if (outputData.getStatus()) {
            logger.debug('Send receiveTask: docId = %s %s', docId, JSON.stringify(outputData));
            var output = new OutputDataWrap('documentOpen', outputData);
            yield* docsCoServer.publish({
              type: commonDefines.c_oPublishType.receiveTask, cmd: cmd, output: output,
              needUrlKey: additionalOutput.needUrlKey, needUrlMethod: additionalOutput.needUrlMethod
            });
          }
        }
        yield* docsCoServer.removeResponse(dataRaw);
        logger.debug('End receiveTask: docId = %s', docId);
      }
    } catch (err) {
      logger.debug('Error receiveTask: docId = %s\r\n%s', docId, err.stack);
    }
  });
};

exports.cleanupCache = cleanupCache;
exports.commandSfctByCmd = commandSfctByCmd;
exports.commandOpenStartPromise = commandOpenStartPromise;
exports.OutputDataWrap = OutputDataWrap;
exports.OutputData = OutputData;
\ No newline at end of file
+/*
 * (c) Copyright Ascensio System SIA 2010-2016
 *
 * This program is a free software product. You can redistribute it and/or
 * modify it under the terms of the GNU Affero General Public License (AGPL)
 * version 3 as published by the Free Software Foundation. In accordance with
 * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect
 * that Ascensio System SIA expressly excludes the warranty of non-infringement
 * of any third-party rights.
 *
 * This program is distributed WITHOUT ANY WARRANTY; without even the implied
 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE. For
 * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
 *
 * You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia,
 * EU, LV-1021.
 *
 * The  interactive user interfaces in modified source and object code versions
 * of the Program must display Appropriate Legal Notices, as required under
 * Section 5 of the GNU AGPL version 3.
 *
 * Pursuant to Section 7(b) of the License you must retain the original Product
 * logo when distributing the program. Pursuant to Section 7(e) we decline to
 * grant you any rights under trademark law for use of our trademarks.
 *
 * All the Product's GUI elements, including illustrations and icon sets, as
 * well as technical writing content are licensed under the terms of the
 * Creative Commons Attribution-ShareAlike 4.0 International. See the License
 * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
 *
 */

var pathModule = require('path');
var urlModule = require('url');
var co = require('co');
var sqlBase = require('./baseConnector');
var docsCoServer = require('./DocsCoServer');
var taskResult = require('./taskresult');
var logger = require('./../../Common/sources/logger');
var utils = require('./../../Common/sources/utils');
var constants = require('./../../Common/sources/constants');
var commonDefines = require('./../../Common/sources/commondefines');
var storage = require('./../../Common/sources/storage-base');
var formatChecker = require('./../../Common/sources/formatchecker');
var statsDClient = require('./../../Common/sources/statsdclient');
var config = require('config');
var config_server = config.get('services.CoAuthoring.server');
var config_utils = config.get('services.CoAuthoring.utils');
var pubsubRedis = require('./pubsubRedis');


var cfgTypesUpload = config_utils.get('limits_image_types_upload');
var cfgTypesCopy = config_utils.get('limits_image_types_copy');
var cfgImageSize = config_server.get('limits_image_size');
var cfgImageDownloadTimeout = config_server.get('limits_image_download_timeout');
var cfgRedisPrefix = config.get('services.CoAuthoring.redis.prefix');

var SAVE_TYPE_PART_START = 0;
var SAVE_TYPE_PART = 1;
var SAVE_TYPE_COMPLETE = 2;
var SAVE_TYPE_COMPLETE_ALL = 3;

var clientStatsD = statsDClient.getClient();
var redisClient = pubsubRedis.getClientRedis();
var redisKeySaved = cfgRedisPrefix + constants.REDIS_KEY_SAVED;
var redisKeyShutdown = cfgRedisPrefix + constants.REDIS_KEY_SHUTDOWN;

function OutputDataWrap(type, data) {
  this['type'] = type;
  this['data'] = data;
}
OutputDataWrap.prototype = {
  fromObject: function(data) {
    this['type'] = data['type'];
    this['data'] = new OutputData();
    this['data'].fromObject(data['data']);
  },
  getType: function() {
    return this['type'];
  },
  setType: function(data) {
    this['type'] = data;
  },
  getData: function() {
    return this['data'];
  },
  setData: function(data) {
    this['data'] = data;
  }
};
function OutputData(type) {
  this['type'] = type;
  this['status'] = undefined;
  this['data'] = undefined;
}
OutputData.prototype = {
  fromObject: function(data) {
    this['type'] = data['type'];
    this['status'] = data['status'];
    this['data'] = data['data'];
  },
  getType: function() {
    return this['type'];
  },
  setType: function(data) {
    this['type'] = data;
  },
  getStatus: function() {
    return this['status'];
  },
  setStatus: function(data) {
    this['status'] = data;
  },
  getData: function() {
    return this['data'];
  },
  setData: function(data) {
    this['data'] = data;
  }
};

function* getOutputData(cmd, outputData, key, status, statusInfo, optConn, optAdditionalOutput) {
  var docId = cmd.getDocId();
  switch (status) {
    case taskResult.FileStatus.SaveVersion:
    case taskResult.FileStatus.UpdateVersion:
    case taskResult.FileStatus.Ok:
      if(taskResult.FileStatus.Ok == status) {
        outputData.setStatus('ok');
      } else if(taskResult.FileStatus.SaveVersion == status) {
        if (optConn && optConn.user.view) {
          outputData.setStatus('updateversion');
        } else {
          var updateMask = new taskResult.TaskResultData();
          updateMask.key = docId;
          updateMask.status = status;
          updateMask.statusInfo = statusInfo;
          var updateTask = new taskResult.TaskResultData();
          updateTask.status = taskResult.FileStatus.Ok;
          updateTask.statusInfo = constants.NO_ERROR;
          var updateIfRes = yield taskResult.updateIf(updateTask, updateMask);
          if (updateIfRes.affectedRows > 0) {
            outputData.setStatus('ok');
          } else {
            outputData.setStatus('updateversion');
          }
        }
      } else {
        outputData.setStatus('updateversion');
      }
      var command = cmd.getCommand();
      if ('open' != command && 'reopen' != command) {
        var strPath = key + '/' + cmd.getOutputPath();
        if (optConn) {
          var contentDisposition = cmd.getInline() ? constants.CONTENT_DISPOSITION_INLINE : constants.CONTENT_DISPOSITION_ATTACHMENT;
          outputData.setData(yield storage.getSignedUrl(optConn.baseUrl, strPath, null, cmd.getTitle(), contentDisposition));
        } else if (optAdditionalOutput) {
          optAdditionalOutput.needUrlKey = strPath;
          optAdditionalOutput.needUrlMethod = 2;
        }
      } else {
        if (optConn) {
          outputData.setData(yield storage.getSignedUrls(optConn.baseUrl, key));
        } else if (optAdditionalOutput) {
          optAdditionalOutput.needUrlKey = key;
          optAdditionalOutput.needUrlMethod = 0;
        }
      }
      break;
    case taskResult.FileStatus.NeedParams:
      outputData.setStatus('needparams');
      var settingsPath = key + '/' + 'settings.json';
      if (optConn) {
        outputData.setData(yield storage.getSignedUrl(optConn.baseUrl, settingsPath));
      } else if (optAdditionalOutput) {
        optAdditionalOutput.needUrlKey = settingsPath;
        optAdditionalOutput.needUrlMethod = 1;
      }
      break;
    case taskResult.FileStatus.NeedPassword:
      outputData.setStatus('needpassword');
      break;
    case taskResult.FileStatus.Err:
    case taskResult.FileStatus.ErrToReload:
      outputData.setStatus('err');
      outputData.setData(statusInfo);
      if (taskResult.FileStatus.ErrToReload == status) {
        yield cleanupCache(key);
      }
      break;
  }
}
function* addRandomKeyTaskCmd(cmd) {
  var task = yield* taskResult.addRandomKeyTask(cmd.getDocId());
  cmd.setSaveKey(task.key);
}
function* saveParts(cmd) {
  var result = false;
  var saveType = cmd.getSaveType();
  var filename;
  if (SAVE_TYPE_COMPLETE_ALL === saveType) {
    filename = 'Editor.bin';
  } else {
    filename = 'Editor' + (cmd.getSaveIndex() || '') + '.bin';
  }
  if (SAVE_TYPE_PART_START === saveType || SAVE_TYPE_COMPLETE_ALL === saveType) {
    yield* addRandomKeyTaskCmd(cmd);
  }
  if (cmd.getUrl()) {
    result = true;
  } else {
    var buffer = cmd.getData();
    yield storage.putObject(cmd.getSaveKey() + '/' + filename, buffer, buffer.length);
    //delete data to prevent serialize into json
    cmd.data = null;
    result = (SAVE_TYPE_COMPLETE_ALL === saveType || SAVE_TYPE_COMPLETE === saveType);
  }
  return result;
}
function getSaveTask(cmd) {
  cmd.setData(null);
  var queueData = new commonDefines.TaskQueueData();
  queueData.setCmd(cmd);
  queueData.setToFile(constants.OUTPUT_NAME + '.' + formatChecker.getStringFromFormat(cmd.getOutputFormat()));
  //todo paid
  //if (cmd.vkey) {
  //  bool
  //  bPaid;
  //  Signature.getVKeyParams(cmd.vkey, out bPaid);
  //  oTaskQueueData.m_bPaid = bPaid;
  //}
  return queueData;
}
function getUpdateResponse(cmd) {
  var updateTask = new taskResult.TaskResultData();
  updateTask.key = cmd.getSaveKey() ? cmd.getSaveKey() : cmd.getDocId();
  var statusInfo = cmd.getStatusInfo();
  if (constants.NO_ERROR == statusInfo) {
    updateTask.status = taskResult.FileStatus.Ok;
  } else if (constants.CONVERT_DOWNLOAD == statusInfo) {
    updateTask.status = taskResult.FileStatus.ErrToReload;
  } else if (constants.CONVERT_NEED_PARAMS == statusInfo) {
    updateTask.status = taskResult.FileStatus.NeedParams;
  } else if (constants.CONVERT_DRM == statusInfo) {
    updateTask.status = taskResult.FileStatus.NeedPassword;
  } else {
    updateTask.status = taskResult.FileStatus.Err;
  }
  updateTask.statusInfo = statusInfo;
  if (cmd.getTitle()) {
    updateTask.title = cmd.getTitle();
  }
  return updateTask;
}
var cleanupCache = co.wrap(function* (docId) {
  //todo redis ?
  var res = false;
  var removeRes = yield taskResult.remove(docId);
  if (removeRes.affectedRows > 0) {
    yield storage.deletePath(docId);
    res = true;
  }
  return res;
});

function commandOpenStartPromise(docId, cmd, opt_updateUserIndex) {
  var task = new taskResult.TaskResultData();
  task.key = docId;
  task.status = taskResult.FileStatus.WaitQueue;
  task.statusInfo = constants.NO_ERROR;
  if (cmd) {
    task.title = cmd.getTitle();
  } else {
    logger.warn("commandOpenStartPromise empty cmd: docId = %s", docId);
  }

  return taskResult.upsert(task, opt_updateUserIndex);
}
function* commandOpen(conn, cmd, outputData, opt_upsertRes) {
  var upsertRes;
  if (opt_upsertRes) {
    upsertRes = opt_upsertRes;
  } else {
    upsertRes = yield commandOpenStartPromise(cmd.getDocId(), cmd);
  }
  //if CLIENT_FOUND_ROWS don't specify 1 row is inserted , 2 row is updated, and 0 row is set to its current values
  //http://dev.mysql.com/doc/refman/5.7/en/insert-on-duplicate.html
  var bCreate = upsertRes.affectedRows == 1;
  if (!bCreate) {
    var selectRes = yield taskResult.select(cmd.getDocId());
    if (selectRes.length > 0) {
      var row = selectRes[0];
      yield* getOutputData(cmd, outputData, cmd.getDocId(), row.status, row.status_info, conn);
    }
  } else {
    //add task
    cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CANVAS);
    cmd.setEmbeddedFonts(false);
    var dataQueue = new commonDefines.TaskQueueData();
    dataQueue.setCmd(cmd);
    dataQueue.setToFile('Editor.bin');
    var priority = constants.QUEUE_PRIORITY_HIGH;
    var formatIn = formatChecker.getFormatFromString(cmd.getFormat());
    //decrease pdf, djvu, xps convert priority becase long open time
    if (constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === formatIn || constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_DJVU === formatIn ||
      constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_XPS === formatIn) {
      priority = constants.QUEUE_PRIORITY_LOW;
    }
    yield* docsCoServer.addTask(dataQueue, priority);
  }
}
function* commandReopen(cmd) {
  var task = new taskResult.TaskResultData();
  task.key = cmd.getDocId();
  task.status = taskResult.FileStatus.WaitQueue;
  task.statusInfo = constants.NO_ERROR;

  var upsertRes = yield taskResult.update(task);
  if (upsertRes.affectedRows > 0) {
    //add task
    cmd.setUrl(null);//url may expire
    cmd.setSaveKey(cmd.getDocId());
    cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CANVAS);
    cmd.setEmbeddedFonts(false);
    var dataQueue = new commonDefines.TaskQueueData();
    dataQueue.setCmd(cmd);
    dataQueue.setToFile('Editor.bin');
    dataQueue.setFromSettings(true);
    yield* docsCoServer.addTask(dataQueue, constants.QUEUE_PRIORITY_HIGH);
  }
}
function* commandSave(cmd, outputData) {
  var completeParts = yield* saveParts(cmd);
  if (completeParts) {
    var queueData = getSaveTask(cmd);
    yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
  }
  outputData.setStatus('ok');
  outputData.setData(cmd.getSaveKey());
}
function* commandSendMailMerge(cmd, outputData) {
  var completeParts = yield* saveParts(cmd);
  var isErr = false;
  if (completeParts) {
    isErr = true;
    var getRes = yield* docsCoServer.getCallback(cmd.getDocId());
    if (getRes) {
      var mailMergeSend = cmd.getMailMergeSend();
      mailMergeSend.setUrl(getRes.server.href);
      mailMergeSend.setBaseUrl(getRes.baseUrl);
      //меняем JsonKey и SaveKey, новый key нужет потому что за одну конвертацию делается часть, а json нужен всегда
      mailMergeSend.setJsonKey(cmd.getSaveKey());
      mailMergeSend.setRecordErrorCount(0);
      yield* addRandomKeyTaskCmd(cmd);
      var queueData = getSaveTask(cmd);
      yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
      isErr = false;
    }
  }
  if (isErr) {
    outputData.setStatus('err');
    outputData.setData(constants.UNKNOWN);
  } else {
    outputData.setStatus('ok');
    outputData.setData(cmd.getSaveKey());
  }
}
function* commandSfctByCmd(cmd) {
  yield* addRandomKeyTaskCmd(cmd);
  var queueData = getSaveTask(cmd);
  queueData.setFromChanges(true);
  yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
}
function* commandSfct(cmd, outputData) {
  yield* commandSfctByCmd(cmd);
  outputData.setStatus('ok');
}
function isDisplayedImage(strName) {
  var res = 0;
  if (strName) {
    //шаблон display[N]image.ext
    var findStr = constants.DISPLAY_PREFIX;
    var index = strName.indexOf(findStr);
    if (-1 != index) {
      if (index + findStr.length < strName.length) {
        var displayN = parseInt(strName[index + findStr.length]);
        if (1 <= displayN && displayN <= 6) {
          var imageIndex = index + findStr.length + 1;
          if (imageIndex == strName.indexOf("image", imageIndex))
            res = displayN;
        }
      }
    }
  }
  return res;
}
function* commandImgurls(conn, cmd, outputData) {
  var supportedFormats;
  var urls;
  var errorCode = constants.NO_ERROR;
  var isImgUrl = 'imgurl' == cmd.getCommand();
  if (isImgUrl) {
    urls = [cmd.getData()];
    supportedFormats = cfgTypesUpload || 'jpg';
  } else {
    urls = cmd.getData();
    supportedFormats = cfgTypesCopy || 'jpg';
  }
  //todo Promise.all()
  var displayedImageMap = {};//to make one imageIndex for ole object urls
  var imageCount = 0;
  var outputUrls = [];
  for (var i = 0; i < urls.length; ++i) {
    var urlSource = urls[i];
    var urlParsed;
    var data = undefined;
    if (urlSource.startsWith('data:')) {
      var delimiterIndex = urlSource.indexOf(',');
      if (-1 != delimiterIndex && (urlSource.length - (delimiterIndex + 1)) * 0.75 <= cfgImageSize) {
        data = new Buffer(urlSource.substring(delimiterIndex + 1), 'base64');
      }
    } else if(urlSource) {
      //todo stream
      data = yield utils.downloadUrlPromise(urlSource, cfgImageDownloadTimeout * 1000, cfgImageSize);
      urlParsed = urlModule.parse(urlSource);
    }
    var outputUrl = {url: 'error', path: 'error'};
    if (data) {
      var format = formatChecker.getImageFormat(data);
      var formatStr;
      if (constants.AVS_OFFICESTUDIO_FILE_UNKNOWN == format && urlParsed) {
        //bin, txt occur in ole object case
        var ext = pathModule.extname(urlParsed.pathname);
        if ('.bin' == ext || '.txt' == ext) {
          formatStr = ext.substring(1);
        }
      } else {
        formatStr = formatChecker.getStringFromFormat(format);
      }
      if (formatStr && -1 !== supportedFormats.indexOf(formatStr)) {
        var userid = cmd.getUserId();
        var imageIndex = cmd.getSaveIndex() + imageCount;
        imageCount++;
        var strLocalPath = 'media/' + utils.crc32(userid).toString(16) + '_';
        if (urlParsed) {
          var urlBasename = pathModule.basename(urlParsed.pathname);
          var displayN = isDisplayedImage(urlBasename);
          if (displayN > 0) {
            var displayedImageName = urlBasename.substring(0, urlBasename.length - formatStr.length - 1);
            var tempIndex = displayedImageMap[displayedImageName];
            if (null != tempIndex) {
              imageIndex = tempIndex;
              imageCount--;
            } else {
              displayedImageMap[displayedImageName] = imageIndex;
            }
            strLocalPath += constants.DISPLAY_PREFIX + displayN;
          }
        }
        strLocalPath += 'image' + imageIndex + '.' + formatStr;
        var strPath = cmd.getDocId() + '/' + strLocalPath;
        yield storage.putObject(strPath, data, data.length);
        var imgUrl = yield storage.getSignedUrl(conn.baseUrl, strPath);
        outputUrl = {url: imgUrl, path: strLocalPath};
      }
    }
    if (isImgUrl && ('error' === outputUrl.url || 'error' === outputUrl.path)) {
      errorCode = constants.UPLOAD_EXTENSION;
      break;
    }
    outputUrls.push(outputUrl);
  }
  if (constants.NO_ERROR !== errorCode) {
    outputData.setStatus('err');
    outputData.setData(errorCode);
  } else {
    outputData.setStatus('ok');
    outputData.setData(outputUrls);
  }
}
function* commandPathUrl(conn, cmd, outputData) {
  var contentDisposition = cmd.getInline() ? constants.CONTENT_DISPOSITION_INLINE :
    constants.CONTENT_DISPOSITION_ATTACHMENT;
  var strPath = cmd.getDocId() + '/' + cmd.getData();
  var url = yield storage.getSignedUrl(conn.baseUrl, strPath, null, cmd.getTitle(), contentDisposition);
  var errorCode = constants.NO_ERROR;
  if (constants.NO_ERROR !== errorCode) {
    outputData.setStatus('err');
    outputData.setData(errorCode);
  } else {
    outputData.setStatus('ok');
    outputData.setData(url);
  }
}
function* commandSaveFromOrigin(cmd, outputData) {
  yield* addRandomKeyTaskCmd(cmd);
  var queueData = getSaveTask(cmd);
  queueData.setFromOrigin(true);
  yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
  outputData.setStatus('ok');
  outputData.setData(cmd.getSaveKey());
}
function* commandSfcCallback(cmd, isSfcm) {
  var docId = cmd.getDocId();
  logger.debug('Start commandSfcCallback: docId = %s', docId);
  var saveKey = cmd.getSaveKey();
  var statusInfo = cmd.getStatusInfo();
  var isError = constants.NO_ERROR != statusInfo && constants.CONVERT_CORRUPTED != statusInfo;
  var savePathDoc = saveKey + '/' + cmd.getOutputPath();
  var savePathChanges = saveKey + '/changes.zip';
  var savePathHistory = saveKey + '/changesHistory.json';
  var getRes = yield* docsCoServer.getCallback(docId);
  if (getRes) {
    logger.debug('Callback commandSfcCallback: docId = %s callback = %s', docId, getRes.server.href);
    var outputSfc = new commonDefines.OutputSfcData();
    outputSfc.setKey(docId);
    var users = [];
    if (cmd.getUserId()) {
      users.push(cmd.getUserId());
    }
    outputSfc.setUsers(users);
    if (!isSfcm) {
      var actions = [];
      //use UserId case UserActionId miss in gc convertion
      var userActionId = cmd.getUserActionId() || cmd.getUserId();
      if (userActionId) {
        actions.push(new commonDefines.OutputAction(commonDefines.c_oAscUserAction.Out, userActionId));
      }
      outputSfc.setActions(actions);
    }
    outputSfc.setUserData(cmd.getUserData());
    if (!isError) {
      try {
        var data = yield storage.getObject(savePathHistory);
        outputSfc.setChangeHistory(data.toString('utf-8'));
        outputSfc.setUrl(yield storage.getSignedUrl(getRes.baseUrl, savePathDoc));
        outputSfc.setChangeUrl(yield storage.getSignedUrl(getRes.baseUrl, savePathChanges));
      } catch (e) {
        logger.error('Error commandSfcCallback: docId = %s\r\n%s', docId, e.stack);
      }
      if (outputSfc.getUrl() && outputSfc.getUsers().length > 0) {
        if (isSfcm) {
          outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MustSaveForce);
        } else {
          outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MustSave);
        }
      } else {
        isError = true;
      }
    }
    if (isError) {
      if (isSfcm) {
        outputSfc.setStatus(docsCoServer.c_oAscServerStatus.CorruptedForce);
      } else {
        outputSfc.setStatus(docsCoServer.c_oAscServerStatus.Corrupted);
      }
    }
    var uri = getRes.server.href;
    var postData = JSON.stringify(outputSfc);
    if (isSfcm) {
      var lastSave = cmd.getLastSave();
      if (lastSave && !isError) {
        yield* docsCoServer.setForceSave(docId, lastSave, savePathDoc);
      }
      try {
        yield* docsCoServer.sendServerRequest(docId, uri, postData);
      } catch (err) {
        logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack);
      }
    } else {
      //if anybody in document stop save
      var hasEditors = yield* docsCoServer.hasEditors(docId);
      logger.debug('hasEditors commandSfcCallback: docId = %s hasEditors = %d', docId, hasEditors);
      if (!hasEditors) {
        var updateMask = new taskResult.TaskResultData();
        updateMask.key = docId;
        updateMask.status = taskResult.FileStatus.SaveVersion;
        updateMask.statusInfo = cmd.getData();
        var updateIfTask = new taskResult.TaskResultData();
        updateIfTask.status = taskResult.FileStatus.UpdateVersion;
        updateIfTask.statusInfo = constants.NO_ERROR;
        var updateIfRes = yield taskResult.updateIf(updateIfTask, updateMask);
        if (updateIfRes.affectedRows > 0) {
          var replyStr = null;
          try {
            replyStr = yield* docsCoServer.sendServerRequest(docId, uri, postData);
          } catch (err) {
            replyStr = null;
            logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack);
          }
          var requestRes = false;
          var replyData = docsCoServer.parseReplyData(docId, replyStr);
          if (replyData && commonDefines.c_oAscServerCommandErrors.NoError == replyData.error) {
            //в случае comunity server придет запрос в CommandService проверяем результат
            var multi = redisClient.multi([
              ['get', redisKeySaved + docId],
              ['del', redisKeySaved + docId]
            ]);
            var execRes = yield utils.promiseRedis(multi, multi.exec);
            var savedVal = execRes[0];
            requestRes = (null == savedVal || '1' === savedVal);
          }
          if (requestRes) {
            yield docsCoServer.cleanDocumentOnExitPromise(docId, true);
          } else {
            var updateTask = new taskResult.TaskResultData();
            updateTask.key = docId;
            updateTask.status = taskResult.FileStatus.Ok;
            updateTask.statusInfo = constants.NO_ERROR;
            yield taskResult.update(updateTask);
          }
        }
      }
    }
  }
  if (docsCoServer.getIsShutdown() && !isSfcm) {
    yield utils.promiseRedis(redisClient, redisClient.srem, redisKeyShutdown, docId);
  }
  logger.debug('End commandSfcCallback: docId = %s', docId);
}
function* commandSendMMCallback(cmd) {
  var docId = cmd.getDocId();
  logger.debug('Start commandSendMMCallback: docId = %s', docId);
  var saveKey = cmd.getSaveKey();
  var statusInfo = cmd.getStatusInfo();
  var outputSfc = new commonDefines.OutputSfcData();
  outputSfc.setKey(docId);
  if (constants.NO_ERROR == statusInfo) {
    outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MailMerge);
  } else {
    outputSfc.setStatus(docsCoServer.c_oAscServerStatus.Corrupted);
  }
  var mailMergeSendData = cmd.getMailMergeSend();
  var outputMailMerge = new commonDefines.OutputMailMerge(mailMergeSendData);
  outputSfc.setMailMerge(outputMailMerge);
  outputSfc.setUsers([mailMergeSendData.getUserId()]);
  var data = yield storage.getObject(saveKey + '/' + cmd.getOutputPath());
  var xml = data.toString('utf8');
  var files = xml.match(/[< ]file.*?\/>/g);
  var recordRemain = (mailMergeSendData.getRecordTo() - mailMergeSendData.getRecordFrom() + 1);
  var recordIndexStart = mailMergeSendData.getRecordCount() - recordRemain;
  for (var i = 0; i < files.length; ++i) {
    var file = files[i];
    var fieldRes = /field=["'](.*?)["']/.exec(file);
    outputMailMerge.setTo(fieldRes[1]);
    outputMailMerge.setRecordIndex(recordIndexStart + i);
    var pathRes = /path=["'](.*?)["']/.exec(file);
    var signedUrl = yield storage.getSignedUrl(mailMergeSendData.getBaseUrl(), saveKey + '/' + pathRes[1]);
    outputSfc.setUrl(signedUrl);
    var uri = mailMergeSendData.getUrl();
    var replyStr = null;
    var postData = JSON.stringify(outputSfc);
    try {
      replyStr = yield* docsCoServer.sendServerRequest(docId, uri, postData);
    } catch (err) {
      replyStr = null;
      logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack);
    }
    var replyData = docsCoServer.parseReplyData(docId, replyStr);
    if (!(replyData && commonDefines.c_oAscServerCommandErrors.NoError == replyData.error)) {
      var recordErrorCount = mailMergeSendData.getRecordErrorCount();
      recordErrorCount++;
      outputMailMerge.setRecordErrorCount(recordErrorCount);
      mailMergeSendData.setRecordErrorCount(recordErrorCount);
    }
  }
  var newRecordFrom = mailMergeSendData.getRecordFrom() + Math.max(files.length, 1);
  if (newRecordFrom <= mailMergeSendData.getRecordTo()) {
    mailMergeSendData.setRecordFrom(newRecordFrom);
    yield* addRandomKeyTaskCmd(cmd);
    var queueData = getSaveTask(cmd);
    yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
  } else {
    logger.debug('End MailMerge: docId = %s', docId);
  }
  logger.debug('End commandSendMMCallback: docId = %s', docId);
}

exports.openDocument = function(conn, cmd, opt_upsertRes) {
  return co(function* () {
    var outputData;
    var docId = conn ? conn.docId : 'null';
    try {
      var startDate = null;
      if(clientStatsD) {
        startDate = new Date();
      }
      logger.debug('Start command: docId = %s %s', docId, JSON.stringify(cmd));
      outputData = new OutputData(cmd.getCommand());
      switch (cmd.getCommand()) {
        case 'open':
          //yield utils.sleep(5000);
          yield* commandOpen(conn, cmd, outputData, opt_upsertRes);
          break;
        case 'reopen':
          yield* commandReopen(cmd);
          break;
        case 'imgurl':
        case 'imgurls':
          yield* commandImgurls(conn, cmd, outputData);
          break;
        case 'pathurl':
          yield* commandPathUrl(conn, cmd, outputData);
          break;
        default:
          outputData.setStatus('err');
          outputData.setData(constants.UNKNOWN);
          break;
      }
      if(clientStatsD) {
        clientStatsD.timing('coauth.openDocument.' + cmd.getCommand(), new Date() - startDate);
      }
    }
    catch (e) {
      logger.error('Error openDocument: docId = %s\r\n%s', docId, e.stack);
      if (!outputData) {
        outputData = new OutputData();
      }
      outputData.setStatus('err');
      outputData.setData(constants.UNKNOWN);
    }
    finally {
      if (outputData && outputData.getStatus()) {
        logger.debug('Response command: docId = %s %s', docId, JSON.stringify(outputData));
        docsCoServer.sendData(conn, new OutputDataWrap('documentOpen', outputData));
      }
      logger.debug('End command: docId = %s', docId);
    }
  });
};
exports.downloadAs = function(req, res) {
  return co(function* () {
    var docId = 'null';
    try {
      var startDate = null;
      if(clientStatsD) {
        startDate = new Date();
      }
      var strCmd = req.query['cmd'];
      var cmd = new commonDefines.InputCommand(JSON.parse(strCmd));
      docId = cmd.getDocId();
      logger.debug('Start downloadAs: docId = %s %s', docId, strCmd);
      cmd.setData(req.body);
      var outputData = new OutputData(cmd.getCommand());
      switch (cmd.getCommand()) {
        case 'save':
          yield* commandSave(cmd, outputData);
          break;
        case 'savefromorigin':
          yield* commandSaveFromOrigin(cmd, outputData);
          break;
        case 'sendmm':
          yield* commandSendMailMerge(cmd, outputData);
          break;
        case 'sfct':
          yield* commandSfct(cmd, outputData);
          break;
        default:
          outputData.setStatus('err');
          outputData.setData(constants.UNKNOWN);
          break;
      }
      var strRes = JSON.stringify(outputData);
      res.send(strRes);
      logger.debug('End downloadAs: docId = %s %s', docId, strRes);
      if(clientStatsD) {
        clientStatsD.timing('coauth.downloadAs.' + cmd.getCommand(), new Date() - startDate);
      }
    }
    catch (e) {
      logger.error('Error downloadAs: docId = %s\r\n%s', docId, e.stack);
      res.sendStatus(400);
    }
  });
};
exports.saveFromChanges = function(docId, statusInfo, optFormat, opt_userId, opt_queue) {
  return co(function* () {
    try {
      var startDate = null;
      if(clientStatsD) {
        startDate = new Date();
      }
      logger.debug('Start saveFromChanges: docId = %s', docId);
      var task = new taskResult.TaskResultData();
      task.key = docId;
      //делаем select, потому что за время timeout информация могла измениться
      var selectRes = yield taskResult.select(docId);
      var row = selectRes.length > 0 ? selectRes[0] : null;
      if (row && row.status == taskResult.FileStatus.SaveVersion && row.status_info == statusInfo) {
        if (null == optFormat) {
          optFormat = constants.AVS_OFFICESTUDIO_FILE_OTHER_TEAMLAB_INNER;
        }
        var cmd = new commonDefines.InputCommand();
        cmd.setCommand('sfc');
        cmd.setDocId(docId);
        cmd.setOutputFormat(optFormat);
        cmd.setData(statusInfo);
        cmd.setUserActionId(opt_userId);
        yield* addRandomKeyTaskCmd(cmd);
        var queueData = getSaveTask(cmd);
        queueData.setFromChanges(true);
        yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_NORMAL, opt_queue);
        if (docsCoServer.getIsShutdown()) {
          yield utils.promiseRedis(redisClient, redisClient.sadd, redisKeyShutdown, docId);
        }
        logger.debug('AddTask saveFromChanges: docId = %s', docId);
      } else {
        if (row) {
          logger.debug('saveFromChanges status mismatch: docId = %s; row: %d; %d; expected: %d', docId, row.status, row.status_info, statusInfo);
        }
      }
      if (clientStatsD) {
        clientStatsD.timing('coauth.saveFromChanges', new Date() - startDate);
      }
    }
    catch (e) {
      logger.error('Error saveFromChanges: docId = %s\r\n%s', docId, e.stack);
    }
  });
};
exports.receiveTask = function(data, dataRaw) {
  return co(function* () {
    var docId = 'null';
    try {
      var task = new commonDefines.TaskQueueData(JSON.parse(data));
      if (task) {
        var cmd = task.getCmd();
        docId = cmd.getDocId();
        logger.debug('Start receiveTask: docId = %s %s', docId, data);
        var updateTask = getUpdateResponse(cmd);
        var updateRes = yield taskResult.update(updateTask);
        if (updateRes.affectedRows > 0) {
          var outputData = new OutputData(cmd.getCommand());
          var command = cmd.getCommand();
          var additionalOutput = {needUrlKey: null, needUrlMethod: null};
          if ('open' == command || 'reopen' == command) {
            //yield utils.sleep(5000);
            yield* getOutputData(cmd, outputData, cmd.getDocId(), updateTask.status,
              updateTask.statusInfo, null, additionalOutput);
          } else if ('save' == command || 'savefromorigin' == command || 'sfct' == command) {
            yield* getOutputData(cmd, outputData, cmd.getSaveKey(), updateTask.status,
              updateTask.statusInfo, null, additionalOutput);
          } else if ('sfcm' == command) {
            yield* commandSfcCallback(cmd, true);
          } else if ('sfc' == command) {
            yield* commandSfcCallback(cmd, false);
          } else if ('sendmm' == command) {
            yield* commandSendMMCallback(cmd);
          } else if ('conv' == command) {
            //nothing
          }
          if (outputData.getStatus()) {
            logger.debug('Send receiveTask: docId = %s %s', docId, JSON.stringify(outputData));
            var output = new OutputDataWrap('documentOpen', outputData);
            yield* docsCoServer.publish({
              type: commonDefines.c_oPublishType.receiveTask, cmd: cmd, output: output,
              needUrlKey: additionalOutput.needUrlKey, needUrlMethod: additionalOutput.needUrlMethod
            });
          }
        }
        yield* docsCoServer.removeResponse(dataRaw);
        logger.debug('End receiveTask: docId = %s', docId);
      }
    } catch (err) {
      logger.debug('Error receiveTask: docId = %s\r\n%s', docId, err.stack);
    }
  });
};

exports.cleanupCache = cleanupCache;
exports.commandSfctByCmd = commandSfctByCmd;
exports.commandOpenStartPromise = commandOpenStartPromise;
exports.OutputDataWrap = OutputDataWrap;
exports.OutputData = OutputData;
\ No newline at end of file

From 2a856916aa42bf81abecc4a3029f596ea5c70ece Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Wed, 3 Aug 2016 12:02:18 +0300
Subject: [PATCH 12/34] error PRECONDITION_FAILED - unknown delivery tag

---
 Common/sources/taskqueueRabbitMQ.js | 26 ++++----------------------
 1 file changed, 4 insertions(+), 22 deletions(-)

diff --git a/Common/sources/taskqueueRabbitMQ.js b/Common/sources/taskqueueRabbitMQ.js
index 6492fa47..bd636ac9 100644
--- a/Common/sources/taskqueueRabbitMQ.js
+++ b/Common/sources/taskqueueRabbitMQ.js
@@ -119,24 +119,14 @@ function clear(taskqueue) {
   taskqueue.channelConvertResponseReceive = null;
 }
 function repeat(taskqueue) {
-  var i;
-  for (i = 0; i < taskqueue.addTaskStore.length; ++i) {
+  //repeat addTask because they are lost after the reconnection
+  //unlike unconfirmed task will come again
+  //acknowledge data after reconnect raises an exception 'PRECONDITION_FAILED - unknown delivery tag'
+  for (var i = 0; i < taskqueue.addTaskStore.length; ++i) {
     var elem = taskqueue.addTaskStore[i];
     addTask(taskqueue, elem.task, elem.priority, function () {});
   }
-  for (i = 0; i < taskqueue.addResponseStore.length; ++i) {
-    addResponse(taskqueue, taskqueue.addResponseStore[i], function () {});
-  }
-  for (i = 0; i < taskqueue.removeTaskStore.length; ++i) {
-    removeTask(taskqueue, taskqueue.removeTaskStore[i]);
-  }
-  for (i = 0; i < taskqueue.removeResponseStore.length; ++i) {
-    removeResponse(taskqueue, taskqueue.removeResponseStore[i]);
-  }
   taskqueue.addTaskStore.length = 0;
-  taskqueue.addResponseStore.length = 0;
-  taskqueue.removeTaskStore.length = 0;
-  taskqueue.removeResponseStore.length = 0;
 }
 function addTask(taskqueue, content, priority, callback) {
   var options = {persistent: true, priority: priority};
@@ -161,9 +151,6 @@ function TaskQueueRabbitMQ() {
   this.channelConvertResponse = null;
   this.channelConvertResponseReceive = null;
   this.addTaskStore = [];
-  this.addResponseStore = [];
-  this.removeTaskStore = [];
-  this.removeResponseStore = [];
 }
 util.inherits(TaskQueueRabbitMQ, events.EventEmitter);
 TaskQueueRabbitMQ.prototype.init = function (isAddTask, isAddResponse, isAddTaskReceive, isAddResponseReceive, callback) {
@@ -214,7 +201,6 @@ TaskQueueRabbitMQ.prototype.addResponse = function (task) {
         }
       });
     } else {
-      t.addResponseStore.push(content);
       resolve();
     }
   });
@@ -224,8 +210,6 @@ TaskQueueRabbitMQ.prototype.removeTask = function (data) {
   return new Promise(function (resolve, reject) {
     if (null != t.channelConvertTaskReceive) {
       removeTask(t, data);
-    } else {
-      t.removeTaskStore.push(data);
     }
     resolve();
   });
@@ -235,8 +219,6 @@ TaskQueueRabbitMQ.prototype.removeResponse = function (data) {
   return new Promise(function (resolve, reject) {
     if (null != t.channelConvertResponseReceive) {
       removeResponse(t, data);
-    } else {
-      t.removeResponseStore.push(data);
     }
     resolve();
   });

From d3de2411bc0bc670c2949e675366bfbe7041e049 Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Wed, 3 Aug 2016 12:46:55 +0300
Subject: [PATCH 13/34] error PRECONDITION_FAILED - unknown delivery tag

---
 DocService/sources/pubsubRabbitMQ.js | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/DocService/sources/pubsubRabbitMQ.js b/DocService/sources/pubsubRabbitMQ.js
index 7789ce02..4653260b 100644
--- a/DocService/sources/pubsubRabbitMQ.js
+++ b/DocService/sources/pubsubRabbitMQ.js
@@ -54,14 +54,16 @@ function init(pubsub, callback) {
       pubsub.exchangePublish = yield rabbitMQCore.assertExchangePromise(pubsub.channelPublish, cfgRabbitExchangePubSub,
         'fanout', {durable: true});
 
-      var channelReceive = yield rabbitMQCore.createChannelPromise(conn);
-      var queue = yield rabbitMQCore.assertQueuePromise(channelReceive, '', {autoDelete: true, exclusive: true});
-      channelReceive.bindQueue(queue, cfgRabbitExchangePubSub, '');
-      yield rabbitMQCore.consumePromise(channelReceive, queue, function (message) {
-        if (message) {
-          pubsub.emit('message', message.content.toString());
+      pubsub.channelReceive = yield rabbitMQCore.createChannelPromise(conn);
+      var queue = yield rabbitMQCore.assertQueuePromise(pubsub.channelReceive, '', {autoDelete: true, exclusive: true});
+      pubsub.channelReceive.bindQueue(queue, cfgRabbitExchangePubSub, '');
+      yield rabbitMQCore.consumePromise(pubsub.channelReceive, queue, function (message) {
+        if(null != pubsub.channelReceive){
+          if (message) {
+            pubsub.emit('message', message.content.toString());
+          }
+          pubsub.channelReceive.ack(message);
         }
-        channelReceive.ack(message);
       }, {noAck: false});
       //process messages received while reconnection time
       repeat(pubsub);
@@ -76,6 +78,7 @@ function init(pubsub, callback) {
 function clear(pubsub) {
   pubsub.channelPublish = null;
   pubsub.exchangePublish = null;
+  pubsub.channelReceive = null;
 }
 function repeat(pubsub) {
   for (var i = 0; i < pubsub.publishStore.length; ++i) {
@@ -92,6 +95,7 @@ function PubsubRabbitMQ() {
   this.connection = null;
   this.channelPublish = null;
   this.exchangePublish = null;
+  this.channelReceive = null;
   this.publishStore = [];
 }
 util.inherits(PubsubRabbitMQ, events.EventEmitter);

From 79c2e86643316ea7f418daccfb00221aa436526b Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Thu, 4 Aug 2016 13:43:37 +0300
Subject: [PATCH 14/34] http DELETE works like GET on static content

---
 DocService/sources/server.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/DocService/sources/server.js b/DocService/sources/server.js
index 2fbc82ce..98c0f4f6 100644
--- a/DocService/sources/server.js
+++ b/DocService/sources/server.js
@@ -132,7 +132,7 @@ if (cluster.isMaster) {
     var cfgStorageFolderName = configStorage.get('storageFolderName');
     app.use('/' + cfgBucketName + '/' + cfgStorageFolderName, (req, res, next) => {
       var index = req.url.lastIndexOf('/');
-      if (-1 != index) {
+      if ('GET' === req.method && -1 != index) {
         var contentDisposition = req.query['disposition'] || 'attachment';
         var sendFileOptions = {
           root: configStorage.get('fs.folderPath'),
@@ -154,7 +154,7 @@ if (cluster.isMaster) {
           }
         });
       } else {
-        req.sendStatus(404)
+        res.sendStatus(404)
       }
     });
   }

From 63352502f03324661c556465de0c3dc2466c9f1a Mon Sep 17 00:00:00 2001
From: "Alexander.Trofimov" <Alexander.Trofimov@onlyoffice.com>
Date: Thu, 4 Aug 2016 15:12:14 +0300
Subject: [PATCH 15/34] update version packages

---
 Common/package.json        | 10 +++++-----
 DocService/package.json    | 14 +++++++-------
 FileConverter/package.json |  2 +-
 SpellChecker/package.json  |  8 ++++----
 4 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/Common/package.json b/Common/package.json
index 0da38fc9..c979eb1f 100644
--- a/Common/package.json
+++ b/Common/package.json
@@ -5,15 +5,15 @@
   "private": true,
   "dependencies": {
     "amazon-s3-url-signer": "https://github.com/agolybev/amazon-s3-url-signer/archive/v0.0.9.tar.gz",
-    "amqplib": "^0.4.0",
-    "aws-sdk": "^2.2.33",
+    "amqplib": "^0.4.1",
+    "aws-sdk": "^2.4.12",
     "co": "^4.6.0",
-    "config": "^1.19.0",
-    "log4js": "^0.6.31",
+    "config": "^1.21.0",
+    "log4js": "^0.6.38",
     "mime": "^1.3.4",
     "mkdirp": "^0.5.1",
     "node-statsd": "^0.1.1",
-    "request": "^2.69.0",
+    "request": "^2.74.0",
     "uri-js": "^2.1.1"
   }
 }
diff --git a/DocService/package.json b/DocService/package.json
index 4745e9e5..e3809e17 100644
--- a/DocService/package.json
+++ b/DocService/package.json
@@ -5,17 +5,17 @@
   "private": true,
   "dependencies": {
     "base64-stream": "^0.1.3",
-    "body-parser": "^1.14.2",
+    "body-parser": "^1.15.2",
     "co": "^4.6.0",
-    "config": "^1.19.0",
+    "config": "^1.21.0",
     "cron": "^1.1.0",
-    "express": "^4.13.4",
-    "fakeredis": "^1.0.2",
+    "express": "^4.14.0",
+    "fakeredis": "^1.0.3",
     "mime": "^1.3.4",
     "multiparty": "^4.1.2",
-    "mysql": "^2.10.2",
-    "pg": "^4.4.4",
-    "redis": "^2.4.2",
+    "mysql": "^2.11.1",
+    "pg": "^6.0.3",
+    "redis": "^2.6.2",
     "sockjs": "http://d2ettrnqo7v976.cloudfront.net/npm/sockjs-node/v0.3.15.14.tar.gz",
     "underscore": "^1.8.3"
   }
diff --git a/FileConverter/package.json b/FileConverter/package.json
index 5ff37590..e21d1b77 100644
--- a/FileConverter/package.json
+++ b/FileConverter/package.json
@@ -5,6 +5,6 @@
   "private": true,
   "dependencies": {
     "co": "^4.6.0",
-    "config": "^1.19.0"
+    "config": "^1.21.0"
   }
 }
diff --git a/SpellChecker/package.json b/SpellChecker/package.json
index 6fbeb178..ed3e970b 100644
--- a/SpellChecker/package.json
+++ b/SpellChecker/package.json
@@ -4,9 +4,9 @@
   "homepage": "http://www.onlyoffice.com",
   "private": true,
   "dependencies": {
-	"express"	: "^4.13.4",
-	"sockjs"	: "^0.3.15",
-	"nodehun"	: "^2.0.8",
-	"config":	"^1.19.0"
+	"express"	: "^4.14.0",
+	"sockjs"	: "^0.3.17",
+	"nodehun"	: "^2.0.10",
+	"config"	: "^1.21.0"
     }
 }
\ No newline at end of file

From 8ff93cbfb8d9c634bb90e900661a16d155ecd84a Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Wed, 10 Aug 2016 11:29:31 +0300
Subject: [PATCH 16/34] both CONVERT_DRM,CONVERT_PASSWORD errors produce
 needpassword, remove error CONVERT_MS_OFFCRYPTO

---
 Common/sources/constants.js            | 1 -
 DocService/sources/canvasservice.js    | 2 +-
 DocService/sources/converterservice.js | 2 +-
 FileConverter/sources/converter.js     | 5 ++---
 4 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/Common/sources/constants.js b/Common/sources/constants.js
index 3005934e..ed2a6de7 100644
--- a/Common/sources/constants.js
+++ b/Common/sources/constants.js
@@ -127,7 +127,6 @@ exports.CONVERT_DOWNLOAD = -81;
 exports.CONVERT_UNKNOWN_FORMAT = -82;
 exports.CONVERT_TIMEOUT = -83;
 exports.CONVERT_READ_FILE = -84;
-exports.CONVERT_MS_OFFCRYPTO = -85;
 exports.CONVERT_CORRUPTED = -86;
 exports.CONVERT_LIBREOFFICE = -87;
 exports.CONVERT_PARAMS = -88;
diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js
index 25124945..9e521779 100644
--- a/DocService/sources/canvasservice.js
+++ b/DocService/sources/canvasservice.js
@@ -1 +1 @@
-/*
 * (c) Copyright Ascensio System SIA 2010-2016
 *
 * This program is a free software product. You can redistribute it and/or
 * modify it under the terms of the GNU Affero General Public License (AGPL)
 * version 3 as published by the Free Software Foundation. In accordance with
 * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect
 * that Ascensio System SIA expressly excludes the warranty of non-infringement
 * of any third-party rights.
 *
 * This program is distributed WITHOUT ANY WARRANTY; without even the implied
 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE. For
 * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
 *
 * You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia,
 * EU, LV-1021.
 *
 * The  interactive user interfaces in modified source and object code versions
 * of the Program must display Appropriate Legal Notices, as required under
 * Section 5 of the GNU AGPL version 3.
 *
 * Pursuant to Section 7(b) of the License you must retain the original Product
 * logo when distributing the program. Pursuant to Section 7(e) we decline to
 * grant you any rights under trademark law for use of our trademarks.
 *
 * All the Product's GUI elements, including illustrations and icon sets, as
 * well as technical writing content are licensed under the terms of the
 * Creative Commons Attribution-ShareAlike 4.0 International. See the License
 * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
 *
 */

var pathModule = require('path');
var urlModule = require('url');
var co = require('co');
var sqlBase = require('./baseConnector');
var docsCoServer = require('./DocsCoServer');
var taskResult = require('./taskresult');
var logger = require('./../../Common/sources/logger');
var utils = require('./../../Common/sources/utils');
var constants = require('./../../Common/sources/constants');
var commonDefines = require('./../../Common/sources/commondefines');
var storage = require('./../../Common/sources/storage-base');
var formatChecker = require('./../../Common/sources/formatchecker');
var statsDClient = require('./../../Common/sources/statsdclient');
var config = require('config');
var config_server = config.get('services.CoAuthoring.server');
var config_utils = config.get('services.CoAuthoring.utils');
var pubsubRedis = require('./pubsubRedis');


var cfgTypesUpload = config_utils.get('limits_image_types_upload');
var cfgTypesCopy = config_utils.get('limits_image_types_copy');
var cfgImageSize = config_server.get('limits_image_size');
var cfgImageDownloadTimeout = config_server.get('limits_image_download_timeout');
var cfgRedisPrefix = config.get('services.CoAuthoring.redis.prefix');

var SAVE_TYPE_PART_START = 0;
var SAVE_TYPE_PART = 1;
var SAVE_TYPE_COMPLETE = 2;
var SAVE_TYPE_COMPLETE_ALL = 3;

var clientStatsD = statsDClient.getClient();
var redisClient = pubsubRedis.getClientRedis();
var redisKeySaved = cfgRedisPrefix + constants.REDIS_KEY_SAVED;
var redisKeyShutdown = cfgRedisPrefix + constants.REDIS_KEY_SHUTDOWN;

function OutputDataWrap(type, data) {
  this['type'] = type;
  this['data'] = data;
}
OutputDataWrap.prototype = {
  fromObject: function(data) {
    this['type'] = data['type'];
    this['data'] = new OutputData();
    this['data'].fromObject(data['data']);
  },
  getType: function() {
    return this['type'];
  },
  setType: function(data) {
    this['type'] = data;
  },
  getData: function() {
    return this['data'];
  },
  setData: function(data) {
    this['data'] = data;
  }
};
function OutputData(type) {
  this['type'] = type;
  this['status'] = undefined;
  this['data'] = undefined;
}
OutputData.prototype = {
  fromObject: function(data) {
    this['type'] = data['type'];
    this['status'] = data['status'];
    this['data'] = data['data'];
  },
  getType: function() {
    return this['type'];
  },
  setType: function(data) {
    this['type'] = data;
  },
  getStatus: function() {
    return this['status'];
  },
  setStatus: function(data) {
    this['status'] = data;
  },
  getData: function() {
    return this['data'];
  },
  setData: function(data) {
    this['data'] = data;
  }
};

function* getOutputData(cmd, outputData, key, status, statusInfo, optConn, optAdditionalOutput) {
  var docId = cmd.getDocId();
  switch (status) {
    case taskResult.FileStatus.SaveVersion:
    case taskResult.FileStatus.UpdateVersion:
    case taskResult.FileStatus.Ok:
      if(taskResult.FileStatus.Ok == status) {
        outputData.setStatus('ok');
      } else if(taskResult.FileStatus.SaveVersion == status) {
        if (optConn && optConn.user.view) {
          outputData.setStatus('updateversion');
        } else {
          var updateMask = new taskResult.TaskResultData();
          updateMask.key = docId;
          updateMask.status = status;
          updateMask.statusInfo = statusInfo;
          var updateTask = new taskResult.TaskResultData();
          updateTask.status = taskResult.FileStatus.Ok;
          updateTask.statusInfo = constants.NO_ERROR;
          var updateIfRes = yield taskResult.updateIf(updateTask, updateMask);
          if (updateIfRes.affectedRows > 0) {
            outputData.setStatus('ok');
          } else {
            outputData.setStatus('updateversion');
          }
        }
      } else {
        outputData.setStatus('updateversion');
      }
      var command = cmd.getCommand();
      if ('open' != command && 'reopen' != command) {
        var strPath = key + '/' + cmd.getOutputPath();
        if (optConn) {
          var contentDisposition = cmd.getInline() ? constants.CONTENT_DISPOSITION_INLINE : constants.CONTENT_DISPOSITION_ATTACHMENT;
          outputData.setData(yield storage.getSignedUrl(optConn.baseUrl, strPath, null, cmd.getTitle(), contentDisposition));
        } else if (optAdditionalOutput) {
          optAdditionalOutput.needUrlKey = strPath;
          optAdditionalOutput.needUrlMethod = 2;
        }
      } else {
        if (optConn) {
          outputData.setData(yield storage.getSignedUrls(optConn.baseUrl, key));
        } else if (optAdditionalOutput) {
          optAdditionalOutput.needUrlKey = key;
          optAdditionalOutput.needUrlMethod = 0;
        }
      }
      break;
    case taskResult.FileStatus.NeedParams:
      outputData.setStatus('needparams');
      var settingsPath = key + '/' + 'settings.json';
      if (optConn) {
        outputData.setData(yield storage.getSignedUrl(optConn.baseUrl, settingsPath));
      } else if (optAdditionalOutput) {
        optAdditionalOutput.needUrlKey = settingsPath;
        optAdditionalOutput.needUrlMethod = 1;
      }
      break;
    case taskResult.FileStatus.NeedPassword:
      outputData.setStatus('needpassword');
      break;
    case taskResult.FileStatus.Err:
    case taskResult.FileStatus.ErrToReload:
      outputData.setStatus('err');
      outputData.setData(statusInfo);
      if (taskResult.FileStatus.ErrToReload == status) {
        yield cleanupCache(key);
      }
      break;
  }
}
function* addRandomKeyTaskCmd(cmd) {
  var task = yield* taskResult.addRandomKeyTask(cmd.getDocId());
  cmd.setSaveKey(task.key);
}
function* saveParts(cmd) {
  var result = false;
  var saveType = cmd.getSaveType();
  var filename;
  if (SAVE_TYPE_COMPLETE_ALL === saveType) {
    filename = 'Editor.bin';
  } else {
    filename = 'Editor' + (cmd.getSaveIndex() || '') + '.bin';
  }
  if (SAVE_TYPE_PART_START === saveType || SAVE_TYPE_COMPLETE_ALL === saveType) {
    yield* addRandomKeyTaskCmd(cmd);
  }
  if (cmd.getUrl()) {
    result = true;
  } else {
    var buffer = cmd.getData();
    yield storage.putObject(cmd.getSaveKey() + '/' + filename, buffer, buffer.length);
    //delete data to prevent serialize into json
    cmd.data = null;
    result = (SAVE_TYPE_COMPLETE_ALL === saveType || SAVE_TYPE_COMPLETE === saveType);
  }
  return result;
}
function getSaveTask(cmd) {
  cmd.setData(null);
  var queueData = new commonDefines.TaskQueueData();
  queueData.setCmd(cmd);
  queueData.setToFile(constants.OUTPUT_NAME + '.' + formatChecker.getStringFromFormat(cmd.getOutputFormat()));
  //todo paid
  //if (cmd.vkey) {
  //  bool
  //  bPaid;
  //  Signature.getVKeyParams(cmd.vkey, out bPaid);
  //  oTaskQueueData.m_bPaid = bPaid;
  //}
  return queueData;
}
function getUpdateResponse(cmd) {
  var updateTask = new taskResult.TaskResultData();
  updateTask.key = cmd.getSaveKey() ? cmd.getSaveKey() : cmd.getDocId();
  var statusInfo = cmd.getStatusInfo();
  if (constants.NO_ERROR == statusInfo) {
    updateTask.status = taskResult.FileStatus.Ok;
  } else if (constants.CONVERT_DOWNLOAD == statusInfo) {
    updateTask.status = taskResult.FileStatus.ErrToReload;
  } else if (constants.CONVERT_NEED_PARAMS == statusInfo) {
    updateTask.status = taskResult.FileStatus.NeedParams;
  } else if (constants.CONVERT_DRM == statusInfo) {
    updateTask.status = taskResult.FileStatus.NeedPassword;
  } else {
    updateTask.status = taskResult.FileStatus.Err;
  }
  updateTask.statusInfo = statusInfo;
  if (cmd.getTitle()) {
    updateTask.title = cmd.getTitle();
  }
  return updateTask;
}
var cleanupCache = co.wrap(function* (docId) {
  //todo redis ?
  var res = false;
  var removeRes = yield taskResult.remove(docId);
  if (removeRes.affectedRows > 0) {
    yield storage.deletePath(docId);
    res = true;
  }
  return res;
});

function commandOpenStartPromise(docId, cmd, opt_updateUserIndex) {
  var task = new taskResult.TaskResultData();
  task.key = docId;
  task.status = taskResult.FileStatus.WaitQueue;
  task.statusInfo = constants.NO_ERROR;
  if (cmd) {
    task.title = cmd.getTitle();
  } else {
    logger.warn("commandOpenStartPromise empty cmd: docId = %s", docId);
  }

  return taskResult.upsert(task, opt_updateUserIndex);
}
function* commandOpen(conn, cmd, outputData, opt_upsertRes) {
  var upsertRes;
  if (opt_upsertRes) {
    upsertRes = opt_upsertRes;
  } else {
    upsertRes = yield commandOpenStartPromise(cmd.getDocId(), cmd);
  }
  //if CLIENT_FOUND_ROWS don't specify 1 row is inserted , 2 row is updated, and 0 row is set to its current values
  //http://dev.mysql.com/doc/refman/5.7/en/insert-on-duplicate.html
  var bCreate = upsertRes.affectedRows == 1;
  if (!bCreate) {
    var selectRes = yield taskResult.select(cmd.getDocId());
    if (selectRes.length > 0) {
      var row = selectRes[0];
      yield* getOutputData(cmd, outputData, cmd.getDocId(), row.status, row.status_info, conn);
    }
  } else {
    //add task
    cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CANVAS);
    cmd.setEmbeddedFonts(false);
    var dataQueue = new commonDefines.TaskQueueData();
    dataQueue.setCmd(cmd);
    dataQueue.setToFile('Editor.bin');
    var priority = constants.QUEUE_PRIORITY_HIGH;
    var formatIn = formatChecker.getFormatFromString(cmd.getFormat());
    //decrease pdf, djvu, xps convert priority becase long open time
    if (constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === formatIn || constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_DJVU === formatIn ||
      constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_XPS === formatIn) {
      priority = constants.QUEUE_PRIORITY_LOW;
    }
    yield* docsCoServer.addTask(dataQueue, priority);
  }
}
function* commandReopen(cmd) {
  var task = new taskResult.TaskResultData();
  task.key = cmd.getDocId();
  task.status = taskResult.FileStatus.WaitQueue;
  task.statusInfo = constants.NO_ERROR;

  var upsertRes = yield taskResult.update(task);
  if (upsertRes.affectedRows > 0) {
    //add task
    cmd.setUrl(null);//url may expire
    cmd.setSaveKey(cmd.getDocId());
    cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CANVAS);
    cmd.setEmbeddedFonts(false);
    var dataQueue = new commonDefines.TaskQueueData();
    dataQueue.setCmd(cmd);
    dataQueue.setToFile('Editor.bin');
    dataQueue.setFromSettings(true);
    yield* docsCoServer.addTask(dataQueue, constants.QUEUE_PRIORITY_HIGH);
  }
}
function* commandSave(cmd, outputData) {
  var completeParts = yield* saveParts(cmd);
  if (completeParts) {
    var queueData = getSaveTask(cmd);
    yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
  }
  outputData.setStatus('ok');
  outputData.setData(cmd.getSaveKey());
}
function* commandSendMailMerge(cmd, outputData) {
  var completeParts = yield* saveParts(cmd);
  var isErr = false;
  if (completeParts) {
    isErr = true;
    var getRes = yield* docsCoServer.getCallback(cmd.getDocId());
    if (getRes) {
      var mailMergeSend = cmd.getMailMergeSend();
      mailMergeSend.setUrl(getRes.server.href);
      mailMergeSend.setBaseUrl(getRes.baseUrl);
      //меняем JsonKey и SaveKey, новый key нужет потому что за одну конвертацию делается часть, а json нужен всегда
      mailMergeSend.setJsonKey(cmd.getSaveKey());
      mailMergeSend.setRecordErrorCount(0);
      yield* addRandomKeyTaskCmd(cmd);
      var queueData = getSaveTask(cmd);
      yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
      isErr = false;
    }
  }
  if (isErr) {
    outputData.setStatus('err');
    outputData.setData(constants.UNKNOWN);
  } else {
    outputData.setStatus('ok');
    outputData.setData(cmd.getSaveKey());
  }
}
function* commandSfctByCmd(cmd) {
  yield* addRandomKeyTaskCmd(cmd);
  var queueData = getSaveTask(cmd);
  queueData.setFromChanges(true);
  yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
}
function* commandSfct(cmd, outputData) {
  yield* commandSfctByCmd(cmd);
  outputData.setStatus('ok');
}
function isDisplayedImage(strName) {
  var res = 0;
  if (strName) {
    //шаблон display[N]image.ext
    var findStr = constants.DISPLAY_PREFIX;
    var index = strName.indexOf(findStr);
    if (-1 != index) {
      if (index + findStr.length < strName.length) {
        var displayN = parseInt(strName[index + findStr.length]);
        if (1 <= displayN && displayN <= 6) {
          var imageIndex = index + findStr.length + 1;
          if (imageIndex == strName.indexOf("image", imageIndex))
            res = displayN;
        }
      }
    }
  }
  return res;
}
function* commandImgurls(conn, cmd, outputData) {
  var supportedFormats;
  var urls;
  var errorCode = constants.NO_ERROR;
  var isImgUrl = 'imgurl' == cmd.getCommand();
  if (isImgUrl) {
    urls = [cmd.getData()];
    supportedFormats = cfgTypesUpload || 'jpg';
  } else {
    urls = cmd.getData();
    supportedFormats = cfgTypesCopy || 'jpg';
  }
  //todo Promise.all()
  var displayedImageMap = {};//to make one imageIndex for ole object urls
  var imageCount = 0;
  var outputUrls = [];
  for (var i = 0; i < urls.length; ++i) {
    var urlSource = urls[i];
    var urlParsed;
    var data = undefined;
    if (urlSource.startsWith('data:')) {
      var delimiterIndex = urlSource.indexOf(',');
      if (-1 != delimiterIndex && (urlSource.length - (delimiterIndex + 1)) * 0.75 <= cfgImageSize) {
        data = new Buffer(urlSource.substring(delimiterIndex + 1), 'base64');
      }
    } else if(urlSource) {
      //todo stream
      data = yield utils.downloadUrlPromise(urlSource, cfgImageDownloadTimeout * 1000, cfgImageSize);
      urlParsed = urlModule.parse(urlSource);
    }
    var outputUrl = {url: 'error', path: 'error'};
    if (data) {
      var format = formatChecker.getImageFormat(data);
      var formatStr;
      if (constants.AVS_OFFICESTUDIO_FILE_UNKNOWN == format && urlParsed) {
        //bin, txt occur in ole object case
        var ext = pathModule.extname(urlParsed.pathname);
        if ('.bin' == ext || '.txt' == ext) {
          formatStr = ext.substring(1);
        }
      } else {
        formatStr = formatChecker.getStringFromFormat(format);
      }
      if (formatStr && -1 !== supportedFormats.indexOf(formatStr)) {
        var userid = cmd.getUserId();
        var imageIndex = cmd.getSaveIndex() + imageCount;
        imageCount++;
        var strLocalPath = 'media/' + utils.crc32(userid).toString(16) + '_';
        if (urlParsed) {
          var urlBasename = pathModule.basename(urlParsed.pathname);
          var displayN = isDisplayedImage(urlBasename);
          if (displayN > 0) {
            var displayedImageName = urlBasename.substring(0, urlBasename.length - formatStr.length - 1);
            var tempIndex = displayedImageMap[displayedImageName];
            if (null != tempIndex) {
              imageIndex = tempIndex;
              imageCount--;
            } else {
              displayedImageMap[displayedImageName] = imageIndex;
            }
            strLocalPath += constants.DISPLAY_PREFIX + displayN;
          }
        }
        strLocalPath += 'image' + imageIndex + '.' + formatStr;
        var strPath = cmd.getDocId() + '/' + strLocalPath;
        yield storage.putObject(strPath, data, data.length);
        var imgUrl = yield storage.getSignedUrl(conn.baseUrl, strPath);
        outputUrl = {url: imgUrl, path: strLocalPath};
      }
    }
    if (isImgUrl && ('error' === outputUrl.url || 'error' === outputUrl.path)) {
      errorCode = constants.UPLOAD_EXTENSION;
      break;
    }
    outputUrls.push(outputUrl);
  }
  if (constants.NO_ERROR !== errorCode) {
    outputData.setStatus('err');
    outputData.setData(errorCode);
  } else {
    outputData.setStatus('ok');
    outputData.setData(outputUrls);
  }
}
function* commandPathUrl(conn, cmd, outputData) {
  var contentDisposition = cmd.getInline() ? constants.CONTENT_DISPOSITION_INLINE :
    constants.CONTENT_DISPOSITION_ATTACHMENT;
  var strPath = cmd.getDocId() + '/' + cmd.getData();
  var url = yield storage.getSignedUrl(conn.baseUrl, strPath, null, cmd.getTitle(), contentDisposition);
  var errorCode = constants.NO_ERROR;
  if (constants.NO_ERROR !== errorCode) {
    outputData.setStatus('err');
    outputData.setData(errorCode);
  } else {
    outputData.setStatus('ok');
    outputData.setData(url);
  }
}
function* commandSaveFromOrigin(cmd, outputData) {
  yield* addRandomKeyTaskCmd(cmd);
  var queueData = getSaveTask(cmd);
  queueData.setFromOrigin(true);
  yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
  outputData.setStatus('ok');
  outputData.setData(cmd.getSaveKey());
}
function* commandSfcCallback(cmd, isSfcm) {
  var docId = cmd.getDocId();
  logger.debug('Start commandSfcCallback: docId = %s', docId);
  var saveKey = cmd.getSaveKey();
  var statusInfo = cmd.getStatusInfo();
  var isError = constants.NO_ERROR != statusInfo && constants.CONVERT_CORRUPTED != statusInfo;
  var savePathDoc = saveKey + '/' + cmd.getOutputPath();
  var savePathChanges = saveKey + '/changes.zip';
  var savePathHistory = saveKey + '/changesHistory.json';
  var getRes = yield* docsCoServer.getCallback(docId);
  if (getRes) {
    logger.debug('Callback commandSfcCallback: docId = %s callback = %s', docId, getRes.server.href);
    var outputSfc = new commonDefines.OutputSfcData();
    outputSfc.setKey(docId);
    var users = [];
    if (cmd.getUserId()) {
      users.push(cmd.getUserId());
    }
    outputSfc.setUsers(users);
    if (!isSfcm) {
      var actions = [];
      //use UserId case UserActionId miss in gc convertion
      var userActionId = cmd.getUserActionId() || cmd.getUserId();
      if (userActionId) {
        actions.push(new commonDefines.OutputAction(commonDefines.c_oAscUserAction.Out, userActionId));
      }
      outputSfc.setActions(actions);
    }
    outputSfc.setUserData(cmd.getUserData());
    if (!isError) {
      try {
        var data = yield storage.getObject(savePathHistory);
        outputSfc.setChangeHistory(data.toString('utf-8'));
        outputSfc.setUrl(yield storage.getSignedUrl(getRes.baseUrl, savePathDoc));
        outputSfc.setChangeUrl(yield storage.getSignedUrl(getRes.baseUrl, savePathChanges));
      } catch (e) {
        logger.error('Error commandSfcCallback: docId = %s\r\n%s', docId, e.stack);
      }
      if (outputSfc.getUrl() && outputSfc.getUsers().length > 0) {
        if (isSfcm) {
          outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MustSaveForce);
        } else {
          outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MustSave);
        }
      } else {
        isError = true;
      }
    }
    if (isError) {
      if (isSfcm) {
        outputSfc.setStatus(docsCoServer.c_oAscServerStatus.CorruptedForce);
      } else {
        outputSfc.setStatus(docsCoServer.c_oAscServerStatus.Corrupted);
      }
    }
    var uri = getRes.server.href;
    var postData = JSON.stringify(outputSfc);
    if (isSfcm) {
      var lastSave = cmd.getLastSave();
      if (lastSave && !isError) {
        yield* docsCoServer.setForceSave(docId, lastSave, savePathDoc);
      }
      try {
        yield* docsCoServer.sendServerRequest(docId, uri, postData);
      } catch (err) {
        logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack);
      }
    } else {
      //if anybody in document stop save
      var hasEditors = yield* docsCoServer.hasEditors(docId);
      logger.debug('hasEditors commandSfcCallback: docId = %s hasEditors = %d', docId, hasEditors);
      if (!hasEditors) {
        var updateMask = new taskResult.TaskResultData();
        updateMask.key = docId;
        updateMask.status = taskResult.FileStatus.SaveVersion;
        updateMask.statusInfo = cmd.getData();
        var updateIfTask = new taskResult.TaskResultData();
        updateIfTask.status = taskResult.FileStatus.UpdateVersion;
        updateIfTask.statusInfo = constants.NO_ERROR;
        var updateIfRes = yield taskResult.updateIf(updateIfTask, updateMask);
        if (updateIfRes.affectedRows > 0) {
          var replyStr = null;
          try {
            replyStr = yield* docsCoServer.sendServerRequest(docId, uri, postData);
          } catch (err) {
            replyStr = null;
            logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack);
          }
          var requestRes = false;
          var replyData = docsCoServer.parseReplyData(docId, replyStr);
          if (replyData && commonDefines.c_oAscServerCommandErrors.NoError == replyData.error) {
            //в случае comunity server придет запрос в CommandService проверяем результат
            var multi = redisClient.multi([
              ['get', redisKeySaved + docId],
              ['del', redisKeySaved + docId]
            ]);
            var execRes = yield utils.promiseRedis(multi, multi.exec);
            var savedVal = execRes[0];
            requestRes = (null == savedVal || '1' === savedVal);
          }
          if (requestRes) {
            yield docsCoServer.cleanDocumentOnExitPromise(docId, true);
          } else {
            var updateTask = new taskResult.TaskResultData();
            updateTask.key = docId;
            updateTask.status = taskResult.FileStatus.Ok;
            updateTask.statusInfo = constants.NO_ERROR;
            yield taskResult.update(updateTask);
          }
        }
      }
    }
  }
  if (docsCoServer.getIsShutdown() && !isSfcm) {
    yield utils.promiseRedis(redisClient, redisClient.srem, redisKeyShutdown, docId);
  }
  logger.debug('End commandSfcCallback: docId = %s', docId);
}
function* commandSendMMCallback(cmd) {
  var docId = cmd.getDocId();
  logger.debug('Start commandSendMMCallback: docId = %s', docId);
  var saveKey = cmd.getSaveKey();
  var statusInfo = cmd.getStatusInfo();
  var outputSfc = new commonDefines.OutputSfcData();
  outputSfc.setKey(docId);
  if (constants.NO_ERROR == statusInfo) {
    outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MailMerge);
  } else {
    outputSfc.setStatus(docsCoServer.c_oAscServerStatus.Corrupted);
  }
  var mailMergeSendData = cmd.getMailMergeSend();
  var outputMailMerge = new commonDefines.OutputMailMerge(mailMergeSendData);
  outputSfc.setMailMerge(outputMailMerge);
  outputSfc.setUsers([mailMergeSendData.getUserId()]);
  var data = yield storage.getObject(saveKey + '/' + cmd.getOutputPath());
  var xml = data.toString('utf8');
  var files = xml.match(/[< ]file.*?\/>/g);
  var recordRemain = (mailMergeSendData.getRecordTo() - mailMergeSendData.getRecordFrom() + 1);
  var recordIndexStart = mailMergeSendData.getRecordCount() - recordRemain;
  for (var i = 0; i < files.length; ++i) {
    var file = files[i];
    var fieldRes = /field=["'](.*?)["']/.exec(file);
    outputMailMerge.setTo(fieldRes[1]);
    outputMailMerge.setRecordIndex(recordIndexStart + i);
    var pathRes = /path=["'](.*?)["']/.exec(file);
    var signedUrl = yield storage.getSignedUrl(mailMergeSendData.getBaseUrl(), saveKey + '/' + pathRes[1]);
    outputSfc.setUrl(signedUrl);
    var uri = mailMergeSendData.getUrl();
    var replyStr = null;
    var postData = JSON.stringify(outputSfc);
    try {
      replyStr = yield* docsCoServer.sendServerRequest(docId, uri, postData);
    } catch (err) {
      replyStr = null;
      logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack);
    }
    var replyData = docsCoServer.parseReplyData(docId, replyStr);
    if (!(replyData && commonDefines.c_oAscServerCommandErrors.NoError == replyData.error)) {
      var recordErrorCount = mailMergeSendData.getRecordErrorCount();
      recordErrorCount++;
      outputMailMerge.setRecordErrorCount(recordErrorCount);
      mailMergeSendData.setRecordErrorCount(recordErrorCount);
    }
  }
  var newRecordFrom = mailMergeSendData.getRecordFrom() + Math.max(files.length, 1);
  if (newRecordFrom <= mailMergeSendData.getRecordTo()) {
    mailMergeSendData.setRecordFrom(newRecordFrom);
    yield* addRandomKeyTaskCmd(cmd);
    var queueData = getSaveTask(cmd);
    yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
  } else {
    logger.debug('End MailMerge: docId = %s', docId);
  }
  logger.debug('End commandSendMMCallback: docId = %s', docId);
}

exports.openDocument = function(conn, cmd, opt_upsertRes) {
  return co(function* () {
    var outputData;
    var docId = conn ? conn.docId : 'null';
    try {
      var startDate = null;
      if(clientStatsD) {
        startDate = new Date();
      }
      logger.debug('Start command: docId = %s %s', docId, JSON.stringify(cmd));
      outputData = new OutputData(cmd.getCommand());
      switch (cmd.getCommand()) {
        case 'open':
          //yield utils.sleep(5000);
          yield* commandOpen(conn, cmd, outputData, opt_upsertRes);
          break;
        case 'reopen':
          yield* commandReopen(cmd);
          break;
        case 'imgurl':
        case 'imgurls':
          yield* commandImgurls(conn, cmd, outputData);
          break;
        case 'pathurl':
          yield* commandPathUrl(conn, cmd, outputData);
          break;
        default:
          outputData.setStatus('err');
          outputData.setData(constants.UNKNOWN);
          break;
      }
      if(clientStatsD) {
        clientStatsD.timing('coauth.openDocument.' + cmd.getCommand(), new Date() - startDate);
      }
    }
    catch (e) {
      logger.error('Error openDocument: docId = %s\r\n%s', docId, e.stack);
      if (!outputData) {
        outputData = new OutputData();
      }
      outputData.setStatus('err');
      outputData.setData(constants.UNKNOWN);
    }
    finally {
      if (outputData && outputData.getStatus()) {
        logger.debug('Response command: docId = %s %s', docId, JSON.stringify(outputData));
        docsCoServer.sendData(conn, new OutputDataWrap('documentOpen', outputData));
      }
      logger.debug('End command: docId = %s', docId);
    }
  });
};
exports.downloadAs = function(req, res) {
  return co(function* () {
    var docId = 'null';
    try {
      var startDate = null;
      if(clientStatsD) {
        startDate = new Date();
      }
      var strCmd = req.query['cmd'];
      var cmd = new commonDefines.InputCommand(JSON.parse(strCmd));
      docId = cmd.getDocId();
      logger.debug('Start downloadAs: docId = %s %s', docId, strCmd);
      cmd.setData(req.body);
      var outputData = new OutputData(cmd.getCommand());
      switch (cmd.getCommand()) {
        case 'save':
          yield* commandSave(cmd, outputData);
          break;
        case 'savefromorigin':
          yield* commandSaveFromOrigin(cmd, outputData);
          break;
        case 'sendmm':
          yield* commandSendMailMerge(cmd, outputData);
          break;
        case 'sfct':
          yield* commandSfct(cmd, outputData);
          break;
        default:
          outputData.setStatus('err');
          outputData.setData(constants.UNKNOWN);
          break;
      }
      var strRes = JSON.stringify(outputData);
      res.send(strRes);
      logger.debug('End downloadAs: docId = %s %s', docId, strRes);
      if(clientStatsD) {
        clientStatsD.timing('coauth.downloadAs.' + cmd.getCommand(), new Date() - startDate);
      }
    }
    catch (e) {
      logger.error('Error downloadAs: docId = %s\r\n%s', docId, e.stack);
      res.sendStatus(400);
    }
  });
};
exports.saveFromChanges = function(docId, statusInfo, optFormat, opt_userId, opt_queue) {
  return co(function* () {
    try {
      var startDate = null;
      if(clientStatsD) {
        startDate = new Date();
      }
      logger.debug('Start saveFromChanges: docId = %s', docId);
      var task = new taskResult.TaskResultData();
      task.key = docId;
      //делаем select, потому что за время timeout информация могла измениться
      var selectRes = yield taskResult.select(docId);
      var row = selectRes.length > 0 ? selectRes[0] : null;
      if (row && row.status == taskResult.FileStatus.SaveVersion && row.status_info == statusInfo) {
        if (null == optFormat) {
          optFormat = constants.AVS_OFFICESTUDIO_FILE_OTHER_TEAMLAB_INNER;
        }
        var cmd = new commonDefines.InputCommand();
        cmd.setCommand('sfc');
        cmd.setDocId(docId);
        cmd.setOutputFormat(optFormat);
        cmd.setData(statusInfo);
        cmd.setUserActionId(opt_userId);
        yield* addRandomKeyTaskCmd(cmd);
        var queueData = getSaveTask(cmd);
        queueData.setFromChanges(true);
        yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_NORMAL, opt_queue);
        if (docsCoServer.getIsShutdown()) {
          yield utils.promiseRedis(redisClient, redisClient.sadd, redisKeyShutdown, docId);
        }
        logger.debug('AddTask saveFromChanges: docId = %s', docId);
      } else {
        if (row) {
          logger.debug('saveFromChanges status mismatch: docId = %s; row: %d; %d; expected: %d', docId, row.status, row.status_info, statusInfo);
        }
      }
      if (clientStatsD) {
        clientStatsD.timing('coauth.saveFromChanges', new Date() - startDate);
      }
    }
    catch (e) {
      logger.error('Error saveFromChanges: docId = %s\r\n%s', docId, e.stack);
    }
  });
};
exports.receiveTask = function(data, dataRaw) {
  return co(function* () {
    var docId = 'null';
    try {
      var task = new commonDefines.TaskQueueData(JSON.parse(data));
      if (task) {
        var cmd = task.getCmd();
        docId = cmd.getDocId();
        logger.debug('Start receiveTask: docId = %s %s', docId, data);
        var updateTask = getUpdateResponse(cmd);
        var updateRes = yield taskResult.update(updateTask);
        if (updateRes.affectedRows > 0) {
          var outputData = new OutputData(cmd.getCommand());
          var command = cmd.getCommand();
          var additionalOutput = {needUrlKey: null, needUrlMethod: null};
          if ('open' == command || 'reopen' == command) {
            //yield utils.sleep(5000);
            yield* getOutputData(cmd, outputData, cmd.getDocId(), updateTask.status,
              updateTask.statusInfo, null, additionalOutput);
          } else if ('save' == command || 'savefromorigin' == command || 'sfct' == command) {
            yield* getOutputData(cmd, outputData, cmd.getSaveKey(), updateTask.status,
              updateTask.statusInfo, null, additionalOutput);
          } else if ('sfcm' == command) {
            yield* commandSfcCallback(cmd, true);
          } else if ('sfc' == command) {
            yield* commandSfcCallback(cmd, false);
          } else if ('sendmm' == command) {
            yield* commandSendMMCallback(cmd);
          } else if ('conv' == command) {
            //nothing
          }
          if (outputData.getStatus()) {
            logger.debug('Send receiveTask: docId = %s %s', docId, JSON.stringify(outputData));
            var output = new OutputDataWrap('documentOpen', outputData);
            yield* docsCoServer.publish({
              type: commonDefines.c_oPublishType.receiveTask, cmd: cmd, output: output,
              needUrlKey: additionalOutput.needUrlKey, needUrlMethod: additionalOutput.needUrlMethod
            });
          }
        }
        yield* docsCoServer.removeResponse(dataRaw);
        logger.debug('End receiveTask: docId = %s', docId);
      }
    } catch (err) {
      logger.debug('Error receiveTask: docId = %s\r\n%s', docId, err.stack);
    }
  });
};

exports.cleanupCache = cleanupCache;
exports.commandSfctByCmd = commandSfctByCmd;
exports.commandOpenStartPromise = commandOpenStartPromise;
exports.OutputDataWrap = OutputDataWrap;
exports.OutputData = OutputData;
\ No newline at end of file
+/*
 * (c) Copyright Ascensio System SIA 2010-2016
 *
 * This program is a free software product. You can redistribute it and/or
 * modify it under the terms of the GNU Affero General Public License (AGPL)
 * version 3 as published by the Free Software Foundation. In accordance with
 * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect
 * that Ascensio System SIA expressly excludes the warranty of non-infringement
 * of any third-party rights.
 *
 * This program is distributed WITHOUT ANY WARRANTY; without even the implied
 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE. For
 * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
 *
 * You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia,
 * EU, LV-1021.
 *
 * The  interactive user interfaces in modified source and object code versions
 * of the Program must display Appropriate Legal Notices, as required under
 * Section 5 of the GNU AGPL version 3.
 *
 * Pursuant to Section 7(b) of the License you must retain the original Product
 * logo when distributing the program. Pursuant to Section 7(e) we decline to
 * grant you any rights under trademark law for use of our trademarks.
 *
 * All the Product's GUI elements, including illustrations and icon sets, as
 * well as technical writing content are licensed under the terms of the
 * Creative Commons Attribution-ShareAlike 4.0 International. See the License
 * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
 *
 */

var pathModule = require('path');
var urlModule = require('url');
var co = require('co');
var sqlBase = require('./baseConnector');
var docsCoServer = require('./DocsCoServer');
var taskResult = require('./taskresult');
var logger = require('./../../Common/sources/logger');
var utils = require('./../../Common/sources/utils');
var constants = require('./../../Common/sources/constants');
var commonDefines = require('./../../Common/sources/commondefines');
var storage = require('./../../Common/sources/storage-base');
var formatChecker = require('./../../Common/sources/formatchecker');
var statsDClient = require('./../../Common/sources/statsdclient');
var config = require('config');
var config_server = config.get('services.CoAuthoring.server');
var config_utils = config.get('services.CoAuthoring.utils');
var pubsubRedis = require('./pubsubRedis');


var cfgTypesUpload = config_utils.get('limits_image_types_upload');
var cfgTypesCopy = config_utils.get('limits_image_types_copy');
var cfgImageSize = config_server.get('limits_image_size');
var cfgImageDownloadTimeout = config_server.get('limits_image_download_timeout');
var cfgRedisPrefix = config.get('services.CoAuthoring.redis.prefix');

var SAVE_TYPE_PART_START = 0;
var SAVE_TYPE_PART = 1;
var SAVE_TYPE_COMPLETE = 2;
var SAVE_TYPE_COMPLETE_ALL = 3;

var clientStatsD = statsDClient.getClient();
var redisClient = pubsubRedis.getClientRedis();
var redisKeySaved = cfgRedisPrefix + constants.REDIS_KEY_SAVED;
var redisKeyShutdown = cfgRedisPrefix + constants.REDIS_KEY_SHUTDOWN;

function OutputDataWrap(type, data) {
  this['type'] = type;
  this['data'] = data;
}
OutputDataWrap.prototype = {
  fromObject: function(data) {
    this['type'] = data['type'];
    this['data'] = new OutputData();
    this['data'].fromObject(data['data']);
  },
  getType: function() {
    return this['type'];
  },
  setType: function(data) {
    this['type'] = data;
  },
  getData: function() {
    return this['data'];
  },
  setData: function(data) {
    this['data'] = data;
  }
};
function OutputData(type) {
  this['type'] = type;
  this['status'] = undefined;
  this['data'] = undefined;
}
OutputData.prototype = {
  fromObject: function(data) {
    this['type'] = data['type'];
    this['status'] = data['status'];
    this['data'] = data['data'];
  },
  getType: function() {
    return this['type'];
  },
  setType: function(data) {
    this['type'] = data;
  },
  getStatus: function() {
    return this['status'];
  },
  setStatus: function(data) {
    this['status'] = data;
  },
  getData: function() {
    return this['data'];
  },
  setData: function(data) {
    this['data'] = data;
  }
};

function* getOutputData(cmd, outputData, key, status, statusInfo, optConn, optAdditionalOutput) {
  var docId = cmd.getDocId();
  switch (status) {
    case taskResult.FileStatus.SaveVersion:
    case taskResult.FileStatus.UpdateVersion:
    case taskResult.FileStatus.Ok:
      if(taskResult.FileStatus.Ok == status) {
        outputData.setStatus('ok');
      } else if(taskResult.FileStatus.SaveVersion == status) {
        if (optConn && optConn.user.view) {
          outputData.setStatus('updateversion');
        } else {
          var updateMask = new taskResult.TaskResultData();
          updateMask.key = docId;
          updateMask.status = status;
          updateMask.statusInfo = statusInfo;
          var updateTask = new taskResult.TaskResultData();
          updateTask.status = taskResult.FileStatus.Ok;
          updateTask.statusInfo = constants.NO_ERROR;
          var updateIfRes = yield taskResult.updateIf(updateTask, updateMask);
          if (updateIfRes.affectedRows > 0) {
            outputData.setStatus('ok');
          } else {
            outputData.setStatus('updateversion');
          }
        }
      } else {
        outputData.setStatus('updateversion');
      }
      var command = cmd.getCommand();
      if ('open' != command && 'reopen' != command) {
        var strPath = key + '/' + cmd.getOutputPath();
        if (optConn) {
          var contentDisposition = cmd.getInline() ? constants.CONTENT_DISPOSITION_INLINE : constants.CONTENT_DISPOSITION_ATTACHMENT;
          outputData.setData(yield storage.getSignedUrl(optConn.baseUrl, strPath, null, cmd.getTitle(), contentDisposition));
        } else if (optAdditionalOutput) {
          optAdditionalOutput.needUrlKey = strPath;
          optAdditionalOutput.needUrlMethod = 2;
        }
      } else {
        if (optConn) {
          outputData.setData(yield storage.getSignedUrls(optConn.baseUrl, key));
        } else if (optAdditionalOutput) {
          optAdditionalOutput.needUrlKey = key;
          optAdditionalOutput.needUrlMethod = 0;
        }
      }
      break;
    case taskResult.FileStatus.NeedParams:
      outputData.setStatus('needparams');
      var settingsPath = key + '/' + 'settings.json';
      if (optConn) {
        outputData.setData(yield storage.getSignedUrl(optConn.baseUrl, settingsPath));
      } else if (optAdditionalOutput) {
        optAdditionalOutput.needUrlKey = settingsPath;
        optAdditionalOutput.needUrlMethod = 1;
      }
      break;
    case taskResult.FileStatus.NeedPassword:
      outputData.setStatus('needpassword');
      outputData.setData(statusInfo);
      break;
    case taskResult.FileStatus.Err:
    case taskResult.FileStatus.ErrToReload:
      outputData.setStatus('err');
      outputData.setData(statusInfo);
      if (taskResult.FileStatus.ErrToReload == status) {
        yield cleanupCache(key);
      }
      break;
  }
}
function* addRandomKeyTaskCmd(cmd) {
  var task = yield* taskResult.addRandomKeyTask(cmd.getDocId());
  cmd.setSaveKey(task.key);
}
function* saveParts(cmd) {
  var result = false;
  var saveType = cmd.getSaveType();
  var filename;
  if (SAVE_TYPE_COMPLETE_ALL === saveType) {
    filename = 'Editor.bin';
  } else {
    filename = 'Editor' + (cmd.getSaveIndex() || '') + '.bin';
  }
  if (SAVE_TYPE_PART_START === saveType || SAVE_TYPE_COMPLETE_ALL === saveType) {
    yield* addRandomKeyTaskCmd(cmd);
  }
  if (cmd.getUrl()) {
    result = true;
  } else {
    var buffer = cmd.getData();
    yield storage.putObject(cmd.getSaveKey() + '/' + filename, buffer, buffer.length);
    //delete data to prevent serialize into json
    cmd.data = null;
    result = (SAVE_TYPE_COMPLETE_ALL === saveType || SAVE_TYPE_COMPLETE === saveType);
  }
  return result;
}
function getSaveTask(cmd) {
  cmd.setData(null);
  var queueData = new commonDefines.TaskQueueData();
  queueData.setCmd(cmd);
  queueData.setToFile(constants.OUTPUT_NAME + '.' + formatChecker.getStringFromFormat(cmd.getOutputFormat()));
  //todo paid
  //if (cmd.vkey) {
  //  bool
  //  bPaid;
  //  Signature.getVKeyParams(cmd.vkey, out bPaid);
  //  oTaskQueueData.m_bPaid = bPaid;
  //}
  return queueData;
}
function getUpdateResponse(cmd) {
  var updateTask = new taskResult.TaskResultData();
  updateTask.key = cmd.getSaveKey() ? cmd.getSaveKey() : cmd.getDocId();
  var statusInfo = cmd.getStatusInfo();
  if (constants.NO_ERROR == statusInfo) {
    updateTask.status = taskResult.FileStatus.Ok;
  } else if (constants.CONVERT_DOWNLOAD == statusInfo) {
    updateTask.status = taskResult.FileStatus.ErrToReload;
  } else if (constants.CONVERT_NEED_PARAMS == statusInfo) {
    updateTask.status = taskResult.FileStatus.NeedParams;
  } else if (constants.CONVERT_DRM == statusInfo || constants.CONVERT_PASSWORD == statusInfo) {
    updateTask.status = taskResult.FileStatus.NeedPassword;
  } else {
    updateTask.status = taskResult.FileStatus.Err;
  }
  updateTask.statusInfo = statusInfo;
  if (cmd.getTitle()) {
    updateTask.title = cmd.getTitle();
  }
  return updateTask;
}
var cleanupCache = co.wrap(function* (docId) {
  //todo redis ?
  var res = false;
  var removeRes = yield taskResult.remove(docId);
  if (removeRes.affectedRows > 0) {
    yield storage.deletePath(docId);
    res = true;
  }
  return res;
});

function commandOpenStartPromise(docId, cmd, opt_updateUserIndex) {
  var task = new taskResult.TaskResultData();
  task.key = docId;
  task.status = taskResult.FileStatus.WaitQueue;
  task.statusInfo = constants.NO_ERROR;
  if (cmd) {
    task.title = cmd.getTitle();
  } else {
    logger.warn("commandOpenStartPromise empty cmd: docId = %s", docId);
  }

  return taskResult.upsert(task, opt_updateUserIndex);
}
function* commandOpen(conn, cmd, outputData, opt_upsertRes) {
  var upsertRes;
  if (opt_upsertRes) {
    upsertRes = opt_upsertRes;
  } else {
    upsertRes = yield commandOpenStartPromise(cmd.getDocId(), cmd);
  }
  //if CLIENT_FOUND_ROWS don't specify 1 row is inserted , 2 row is updated, and 0 row is set to its current values
  //http://dev.mysql.com/doc/refman/5.7/en/insert-on-duplicate.html
  var bCreate = upsertRes.affectedRows == 1;
  if (!bCreate) {
    var selectRes = yield taskResult.select(cmd.getDocId());
    if (selectRes.length > 0) {
      var row = selectRes[0];
      yield* getOutputData(cmd, outputData, cmd.getDocId(), row.status, row.status_info, conn);
    }
  } else {
    //add task
    cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CANVAS);
    cmd.setEmbeddedFonts(false);
    var dataQueue = new commonDefines.TaskQueueData();
    dataQueue.setCmd(cmd);
    dataQueue.setToFile('Editor.bin');
    var priority = constants.QUEUE_PRIORITY_HIGH;
    var formatIn = formatChecker.getFormatFromString(cmd.getFormat());
    //decrease pdf, djvu, xps convert priority becase long open time
    if (constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === formatIn || constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_DJVU === formatIn ||
      constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_XPS === formatIn) {
      priority = constants.QUEUE_PRIORITY_LOW;
    }
    yield* docsCoServer.addTask(dataQueue, priority);
  }
}
function* commandReopen(cmd) {
  var task = new taskResult.TaskResultData();
  task.key = cmd.getDocId();
  task.status = taskResult.FileStatus.WaitQueue;
  task.statusInfo = constants.NO_ERROR;

  var upsertRes = yield taskResult.update(task);
  if (upsertRes.affectedRows > 0) {
    //add task
    cmd.setUrl(null);//url may expire
    cmd.setSaveKey(cmd.getDocId());
    cmd.setOutputFormat(constants.AVS_OFFICESTUDIO_FILE_CANVAS);
    cmd.setEmbeddedFonts(false);
    var dataQueue = new commonDefines.TaskQueueData();
    dataQueue.setCmd(cmd);
    dataQueue.setToFile('Editor.bin');
    dataQueue.setFromSettings(true);
    yield* docsCoServer.addTask(dataQueue, constants.QUEUE_PRIORITY_HIGH);
  }
}
function* commandSave(cmd, outputData) {
  var completeParts = yield* saveParts(cmd);
  if (completeParts) {
    var queueData = getSaveTask(cmd);
    yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
  }
  outputData.setStatus('ok');
  outputData.setData(cmd.getSaveKey());
}
function* commandSendMailMerge(cmd, outputData) {
  var completeParts = yield* saveParts(cmd);
  var isErr = false;
  if (completeParts) {
    isErr = true;
    var getRes = yield* docsCoServer.getCallback(cmd.getDocId());
    if (getRes) {
      var mailMergeSend = cmd.getMailMergeSend();
      mailMergeSend.setUrl(getRes.server.href);
      mailMergeSend.setBaseUrl(getRes.baseUrl);
      //меняем JsonKey и SaveKey, новый key нужет потому что за одну конвертацию делается часть, а json нужен всегда
      mailMergeSend.setJsonKey(cmd.getSaveKey());
      mailMergeSend.setRecordErrorCount(0);
      yield* addRandomKeyTaskCmd(cmd);
      var queueData = getSaveTask(cmd);
      yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
      isErr = false;
    }
  }
  if (isErr) {
    outputData.setStatus('err');
    outputData.setData(constants.UNKNOWN);
  } else {
    outputData.setStatus('ok');
    outputData.setData(cmd.getSaveKey());
  }
}
function* commandSfctByCmd(cmd) {
  yield* addRandomKeyTaskCmd(cmd);
  var queueData = getSaveTask(cmd);
  queueData.setFromChanges(true);
  yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
}
function* commandSfct(cmd, outputData) {
  yield* commandSfctByCmd(cmd);
  outputData.setStatus('ok');
}
function isDisplayedImage(strName) {
  var res = 0;
  if (strName) {
    //шаблон display[N]image.ext
    var findStr = constants.DISPLAY_PREFIX;
    var index = strName.indexOf(findStr);
    if (-1 != index) {
      if (index + findStr.length < strName.length) {
        var displayN = parseInt(strName[index + findStr.length]);
        if (1 <= displayN && displayN <= 6) {
          var imageIndex = index + findStr.length + 1;
          if (imageIndex == strName.indexOf("image", imageIndex))
            res = displayN;
        }
      }
    }
  }
  return res;
}
function* commandImgurls(conn, cmd, outputData) {
  var supportedFormats;
  var urls;
  var errorCode = constants.NO_ERROR;
  var isImgUrl = 'imgurl' == cmd.getCommand();
  if (isImgUrl) {
    urls = [cmd.getData()];
    supportedFormats = cfgTypesUpload || 'jpg';
  } else {
    urls = cmd.getData();
    supportedFormats = cfgTypesCopy || 'jpg';
  }
  //todo Promise.all()
  var displayedImageMap = {};//to make one imageIndex for ole object urls
  var imageCount = 0;
  var outputUrls = [];
  for (var i = 0; i < urls.length; ++i) {
    var urlSource = urls[i];
    var urlParsed;
    var data = undefined;
    if (urlSource.startsWith('data:')) {
      var delimiterIndex = urlSource.indexOf(',');
      if (-1 != delimiterIndex && (urlSource.length - (delimiterIndex + 1)) * 0.75 <= cfgImageSize) {
        data = new Buffer(urlSource.substring(delimiterIndex + 1), 'base64');
      }
    } else if(urlSource) {
      //todo stream
      data = yield utils.downloadUrlPromise(urlSource, cfgImageDownloadTimeout * 1000, cfgImageSize);
      urlParsed = urlModule.parse(urlSource);
    }
    var outputUrl = {url: 'error', path: 'error'};
    if (data) {
      var format = formatChecker.getImageFormat(data);
      var formatStr;
      if (constants.AVS_OFFICESTUDIO_FILE_UNKNOWN == format && urlParsed) {
        //bin, txt occur in ole object case
        var ext = pathModule.extname(urlParsed.pathname);
        if ('.bin' == ext || '.txt' == ext) {
          formatStr = ext.substring(1);
        }
      } else {
        formatStr = formatChecker.getStringFromFormat(format);
      }
      if (formatStr && -1 !== supportedFormats.indexOf(formatStr)) {
        var userid = cmd.getUserId();
        var imageIndex = cmd.getSaveIndex() + imageCount;
        imageCount++;
        var strLocalPath = 'media/' + utils.crc32(userid).toString(16) + '_';
        if (urlParsed) {
          var urlBasename = pathModule.basename(urlParsed.pathname);
          var displayN = isDisplayedImage(urlBasename);
          if (displayN > 0) {
            var displayedImageName = urlBasename.substring(0, urlBasename.length - formatStr.length - 1);
            var tempIndex = displayedImageMap[displayedImageName];
            if (null != tempIndex) {
              imageIndex = tempIndex;
              imageCount--;
            } else {
              displayedImageMap[displayedImageName] = imageIndex;
            }
            strLocalPath += constants.DISPLAY_PREFIX + displayN;
          }
        }
        strLocalPath += 'image' + imageIndex + '.' + formatStr;
        var strPath = cmd.getDocId() + '/' + strLocalPath;
        yield storage.putObject(strPath, data, data.length);
        var imgUrl = yield storage.getSignedUrl(conn.baseUrl, strPath);
        outputUrl = {url: imgUrl, path: strLocalPath};
      }
    }
    if (isImgUrl && ('error' === outputUrl.url || 'error' === outputUrl.path)) {
      errorCode = constants.UPLOAD_EXTENSION;
      break;
    }
    outputUrls.push(outputUrl);
  }
  if (constants.NO_ERROR !== errorCode) {
    outputData.setStatus('err');
    outputData.setData(errorCode);
  } else {
    outputData.setStatus('ok');
    outputData.setData(outputUrls);
  }
}
function* commandPathUrl(conn, cmd, outputData) {
  var contentDisposition = cmd.getInline() ? constants.CONTENT_DISPOSITION_INLINE :
    constants.CONTENT_DISPOSITION_ATTACHMENT;
  var strPath = cmd.getDocId() + '/' + cmd.getData();
  var url = yield storage.getSignedUrl(conn.baseUrl, strPath, null, cmd.getTitle(), contentDisposition);
  var errorCode = constants.NO_ERROR;
  if (constants.NO_ERROR !== errorCode) {
    outputData.setStatus('err');
    outputData.setData(errorCode);
  } else {
    outputData.setStatus('ok');
    outputData.setData(url);
  }
}
function* commandSaveFromOrigin(cmd, outputData) {
  yield* addRandomKeyTaskCmd(cmd);
  var queueData = getSaveTask(cmd);
  queueData.setFromOrigin(true);
  yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
  outputData.setStatus('ok');
  outputData.setData(cmd.getSaveKey());
}
function* commandSfcCallback(cmd, isSfcm) {
  var docId = cmd.getDocId();
  logger.debug('Start commandSfcCallback: docId = %s', docId);
  var saveKey = cmd.getSaveKey();
  var statusInfo = cmd.getStatusInfo();
  var isError = constants.NO_ERROR != statusInfo && constants.CONVERT_CORRUPTED != statusInfo;
  var savePathDoc = saveKey + '/' + cmd.getOutputPath();
  var savePathChanges = saveKey + '/changes.zip';
  var savePathHistory = saveKey + '/changesHistory.json';
  var getRes = yield* docsCoServer.getCallback(docId);
  if (getRes) {
    logger.debug('Callback commandSfcCallback: docId = %s callback = %s', docId, getRes.server.href);
    var outputSfc = new commonDefines.OutputSfcData();
    outputSfc.setKey(docId);
    var users = [];
    if (cmd.getUserId()) {
      users.push(cmd.getUserId());
    }
    outputSfc.setUsers(users);
    if (!isSfcm) {
      var actions = [];
      //use UserId case UserActionId miss in gc convertion
      var userActionId = cmd.getUserActionId() || cmd.getUserId();
      if (userActionId) {
        actions.push(new commonDefines.OutputAction(commonDefines.c_oAscUserAction.Out, userActionId));
      }
      outputSfc.setActions(actions);
    }
    outputSfc.setUserData(cmd.getUserData());
    if (!isError) {
      try {
        var data = yield storage.getObject(savePathHistory);
        outputSfc.setChangeHistory(data.toString('utf-8'));
        outputSfc.setUrl(yield storage.getSignedUrl(getRes.baseUrl, savePathDoc));
        outputSfc.setChangeUrl(yield storage.getSignedUrl(getRes.baseUrl, savePathChanges));
      } catch (e) {
        logger.error('Error commandSfcCallback: docId = %s\r\n%s', docId, e.stack);
      }
      if (outputSfc.getUrl() && outputSfc.getUsers().length > 0) {
        if (isSfcm) {
          outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MustSaveForce);
        } else {
          outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MustSave);
        }
      } else {
        isError = true;
      }
    }
    if (isError) {
      if (isSfcm) {
        outputSfc.setStatus(docsCoServer.c_oAscServerStatus.CorruptedForce);
      } else {
        outputSfc.setStatus(docsCoServer.c_oAscServerStatus.Corrupted);
      }
    }
    var uri = getRes.server.href;
    var postData = JSON.stringify(outputSfc);
    if (isSfcm) {
      var lastSave = cmd.getLastSave();
      if (lastSave && !isError) {
        yield* docsCoServer.setForceSave(docId, lastSave, savePathDoc);
      }
      try {
        yield* docsCoServer.sendServerRequest(docId, uri, postData);
      } catch (err) {
        logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack);
      }
    } else {
      //if anybody in document stop save
      var hasEditors = yield* docsCoServer.hasEditors(docId);
      logger.debug('hasEditors commandSfcCallback: docId = %s hasEditors = %d', docId, hasEditors);
      if (!hasEditors) {
        var updateMask = new taskResult.TaskResultData();
        updateMask.key = docId;
        updateMask.status = taskResult.FileStatus.SaveVersion;
        updateMask.statusInfo = cmd.getData();
        var updateIfTask = new taskResult.TaskResultData();
        updateIfTask.status = taskResult.FileStatus.UpdateVersion;
        updateIfTask.statusInfo = constants.NO_ERROR;
        var updateIfRes = yield taskResult.updateIf(updateIfTask, updateMask);
        if (updateIfRes.affectedRows > 0) {
          var replyStr = null;
          try {
            replyStr = yield* docsCoServer.sendServerRequest(docId, uri, postData);
          } catch (err) {
            replyStr = null;
            logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack);
          }
          var requestRes = false;
          var replyData = docsCoServer.parseReplyData(docId, replyStr);
          if (replyData && commonDefines.c_oAscServerCommandErrors.NoError == replyData.error) {
            //в случае comunity server придет запрос в CommandService проверяем результат
            var multi = redisClient.multi([
              ['get', redisKeySaved + docId],
              ['del', redisKeySaved + docId]
            ]);
            var execRes = yield utils.promiseRedis(multi, multi.exec);
            var savedVal = execRes[0];
            requestRes = (null == savedVal || '1' === savedVal);
          }
          if (requestRes) {
            yield docsCoServer.cleanDocumentOnExitPromise(docId, true);
          } else {
            var updateTask = new taskResult.TaskResultData();
            updateTask.key = docId;
            updateTask.status = taskResult.FileStatus.Ok;
            updateTask.statusInfo = constants.NO_ERROR;
            yield taskResult.update(updateTask);
          }
        }
      }
    }
  }
  if (docsCoServer.getIsShutdown() && !isSfcm) {
    yield utils.promiseRedis(redisClient, redisClient.srem, redisKeyShutdown, docId);
  }
  logger.debug('End commandSfcCallback: docId = %s', docId);
}
function* commandSendMMCallback(cmd) {
  var docId = cmd.getDocId();
  logger.debug('Start commandSendMMCallback: docId = %s', docId);
  var saveKey = cmd.getSaveKey();
  var statusInfo = cmd.getStatusInfo();
  var outputSfc = new commonDefines.OutputSfcData();
  outputSfc.setKey(docId);
  if (constants.NO_ERROR == statusInfo) {
    outputSfc.setStatus(docsCoServer.c_oAscServerStatus.MailMerge);
  } else {
    outputSfc.setStatus(docsCoServer.c_oAscServerStatus.Corrupted);
  }
  var mailMergeSendData = cmd.getMailMergeSend();
  var outputMailMerge = new commonDefines.OutputMailMerge(mailMergeSendData);
  outputSfc.setMailMerge(outputMailMerge);
  outputSfc.setUsers([mailMergeSendData.getUserId()]);
  var data = yield storage.getObject(saveKey + '/' + cmd.getOutputPath());
  var xml = data.toString('utf8');
  var files = xml.match(/[< ]file.*?\/>/g);
  var recordRemain = (mailMergeSendData.getRecordTo() - mailMergeSendData.getRecordFrom() + 1);
  var recordIndexStart = mailMergeSendData.getRecordCount() - recordRemain;
  for (var i = 0; i < files.length; ++i) {
    var file = files[i];
    var fieldRes = /field=["'](.*?)["']/.exec(file);
    outputMailMerge.setTo(fieldRes[1]);
    outputMailMerge.setRecordIndex(recordIndexStart + i);
    var pathRes = /path=["'](.*?)["']/.exec(file);
    var signedUrl = yield storage.getSignedUrl(mailMergeSendData.getBaseUrl(), saveKey + '/' + pathRes[1]);
    outputSfc.setUrl(signedUrl);
    var uri = mailMergeSendData.getUrl();
    var replyStr = null;
    var postData = JSON.stringify(outputSfc);
    try {
      replyStr = yield* docsCoServer.sendServerRequest(docId, uri, postData);
    } catch (err) {
      replyStr = null;
      logger.error('sendServerRequest error: docId = %s;url = %s;data = %s\r\n%s', docId, uri, postData, err.stack);
    }
    var replyData = docsCoServer.parseReplyData(docId, replyStr);
    if (!(replyData && commonDefines.c_oAscServerCommandErrors.NoError == replyData.error)) {
      var recordErrorCount = mailMergeSendData.getRecordErrorCount();
      recordErrorCount++;
      outputMailMerge.setRecordErrorCount(recordErrorCount);
      mailMergeSendData.setRecordErrorCount(recordErrorCount);
    }
  }
  var newRecordFrom = mailMergeSendData.getRecordFrom() + Math.max(files.length, 1);
  if (newRecordFrom <= mailMergeSendData.getRecordTo()) {
    mailMergeSendData.setRecordFrom(newRecordFrom);
    yield* addRandomKeyTaskCmd(cmd);
    var queueData = getSaveTask(cmd);
    yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_LOW);
  } else {
    logger.debug('End MailMerge: docId = %s', docId);
  }
  logger.debug('End commandSendMMCallback: docId = %s', docId);
}

exports.openDocument = function(conn, cmd, opt_upsertRes) {
  return co(function* () {
    var outputData;
    var docId = conn ? conn.docId : 'null';
    try {
      var startDate = null;
      if(clientStatsD) {
        startDate = new Date();
      }
      logger.debug('Start command: docId = %s %s', docId, JSON.stringify(cmd));
      outputData = new OutputData(cmd.getCommand());
      switch (cmd.getCommand()) {
        case 'open':
          //yield utils.sleep(5000);
          yield* commandOpen(conn, cmd, outputData, opt_upsertRes);
          break;
        case 'reopen':
          yield* commandReopen(cmd);
          break;
        case 'imgurl':
        case 'imgurls':
          yield* commandImgurls(conn, cmd, outputData);
          break;
        case 'pathurl':
          yield* commandPathUrl(conn, cmd, outputData);
          break;
        default:
          outputData.setStatus('err');
          outputData.setData(constants.UNKNOWN);
          break;
      }
      if(clientStatsD) {
        clientStatsD.timing('coauth.openDocument.' + cmd.getCommand(), new Date() - startDate);
      }
    }
    catch (e) {
      logger.error('Error openDocument: docId = %s\r\n%s', docId, e.stack);
      if (!outputData) {
        outputData = new OutputData();
      }
      outputData.setStatus('err');
      outputData.setData(constants.UNKNOWN);
    }
    finally {
      if (outputData && outputData.getStatus()) {
        logger.debug('Response command: docId = %s %s', docId, JSON.stringify(outputData));
        docsCoServer.sendData(conn, new OutputDataWrap('documentOpen', outputData));
      }
      logger.debug('End command: docId = %s', docId);
    }
  });
};
exports.downloadAs = function(req, res) {
  return co(function* () {
    var docId = 'null';
    try {
      var startDate = null;
      if(clientStatsD) {
        startDate = new Date();
      }
      var strCmd = req.query['cmd'];
      var cmd = new commonDefines.InputCommand(JSON.parse(strCmd));
      docId = cmd.getDocId();
      logger.debug('Start downloadAs: docId = %s %s', docId, strCmd);
      cmd.setData(req.body);
      var outputData = new OutputData(cmd.getCommand());
      switch (cmd.getCommand()) {
        case 'save':
          yield* commandSave(cmd, outputData);
          break;
        case 'savefromorigin':
          yield* commandSaveFromOrigin(cmd, outputData);
          break;
        case 'sendmm':
          yield* commandSendMailMerge(cmd, outputData);
          break;
        case 'sfct':
          yield* commandSfct(cmd, outputData);
          break;
        default:
          outputData.setStatus('err');
          outputData.setData(constants.UNKNOWN);
          break;
      }
      var strRes = JSON.stringify(outputData);
      res.send(strRes);
      logger.debug('End downloadAs: docId = %s %s', docId, strRes);
      if(clientStatsD) {
        clientStatsD.timing('coauth.downloadAs.' + cmd.getCommand(), new Date() - startDate);
      }
    }
    catch (e) {
      logger.error('Error downloadAs: docId = %s\r\n%s', docId, e.stack);
      res.sendStatus(400);
    }
  });
};
exports.saveFromChanges = function(docId, statusInfo, optFormat, opt_userId, opt_queue) {
  return co(function* () {
    try {
      var startDate = null;
      if(clientStatsD) {
        startDate = new Date();
      }
      logger.debug('Start saveFromChanges: docId = %s', docId);
      var task = new taskResult.TaskResultData();
      task.key = docId;
      //делаем select, потому что за время timeout информация могла измениться
      var selectRes = yield taskResult.select(docId);
      var row = selectRes.length > 0 ? selectRes[0] : null;
      if (row && row.status == taskResult.FileStatus.SaveVersion && row.status_info == statusInfo) {
        if (null == optFormat) {
          optFormat = constants.AVS_OFFICESTUDIO_FILE_OTHER_TEAMLAB_INNER;
        }
        var cmd = new commonDefines.InputCommand();
        cmd.setCommand('sfc');
        cmd.setDocId(docId);
        cmd.setOutputFormat(optFormat);
        cmd.setData(statusInfo);
        cmd.setUserActionId(opt_userId);
        yield* addRandomKeyTaskCmd(cmd);
        var queueData = getSaveTask(cmd);
        queueData.setFromChanges(true);
        yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_NORMAL, opt_queue);
        if (docsCoServer.getIsShutdown()) {
          yield utils.promiseRedis(redisClient, redisClient.sadd, redisKeyShutdown, docId);
        }
        logger.debug('AddTask saveFromChanges: docId = %s', docId);
      } else {
        if (row) {
          logger.debug('saveFromChanges status mismatch: docId = %s; row: %d; %d; expected: %d', docId, row.status, row.status_info, statusInfo);
        }
      }
      if (clientStatsD) {
        clientStatsD.timing('coauth.saveFromChanges', new Date() - startDate);
      }
    }
    catch (e) {
      logger.error('Error saveFromChanges: docId = %s\r\n%s', docId, e.stack);
    }
  });
};
exports.receiveTask = function(data, dataRaw) {
  return co(function* () {
    var docId = 'null';
    try {
      var task = new commonDefines.TaskQueueData(JSON.parse(data));
      if (task) {
        var cmd = task.getCmd();
        docId = cmd.getDocId();
        logger.debug('Start receiveTask: docId = %s %s', docId, data);
        var updateTask = getUpdateResponse(cmd);
        var updateRes = yield taskResult.update(updateTask);
        if (updateRes.affectedRows > 0) {
          var outputData = new OutputData(cmd.getCommand());
          var command = cmd.getCommand();
          var additionalOutput = {needUrlKey: null, needUrlMethod: null};
          if ('open' == command || 'reopen' == command) {
            //yield utils.sleep(5000);
            yield* getOutputData(cmd, outputData, cmd.getDocId(), updateTask.status,
              updateTask.statusInfo, null, additionalOutput);
          } else if ('save' == command || 'savefromorigin' == command || 'sfct' == command) {
            yield* getOutputData(cmd, outputData, cmd.getSaveKey(), updateTask.status,
              updateTask.statusInfo, null, additionalOutput);
          } else if ('sfcm' == command) {
            yield* commandSfcCallback(cmd, true);
          } else if ('sfc' == command) {
            yield* commandSfcCallback(cmd, false);
          } else if ('sendmm' == command) {
            yield* commandSendMMCallback(cmd);
          } else if ('conv' == command) {
            //nothing
          }
          if (outputData.getStatus()) {
            logger.debug('Send receiveTask: docId = %s %s', docId, JSON.stringify(outputData));
            var output = new OutputDataWrap('documentOpen', outputData);
            yield* docsCoServer.publish({
              type: commonDefines.c_oPublishType.receiveTask, cmd: cmd, output: output,
              needUrlKey: additionalOutput.needUrlKey, needUrlMethod: additionalOutput.needUrlMethod
            });
          }
        }
        yield* docsCoServer.removeResponse(dataRaw);
        logger.debug('End receiveTask: docId = %s', docId);
      }
    } catch (err) {
      logger.debug('Error receiveTask: docId = %s\r\n%s', docId, err.stack);
    }
  });
};

exports.cleanupCache = cleanupCache;
exports.commandSfctByCmd = commandSfctByCmd;
exports.commandOpenStartPromise = commandOpenStartPromise;
exports.OutputDataWrap = OutputDataWrap;
exports.OutputData = OutputData;
\ No newline at end of file
diff --git a/DocService/sources/converterservice.js b/DocService/sources/converterservice.js
index 7390248a..b27d6c0f 100644
--- a/DocService/sources/converterservice.js
+++ b/DocService/sources/converterservice.js
@@ -74,7 +74,7 @@ function* getConvertStatus(cmd, selectRes, baseUrl) {
         status.err = constants.UNKNOWN;
         break;
       case taskResult.FileStatus.NeedPassword:
-        status.err = constants.CONVERT_DRM;
+        status.err = row.status_info;
         break;
     }
     var lastOpenDate = row.last_open_date;
diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js
index f1b57d48..c5982b51 100644
--- a/FileConverter/sources/converter.js
+++ b/FileConverter/sources/converter.js
@@ -66,10 +66,9 @@ var MAX_OPEN_FILES = 200;
 var TEMP_PREFIX = 'ASC_CONVERT';
 var queue = null;
 var clientStatsD = statsDClient.getClient();
-var exitCodesReturn = [constants.CONVERT_MS_OFFCRYPTO, constants.CONVERT_NEED_PARAMS, constants.CONVERT_CORRUPTED,
-  constants.CONVERT_DRM, constants.CONVERT_PASSWORD];
-var exitCodesMinorError = [constants.CONVERT_MS_OFFCRYPTO, constants.CONVERT_NEED_PARAMS, constants.CONVERT_DRM,
+var exitCodesReturn = [constants.CONVERT_NEED_PARAMS, constants.CONVERT_CORRUPTED, constants.CONVERT_DRM,
   constants.CONVERT_PASSWORD];
+var exitCodesMinorError = [constants.CONVERT_NEED_PARAMS, constants.CONVERT_DRM, constants.CONVERT_PASSWORD];
 var exitCodesUpload = [constants.NO_ERROR, constants.CONVERT_CORRUPTED, constants.CONVERT_NEED_PARAMS,
   constants.CONVERT_DRM];
 

From 4c2b11654fce5c20f0b910b104e01c94494afc51 Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Tue, 16 Aug 2016 17:51:23 +0300
Subject: [PATCH 17/34] slash in document title

---
 Common/sources/storage-fs.js | 2 +-
 Common/sources/storage-s3.js | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Common/sources/storage-fs.js b/Common/sources/storage-fs.js
index 2039e4a5..767d2512 100644
--- a/Common/sources/storage-fs.js
+++ b/Common/sources/storage-fs.js
@@ -1 +1 @@
-/*
 * (c) Copyright Ascensio System SIA 2010-2016
 *
 * This program is a free software product. You can redistribute it and/or
 * modify it under the terms of the GNU Affero General Public License (AGPL)
 * version 3 as published by the Free Software Foundation. In accordance with
 * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect
 * that Ascensio System SIA expressly excludes the warranty of non-infringement
 * of any third-party rights.
 *
 * This program is distributed WITHOUT ANY WARRANTY; without even the implied
 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE. For
 * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
 *
 * You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia,
 * EU, LV-1021.
 *
 * The  interactive user interfaces in modified source and object code versions
 * of the Program must display Appropriate Legal Notices, as required under
 * Section 5 of the GNU AGPL version 3.
 *
 * Pursuant to Section 7(b) of the License you must retain the original Product
 * logo when distributing the program. Pursuant to Section 7(e) we decline to
 * grant you any rights under trademark law for use of our trademarks.
 *
 * All the Product's GUI elements, including illustrations and icon sets, as
 * well as technical writing content are licensed under the terms of the
 * Creative Commons Attribution-ShareAlike 4.0 International. See the License
 * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
 *
 */

var fs = require('fs');
var path = require('path');
var mkdirp = require('mkdirp');
var utils = require("./utils");
var crypto = require('crypto');

var configStorage = require('config').get('storage');
var cfgBucketName = configStorage.get('bucketName');
var cfgStorageFolderName = configStorage.get('storageFolderName');
var cfgStorageExternalHost = configStorage.get('externalHost');
var configFs = configStorage.get('fs');
var cfgStorageFolderPath = configFs.get('folderPath');
var cfgStorageSecretString = configFs.get('secretString');

function getFilePath(strPath) {
  return path.join(cfgStorageFolderPath, strPath);
}
function getOutputPath(strPath) {
  return strPath.replace(/\\/g, '/');
}
function removeEmptyParent(strPath, done) {
  if (cfgStorageFolderPath.length + 1 >= strPath.length) {
    done();
  } else {
    fs.readdir(strPath, function(err, list) {
      if (err) {
        //не реагируем на ошибку, потому скорее всего эта папка удалилась в соседнем потоке
        done();
      } else {
        if (list.length > 0) {
          done();
        } else {
          fs.rmdir(strPath, function(err) {
            if (err) {
              //не реагируем на ошибку, потому скорее всего эта папка удалилась в соседнем потоке
              done();
            } else {
              removeEmptyParent(path.dirname(strPath), function(err) {
                done(err);
              });
            }
          });
        }
      }
    });
  }
}

exports.getObject = function(strPath) {
  return utils.readFile(getFilePath(strPath));
};

exports.putObject = function(strPath, buffer, contentLength) {
  return new Promise(function(resolve, reject) {
    var fsPath = getFilePath(strPath);
    mkdirp(path.dirname(fsPath), function(err) {
      if (err) {
        reject(err);
      } else {
        //todo 0666
        if (Buffer.isBuffer(buffer)) {
          fs.writeFile(fsPath, buffer, function(err) {
            if (err) {
              reject(err);
            } else {
              resolve();
            }
          });
        } else {
          utils.promiseCreateWriteStream(fsPath).then(function(writable) {
            buffer.pipe(writable);
          }).catch(function(err) {
            reject(err);
          });
        }
      }
    });
  });
};
exports.listObjects = function(strPath) {
  return utils.listObjects(getFilePath(strPath)).then(function(values) {
    return values.map(function(curvalue) {
      return getOutputPath(curvalue.substring(cfgStorageFolderPath.length + 1));
    });
  });
};
exports.deleteObject = function(strPath) {
  return new Promise(function(resolve, reject) {
    var fsPath = getFilePath(strPath);
    fs.unlink(fsPath, function(err) {
      if (err) {
        reject(err);
      } else {
        //resolve();
        removeEmptyParent(path.dirname(fsPath), function(err) {
          if (err) {
            reject(err);
          } else {
            resolve();
          }
        });
      }
    });
  });
};
exports.deleteObjects = function(strPaths) {
  return Promise.all(strPaths.map(exports.deleteObject));
};
exports.getSignedUrl = function(baseUrl, strPath, optUrlExpires, optFilename, opt_type) {
  return new Promise(function(resolve, reject) {
    var userFriendlyName = optFilename ? encodeURIComponent(optFilename) : path.basename(strPath);
    var uri = '/' + cfgBucketName + '/' + cfgStorageFolderName + '/' + strPath + '/' + userFriendlyName;
    var url = (cfgStorageExternalHost ? cfgStorageExternalHost : baseUrl) + uri;

    var date = new Date();
    var expires = Math.ceil(date.getTime() / 1000) + (optUrlExpires || 604800);
	
	// отключил время жизни т.к. существует сценарий, при котором объект
	// получаемый по ссылке запрашивается после того как закончилось время
	// его жизни.
    var md5 = crypto.createHash('md5').update(/*expires + */uri + cfgStorageSecretString).digest("base64");
    md5 = md5.replace(/\+/g, "-");
    md5 = md5.replace(/\//g, "_");

    url += ('?md5=' + md5 + '&expires=' + expires);
    url += '&disposition=' + encodeURIComponent(utils.getContentDisposition(null, null, opt_type));
    resolve(utils.changeOnlyOfficeUrl(url, strPath, optFilename));
  });
};
\ No newline at end of file
+/*
 * (c) Copyright Ascensio System SIA 2010-2016
 *
 * This program is a free software product. You can redistribute it and/or
 * modify it under the terms of the GNU Affero General Public License (AGPL)
 * version 3 as published by the Free Software Foundation. In accordance with
 * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect
 * that Ascensio System SIA expressly excludes the warranty of non-infringement
 * of any third-party rights.
 *
 * This program is distributed WITHOUT ANY WARRANTY; without even the implied
 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE. For
 * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
 *
 * You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia,
 * EU, LV-1021.
 *
 * The  interactive user interfaces in modified source and object code versions
 * of the Program must display Appropriate Legal Notices, as required under
 * Section 5 of the GNU AGPL version 3.
 *
 * Pursuant to Section 7(b) of the License you must retain the original Product
 * logo when distributing the program. Pursuant to Section 7(e) we decline to
 * grant you any rights under trademark law for use of our trademarks.
 *
 * All the Product's GUI elements, including illustrations and icon sets, as
 * well as technical writing content are licensed under the terms of the
 * Creative Commons Attribution-ShareAlike 4.0 International. See the License
 * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
 *
 */

var fs = require('fs');
var path = require('path');
var mkdirp = require('mkdirp');
var utils = require("./utils");
var crypto = require('crypto');

var configStorage = require('config').get('storage');
var cfgBucketName = configStorage.get('bucketName');
var cfgStorageFolderName = configStorage.get('storageFolderName');
var cfgStorageExternalHost = configStorage.get('externalHost');
var configFs = configStorage.get('fs');
var cfgStorageFolderPath = configFs.get('folderPath');
var cfgStorageSecretString = configFs.get('secretString');

function getFilePath(strPath) {
  return path.join(cfgStorageFolderPath, strPath);
}
function getOutputPath(strPath) {
  return strPath.replace(/\\/g, '/');
}
function removeEmptyParent(strPath, done) {
  if (cfgStorageFolderPath.length + 1 >= strPath.length) {
    done();
  } else {
    fs.readdir(strPath, function(err, list) {
      if (err) {
        //не реагируем на ошибку, потому скорее всего эта папка удалилась в соседнем потоке
        done();
      } else {
        if (list.length > 0) {
          done();
        } else {
          fs.rmdir(strPath, function(err) {
            if (err) {
              //не реагируем на ошибку, потому скорее всего эта папка удалилась в соседнем потоке
              done();
            } else {
              removeEmptyParent(path.dirname(strPath), function(err) {
                done(err);
              });
            }
          });
        }
      }
    });
  }
}

exports.getObject = function(strPath) {
  return utils.readFile(getFilePath(strPath));
};

exports.putObject = function(strPath, buffer, contentLength) {
  return new Promise(function(resolve, reject) {
    var fsPath = getFilePath(strPath);
    mkdirp(path.dirname(fsPath), function(err) {
      if (err) {
        reject(err);
      } else {
        //todo 0666
        if (Buffer.isBuffer(buffer)) {
          fs.writeFile(fsPath, buffer, function(err) {
            if (err) {
              reject(err);
            } else {
              resolve();
            }
          });
        } else {
          utils.promiseCreateWriteStream(fsPath).then(function(writable) {
            buffer.pipe(writable);
          }).catch(function(err) {
            reject(err);
          });
        }
      }
    });
  });
};
exports.listObjects = function(strPath) {
  return utils.listObjects(getFilePath(strPath)).then(function(values) {
    return values.map(function(curvalue) {
      return getOutputPath(curvalue.substring(cfgStorageFolderPath.length + 1));
    });
  });
};
exports.deleteObject = function(strPath) {
  return new Promise(function(resolve, reject) {
    var fsPath = getFilePath(strPath);
    fs.unlink(fsPath, function(err) {
      if (err) {
        reject(err);
      } else {
        //resolve();
        removeEmptyParent(path.dirname(fsPath), function(err) {
          if (err) {
            reject(err);
          } else {
            resolve();
          }
        });
      }
    });
  });
};
exports.deleteObjects = function(strPaths) {
  return Promise.all(strPaths.map(exports.deleteObject));
};
exports.getSignedUrl = function(baseUrl, strPath, optUrlExpires, optFilename, opt_type) {
  return new Promise(function(resolve, reject) {
    //replace '/' with %2f before encodeURIComponent becase nginx determine %2f as '/' and get wrong system path
    var userFriendlyName = optFilename ? encodeURIComponent(optFilename.replace(/\//g, "%2f")) : path.basename(strPath);
    var uri = '/' + cfgBucketName + '/' + cfgStorageFolderName + '/' + strPath + '/' + userFriendlyName;
    var url = (cfgStorageExternalHost ? cfgStorageExternalHost : baseUrl) + uri;

    var date = new Date();
    var expires = Math.ceil(date.getTime() / 1000) + (optUrlExpires || 604800);
	
	// отключил время жизни т.к. существует сценарий, при котором объект
	// получаемый по ссылке запрашивается после того как закончилось время
	// его жизни.
    var md5 = crypto.createHash('md5').update(/*expires + */uri + cfgStorageSecretString).digest("base64");
    md5 = md5.replace(/\+/g, "-");
    md5 = md5.replace(/\//g, "_");

    url += ('?md5=' + md5 + '&expires=' + expires);
    url += '&disposition=' + encodeURIComponent(utils.getContentDisposition(null, null, opt_type));
    resolve(utils.changeOnlyOfficeUrl(url, strPath, optFilename));
  });
};
\ No newline at end of file
diff --git a/Common/sources/storage-s3.js b/Common/sources/storage-s3.js
index 5af57d5b..fd18a420 100644
--- a/Common/sources/storage-s3.js
+++ b/Common/sources/storage-s3.js
@@ -1 +1 @@
-/*
 * (c) Copyright Ascensio System SIA 2010-2016
 *
 * This program is a free software product. You can redistribute it and/or
 * modify it under the terms of the GNU Affero General Public License (AGPL)
 * version 3 as published by the Free Software Foundation. In accordance with
 * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect
 * that Ascensio System SIA expressly excludes the warranty of non-infringement
 * of any third-party rights.
 *
 * This program is distributed WITHOUT ANY WARRANTY; without even the implied
 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE. For
 * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
 *
 * You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia,
 * EU, LV-1021.
 *
 * The  interactive user interfaces in modified source and object code versions
 * of the Program must display Appropriate Legal Notices, as required under
 * Section 5 of the GNU AGPL version 3.
 *
 * Pursuant to Section 7(b) of the License you must retain the original Product
 * logo when distributing the program. Pursuant to Section 7(e) we decline to
 * grant you any rights under trademark law for use of our trademarks.
 *
 * All the Product's GUI elements, including illustrations and icon sets, as
 * well as technical writing content are licensed under the terms of the
 * Creative Commons Attribution-ShareAlike 4.0 International. See the License
 * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
 *
 */

'use strict';
var url = require('url');
var path = require('path');
var AWS = require('aws-sdk');
var mime = require('mime');
var s3urlSigner = require('amazon-s3-url-signer');
var utils = require('./utils');

var configStorage = require('config').get('storage');
var cfgRegion = configStorage.get('region');
var cfgEndpoint = configStorage.get('endpoint');
var cfgBucketName = configStorage.get('bucketName');
var cfgStorageFolderName = configStorage.get('storageFolderName');
var cfgAccessKeyId = configStorage.get('accessKeyId');
var cfgSecretAccessKey = configStorage.get('secretAccessKey');
var cfgUseRequestToGetUrl = configStorage.get('useRequestToGetUrl');
var cfgUseSignedUrl = configStorage.get('useSignedUrl');
var cfgExternalHost = configStorage.get('externalHost');
/**
 * Don't hard-code your credentials!
 * Export the following environment variables instead:
 *
 * export AWS_ACCESS_KEY_ID='AKID'
 * export AWS_SECRET_ACCESS_KEY='SECRET'
 */
var configS3 = {
  region: cfgRegion,
  endpoint: cfgEndpoint,
  accessKeyId: cfgAccessKeyId,
  secretAccessKey: cfgSecretAccessKey
};
if (configS3.endpoint) {
  configS3.sslEnabled = false;
  configS3.s3ForcePathStyle = true;
}
AWS.config.update(configS3);
var s3Client = new AWS.S3();
if (configS3.endpoint) {
  s3Client.endpoint = new AWS.Endpoint(configS3.endpoint);
}
var cfgEndpointParsed = null;
if (cfgEndpoint) {
  cfgEndpointParsed = url.parse(cfgEndpoint);
}
//This operation enables you to delete multiple objects from a bucket using a single HTTP request. You may specify up to 1000 keys.
var MAX_DELETE_OBJECTS = 1000;

function getFilePath(strPath) {
  //todo
  return cfgStorageFolderName + '/' + strPath;
}
function joinListObjects(inputArray, outputArray) {
  var length = inputArray.length;
  for (var i = 0; i < length; i++) {
    outputArray.push(inputArray[i].Key.substring((cfgStorageFolderName + '/').length));
  }
}
function listObjectsExec(output, params, resolve, reject) {
  s3Client.listObjects(params, function(err, data) {
    if (err) {
      reject(err);
    } else {
      joinListObjects(data.Contents, output);
      if (data.IsTruncated && (data.NextMarker || data.Contents.length > 0)) {
        params.Marker = data.NextMarker || data.Contents[data.Contents.length - 1].Key;
        listObjectsExec(output, params, resolve, reject);
      } else {
        resolve(output);
      }
    }
  });
}
function mapDeleteObjects(currentValue) {
  return {Key: currentValue};
}
function deleteObjectsHelp(aKeys) {
  return new Promise(function(resolve, reject) {
    //By default, the operation uses verbose mode in which the response includes the result of deletion of each key in your request.
    //In quiet mode the response includes only keys where the delete operation encountered an error.
    var params = {Bucket: cfgBucketName, Delete: {Objects: aKeys, Quiet: true}};
    s3Client.deleteObjects(params, function(err, data) {
      if (err) {
        reject(err);
      } else {
        resolve(data);
      }
    });
  });
}

exports.getObject = function(strPath) {
  return new Promise(function(resolve, reject) {
    var params = {Bucket: cfgBucketName, Key: getFilePath(strPath)};
    s3Client.getObject(params, function(err, data) {
      if (err) {
        reject(err);
      } else {
        resolve(data.Body);
      }
    });
  });
};
exports.putObject = function(strPath, buffer, contentLength) {
  return new Promise(function(resolve, reject) {
    //todo рассмотреть Expires
    var params = {Bucket: cfgBucketName, Key: getFilePath(strPath), Body: buffer,
      ContentLength: contentLength, ContentType: mime.lookup(strPath)};
    s3Client.putObject(params, function(err, data) {
      if (err) {
        reject(err);
      } else {
        resolve(data);
      }
    });
  });
};
exports.listObjects = function(strPath) {
  return new Promise(function(resolve, reject) {
    var params = {Bucket: cfgBucketName, Prefix: getFilePath(strPath)};
    var output = [];
    listObjectsExec(output, params, resolve, reject);
  });
};
exports.deleteObject = function(strPath) {
  return new Promise(function(resolve, reject) {
    var params = {Bucket: cfgBucketName, Key: getFilePath(strPath)};
    s3Client.deleteObject(params, function(err, data) {
      if (err) {
        reject(err);
      } else {
        resolve(data);
      }
    });
  });
};
exports.deleteObjects = function(strPaths) {
  var aKeys = strPaths.map(function (currentValue) {
    return {Key: getFilePath(currentValue)};
  });
  var deletePromises = [];
  for (var i = 0; i < aKeys.length; i += MAX_DELETE_OBJECTS) {
    deletePromises.push(deleteObjectsHelp(aKeys.slice(i, i + MAX_DELETE_OBJECTS)));
  }
  return Promise.all(deletePromises);
};
exports.getSignedUrl = function(baseUrl, strPath, optUrlExpires, optFilename, opt_type) {
  return new Promise(function(resolve, reject) {
    var expires = optUrlExpires || 604800;
    var contentDisposition = utils.getContentDispositionS3(optFilename || path.basename(strPath), null, opt_type);
    if (cfgUseRequestToGetUrl) {
      //default Expires 900 seconds
      var params = {
        Bucket: cfgBucketName, Key: getFilePath(strPath), ResponseContentDisposition: contentDisposition, Expires: expires
      };
      s3Client.getSignedUrl('getObject', params, function(err, data) {
        if (err) {
          reject(err);
        } else {
          resolve(utils.changeOnlyOfficeUrl(data, strPath, optFilename));
        }
      });
    } else {
      var host;
      if (cfgRegion) {
        host = 'https://s3-'+cfgRegion+'.amazonaws.com';
      } else if (cfgEndpointParsed &&
        (cfgEndpointParsed.hostname == 'localhost' || cfgEndpointParsed.hostname == '127.0.0.1') &&
        80 == cfgEndpointParsed.port) {
        host = (cfgExternalHost ? cfgExternalHost : baseUrl) + cfgEndpointParsed.path;
      } else {
        host = cfgEndpoint;
      }
      if (host && host.length > 0 && '/' != host[host.length - 1]) {
        host += '/';
      }
      var newUrl;
      if (cfgUseSignedUrl) {
        //todo уйти от parse
        var hostParsed = url.parse(host);
        var protocol = hostParsed.protocol.substring(0, hostParsed.protocol.length - 1);
        var signerOptions = {
          host: hostParsed.hostname, port: hostParsed.port,
          protocol: protocol, useSubdomain: false
        };
        var awsUrlSigner = s3urlSigner.urlSigner(cfgAccessKeyId, cfgSecretAccessKey, signerOptions);
        newUrl = awsUrlSigner.getUrl('GET', getFilePath(strPath), cfgBucketName, expires, contentDisposition);
      } else {
        newUrl = host + cfgBucketName + '/' + cfgStorageFolderName + '/' + strPath;
      }
      resolve(utils.changeOnlyOfficeUrl(newUrl, strPath, optFilename));
    }
  });
};
\ No newline at end of file
+/*
 * (c) Copyright Ascensio System SIA 2010-2016
 *
 * This program is a free software product. You can redistribute it and/or
 * modify it under the terms of the GNU Affero General Public License (AGPL)
 * version 3 as published by the Free Software Foundation. In accordance with
 * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect
 * that Ascensio System SIA expressly excludes the warranty of non-infringement
 * of any third-party rights.
 *
 * This program is distributed WITHOUT ANY WARRANTY; without even the implied
 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE. For
 * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
 *
 * You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia,
 * EU, LV-1021.
 *
 * The  interactive user interfaces in modified source and object code versions
 * of the Program must display Appropriate Legal Notices, as required under
 * Section 5 of the GNU AGPL version 3.
 *
 * Pursuant to Section 7(b) of the License you must retain the original Product
 * logo when distributing the program. Pursuant to Section 7(e) we decline to
 * grant you any rights under trademark law for use of our trademarks.
 *
 * All the Product's GUI elements, including illustrations and icon sets, as
 * well as technical writing content are licensed under the terms of the
 * Creative Commons Attribution-ShareAlike 4.0 International. See the License
 * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
 *
 */

'use strict';
var url = require('url');
var path = require('path');
var AWS = require('aws-sdk');
var mime = require('mime');
var s3urlSigner = require('amazon-s3-url-signer');
var utils = require('./utils');

var configStorage = require('config').get('storage');
var cfgRegion = configStorage.get('region');
var cfgEndpoint = configStorage.get('endpoint');
var cfgBucketName = configStorage.get('bucketName');
var cfgStorageFolderName = configStorage.get('storageFolderName');
var cfgAccessKeyId = configStorage.get('accessKeyId');
var cfgSecretAccessKey = configStorage.get('secretAccessKey');
var cfgUseRequestToGetUrl = configStorage.get('useRequestToGetUrl');
var cfgUseSignedUrl = configStorage.get('useSignedUrl');
var cfgExternalHost = configStorage.get('externalHost');
/**
 * Don't hard-code your credentials!
 * Export the following environment variables instead:
 *
 * export AWS_ACCESS_KEY_ID='AKID'
 * export AWS_SECRET_ACCESS_KEY='SECRET'
 */
var configS3 = {
  region: cfgRegion,
  endpoint: cfgEndpoint,
  accessKeyId: cfgAccessKeyId,
  secretAccessKey: cfgSecretAccessKey
};
if (configS3.endpoint) {
  configS3.sslEnabled = false;
  configS3.s3ForcePathStyle = true;
}
AWS.config.update(configS3);
var s3Client = new AWS.S3();
if (configS3.endpoint) {
  s3Client.endpoint = new AWS.Endpoint(configS3.endpoint);
}
var cfgEndpointParsed = null;
if (cfgEndpoint) {
  cfgEndpointParsed = url.parse(cfgEndpoint);
}
//This operation enables you to delete multiple objects from a bucket using a single HTTP request. You may specify up to 1000 keys.
var MAX_DELETE_OBJECTS = 1000;

function getFilePath(strPath) {
  //todo
  return cfgStorageFolderName + '/' + strPath;
}
function joinListObjects(inputArray, outputArray) {
  var length = inputArray.length;
  for (var i = 0; i < length; i++) {
    outputArray.push(inputArray[i].Key.substring((cfgStorageFolderName + '/').length));
  }
}
function listObjectsExec(output, params, resolve, reject) {
  s3Client.listObjects(params, function(err, data) {
    if (err) {
      reject(err);
    } else {
      joinListObjects(data.Contents, output);
      if (data.IsTruncated && (data.NextMarker || data.Contents.length > 0)) {
        params.Marker = data.NextMarker || data.Contents[data.Contents.length - 1].Key;
        listObjectsExec(output, params, resolve, reject);
      } else {
        resolve(output);
      }
    }
  });
}
function mapDeleteObjects(currentValue) {
  return {Key: currentValue};
}
function deleteObjectsHelp(aKeys) {
  return new Promise(function(resolve, reject) {
    //By default, the operation uses verbose mode in which the response includes the result of deletion of each key in your request.
    //In quiet mode the response includes only keys where the delete operation encountered an error.
    var params = {Bucket: cfgBucketName, Delete: {Objects: aKeys, Quiet: true}};
    s3Client.deleteObjects(params, function(err, data) {
      if (err) {
        reject(err);
      } else {
        resolve(data);
      }
    });
  });
}

exports.getObject = function(strPath) {
  return new Promise(function(resolve, reject) {
    var params = {Bucket: cfgBucketName, Key: getFilePath(strPath)};
    s3Client.getObject(params, function(err, data) {
      if (err) {
        reject(err);
      } else {
        resolve(data.Body);
      }
    });
  });
};
exports.putObject = function(strPath, buffer, contentLength) {
  return new Promise(function(resolve, reject) {
    //todo рассмотреть Expires
    var params = {Bucket: cfgBucketName, Key: getFilePath(strPath), Body: buffer,
      ContentLength: contentLength, ContentType: mime.lookup(strPath)};
    s3Client.putObject(params, function(err, data) {
      if (err) {
        reject(err);
      } else {
        resolve(data);
      }
    });
  });
};
exports.listObjects = function(strPath) {
  return new Promise(function(resolve, reject) {
    var params = {Bucket: cfgBucketName, Prefix: getFilePath(strPath)};
    var output = [];
    listObjectsExec(output, params, resolve, reject);
  });
};
exports.deleteObject = function(strPath) {
  return new Promise(function(resolve, reject) {
    var params = {Bucket: cfgBucketName, Key: getFilePath(strPath)};
    s3Client.deleteObject(params, function(err, data) {
      if (err) {
        reject(err);
      } else {
        resolve(data);
      }
    });
  });
};
exports.deleteObjects = function(strPaths) {
  var aKeys = strPaths.map(function (currentValue) {
    return {Key: getFilePath(currentValue)};
  });
  var deletePromises = [];
  for (var i = 0; i < aKeys.length; i += MAX_DELETE_OBJECTS) {
    deletePromises.push(deleteObjectsHelp(aKeys.slice(i, i + MAX_DELETE_OBJECTS)));
  }
  return Promise.all(deletePromises);
};
exports.getSignedUrl = function(baseUrl, strPath, optUrlExpires, optFilename, opt_type) {
  return new Promise(function(resolve, reject) {
    var expires = optUrlExpires || 604800;
    var userFriendlyName = optFilename ? optFilename.replace(/\//g, "%2f") : path.basename(strPath);
    var contentDisposition = utils.getContentDispositionS3(userFriendlyName, null, opt_type);
    if (cfgUseRequestToGetUrl) {
      //default Expires 900 seconds
      var params = {
        Bucket: cfgBucketName, Key: getFilePath(strPath), ResponseContentDisposition: contentDisposition, Expires: expires
      };
      s3Client.getSignedUrl('getObject', params, function(err, data) {
        if (err) {
          reject(err);
        } else {
          resolve(utils.changeOnlyOfficeUrl(data, strPath, optFilename));
        }
      });
    } else {
      var host;
      if (cfgRegion) {
        host = 'https://s3-'+cfgRegion+'.amazonaws.com';
      } else if (cfgEndpointParsed &&
        (cfgEndpointParsed.hostname == 'localhost' || cfgEndpointParsed.hostname == '127.0.0.1') &&
        80 == cfgEndpointParsed.port) {
        host = (cfgExternalHost ? cfgExternalHost : baseUrl) + cfgEndpointParsed.path;
      } else {
        host = cfgEndpoint;
      }
      if (host && host.length > 0 && '/' != host[host.length - 1]) {
        host += '/';
      }
      var newUrl;
      if (cfgUseSignedUrl) {
        //todo уйти от parse
        var hostParsed = url.parse(host);
        var protocol = hostParsed.protocol.substring(0, hostParsed.protocol.length - 1);
        var signerOptions = {
          host: hostParsed.hostname, port: hostParsed.port,
          protocol: protocol, useSubdomain: false
        };
        var awsUrlSigner = s3urlSigner.urlSigner(cfgAccessKeyId, cfgSecretAccessKey, signerOptions);
        newUrl = awsUrlSigner.getUrl('GET', getFilePath(strPath), cfgBucketName, expires, contentDisposition);
      } else {
        newUrl = host + cfgBucketName + '/' + cfgStorageFolderName + '/' + strPath;
      }
      resolve(utils.changeOnlyOfficeUrl(newUrl, strPath, optFilename));
    }
  });
};
\ No newline at end of file

From 521da294c45e98363cea9f851816c1efff76bb3b Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Tue, 16 Aug 2016 18:08:29 +0300
Subject: [PATCH 18/34] config the waiting time to document assembly when all
 out(not 0 in case of F5 in the browser)

---
 DocService/sources/DocsCoServer.js | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js
index b392807b..9b6b60a5 100644
--- a/DocService/sources/DocsCoServer.js
+++ b/DocService/sources/DocsCoServer.js
@@ -49,7 +49,7 @@
  * d) Когда пользователь остается один, после принятия чужих изменений начинается пункт 'а'
  *-----------------------------------------------------------------------------------------------------------------------
  *--------------------------------------------Схема работы с сервером----------------------------------------------------
- * а) Когда все уходят, спустя время c_oAscSaveTimeOutDelay на сервер документов шлется команда на сборку.
+ * а) Когда все уходят, спустя время cfgAscSaveTimeOutDelay на сервер документов шлется команда на сборку.
  * b) Если приходит статус '1' на CommandService.ashx, то удалось сохранить и поднять версию. Очищаем callback-и и
  * 	изменения из базы и из памяти.
  * с) Если приходит статус, отличный от '1'(сюда можно отнести как генерацию файла, так и работа внешнего подписчика
@@ -95,6 +95,8 @@ var pubsubService = require('./' + config.get('pubsub.name'));
 var queueService = require('./../../Common/sources/taskqueueRabbitMQ');
 var cfgSpellcheckerUrl = config.get('server.editor_settings_spellchecker_url');
 var cfgCallbackRequestTimeout = config.get('server.callbackRequestTimeout');
+//The waiting time to document assembly when all out(not 0 in case of F5 in the browser)
+var cfgAscSaveTimeOutDelay = config.get('server.savetimeoutdelay');
 
 var cfgPubSubMaxChanges = config.get('pubsub.maxChanges');
 
@@ -183,7 +185,6 @@ var c_oAscChangeBase = {
   All: 2
 };
 
-var c_oAscSaveTimeOutDelay = 5000;	// Время ожидания для сохранения на сервере (для отработки F5 в браузере)
 var c_oAscLockTimeOutDelay = 500;	// Время ожидания для сохранения, когда зажата база данных
 
 var c_oAscRecalcIndexTypes = {
@@ -755,7 +756,7 @@ function* _createSaveTimer(docId, opt_userId, opt_queue, opt_noDelay) {
   var updateIfRes = yield taskResult.updateIf(updateTask, updateMask);
   if (updateIfRes.affectedRows > 0) {
     if(!opt_noDelay){
-      yield utils.sleep(c_oAscSaveTimeOutDelay);
+      yield utils.sleep(cfgAscSaveTimeOutDelay);
     }
     while (true) {
       if (!sqlBase.isLockCriticalSection(docId)) {

From 3ada8dc440e1ebcc4a93c6e33e2c06969cbbe913 Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Wed, 17 Aug 2016 12:29:40 +0300
Subject: [PATCH 19/34] miss config savetimeoutdelay

---
 Common/config/default.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Common/config/default.json b/Common/config/default.json
index f5f4cd13..25fd3bc4 100644
--- a/Common/config/default.json
+++ b/Common/config/default.json
@@ -52,7 +52,8 @@
 				"limits_image_download_timeout": 120,
 				"editor_settings_spellchecker_url": "/spellchecker",
 				"callbackRequestTimeout": 120,
-				"healthcheckfilepath": "../public/healthcheck.docx"
+				"healthcheckfilepath": "../public/healthcheck.docx",
+				"savetimeoutdelay": 5000
 			},
 			"utils": {
 				"utils_common_fontdir": "null",

From f74fdce0cc276866e3824a61f4f2a7ab7731197f Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Thu, 18 Aug 2016 16:26:30 +0300
Subject: [PATCH 20/34] ipfilter

---
 Common/config/default.json         |  5 ++
 Common/package.json                |  1 +
 Common/sources/utils.js            | 32 +++++++++++-
 DocService/package.json            |  2 +
 DocService/sources/DocsCoServer.js | 82 ++++++++++++++++++------------
 DocService/sources/server.js       | 40 ++++++++++++---
 FileConverter/sources/converter.js | 39 ++++++++------
 7 files changed, 143 insertions(+), 58 deletions(-)

diff --git a/Common/config/default.json b/Common/config/default.json
index 25fd3bc4..acb0159c 100644
--- a/Common/config/default.json
+++ b/Common/config/default.json
@@ -99,6 +99,11 @@
 				"files": 604800,
 				"filesCron": "00 00 */1 * * *",
 				"filesremovedatonce": 10
+			},
+			"ipfilter": {
+				"rules": [[true, "*"]],
+				"useforrequest": false,
+				"errorcode": 401
 			}
 		}
 	},
diff --git a/Common/package.json b/Common/package.json
index c979eb1f..cde73a51 100644
--- a/Common/package.json
+++ b/Common/package.json
@@ -9,6 +9,7 @@
     "aws-sdk": "^2.4.12",
     "co": "^4.6.0",
     "config": "^1.21.0",
+    "escape-string-regexp": "^1.0.5",
     "log4js": "^0.6.38",
     "mime": "^1.3.4",
     "mkdirp": "^0.5.1",
diff --git a/Common/sources/utils.js b/Common/sources/utils.js
index ae7bf5a7..4527d20e 100644
--- a/Common/sources/utils.js
+++ b/Common/sources/utils.js
@@ -29,17 +29,33 @@
  * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
  *
  */
-
+var config = require('config');
 var fs = require('fs');
 var path = require('path');
 var url = require('url');
 var request = require('request');
 var co = require('co');
 var URI = require("uri-js");
+const escapeStringRegexp = require('escape-string-regexp');
 var constants = require('./constants');
 
+var configIpFilter = config.get('services.CoAuthoring.ipfilter');
+var cfgIpFilterRules = configIpFilter.get('rules');
+var cfgIpFilterErrorCode = configIpFilter.get('errorcode');
+
 var ANDROID_SAFE_FILENAME = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ._-+,@£$€!½§~\'=()[]{}0123456789';
 
+var g_oIpFilterRules = function() {
+  var res = [];
+  for (var i = 0; i < cfgIpFilterRules.length; ++i) {
+    var rule = cfgIpFilterRules[i];
+    var regExpStr = rule[1].split('*').map(escapeStringRegexp).join('.*');
+    var exp = new RegExp("^" + regExpStr + "$", 'i');
+    res.push({allow: rule[0], exp: exp});
+  }
+  return res;
+}();
+
 exports.addSeconds = function(date, sec) {
   date.setSeconds(date.getSeconds() + sec);
 };
@@ -495,3 +511,17 @@ function* pipeFiles(from, to) {
   yield pipeStreams(fromStream, toStream, true);
 }
 exports.pipeFiles = co.wrap(pipeFiles);
+function checkIpFilter(hostname) {
+  var status = 0;
+  for (var i = 0; i < g_oIpFilterRules.length; ++i) {
+    var rule = g_oIpFilterRules[i];
+    if (rule.exp.test(hostname)) {
+      if (!rule.allow) {
+        status = cfgIpFilterErrorCode;
+      }
+      break;
+    }
+  }
+  return status;
+}
+exports.checkIpFilter = checkIpFilter;
diff --git a/DocService/package.json b/DocService/package.json
index e3809e17..041a4217 100644
--- a/DocService/package.json
+++ b/DocService/package.json
@@ -11,6 +11,8 @@
     "cron": "^1.1.0",
     "express": "^4.14.0",
     "fakeredis": "^1.0.3",
+    "forwarded": "^0.1.0",
+    "ipaddr.js": "^1.2.0",
     "mime": "^1.3.4",
     "multiparty": "^4.1.2",
     "mysql": "^2.11.1",
diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js
index 9b6b60a5..4e6edc6a 100644
--- a/DocService/sources/DocsCoServer.js
+++ b/DocService/sources/DocsCoServer.js
@@ -716,6 +716,13 @@ function* bindEvents(docId, callback, baseUrl, opt_userAction, opt_userData) {
   } else {
     oCallbackUrl = parseUrl(callback);
     bChangeBase = c_oAscChangeBase.All;
+    if (null !== oCallbackUrl) {
+      if (utils.checkIpFilter(oCallbackUrl.host) > 0) {
+        logger.error('checkIpFilter error: docId = %s;url = %s', docId, callback);
+        //todo add new error type
+        oCallbackUrl = null;
+      }
+    }
   }
   if (null === oCallbackUrl) {
     return commonDefines.c_oAscServerCommandErrors.ParseError;
@@ -1378,8 +1385,8 @@ exports.install = function(server, callbackFunction) {
         }
       } else {
         conn.sessionId = conn.id;
-        yield* endAuth(conn, false, data.documentCallbackUrl);
-        if (cmd) {
+        var endAuthRes = yield* endAuth(conn, false, data.documentCallbackUrl);
+        if (endAuthRes && cmd) {
           yield canvasService.openDocument(conn, cmd, upsertRes);
         }
       }
@@ -1387,6 +1394,7 @@ exports.install = function(server, callbackFunction) {
   }
 
   function* endAuth(conn, bIsRestore, documentCallbackUrl) {
+    var res = true;
     var docId = conn.docId;
     var tmpUser = conn.user;
     connections.push(conn);
@@ -1404,48 +1412,56 @@ exports.install = function(server, callbackFunction) {
     }
 
     // Отправляем на внешний callback только для тех, кто редактирует
+    var bindEventsRes = commonDefines.c_oAscServerCommandErrors.NoError;
     if (!tmpUser.view) {
       var userAction = new commonDefines.OutputAction(commonDefines.c_oAscUserAction.In, tmpUser.idOriginal);
       // Если пришла информация о ссылке для посылания информации, то добавляем
       if (documentCallbackUrl) {
-        yield* bindEvents(docId, documentCallbackUrl, conn.baseUrl, userAction);
+        bindEventsRes = yield* bindEvents(docId, documentCallbackUrl, conn.baseUrl, userAction);
       } else {
         yield* sendStatusDocument(docId, c_oAscChangeBase.No, userAction);
       }
     }
-    var lockDocument = null;
-    if (!bIsRestore && 2 === countNoView && !tmpUser.view) {
-      // Ставим lock на документ
-      var isLock = yield utils.promiseRedis(redisClient, redisClient.setnx,
-          redisKeyLockDoc + docId, JSON.stringify(firstParticipantNoView));
-      if(isLock) {
-        lockDocument = firstParticipantNoView;
-        yield utils.promiseRedis(redisClient, redisClient.expire, redisKeyLockDoc + docId, cfgExpLockDoc);
-      }
-    }
-    if (!lockDocument) {
-      var getRes = yield utils.promiseRedis(redisClient, redisClient.get, redisKeyLockDoc + docId);
-      if (getRes) {
-        lockDocument = JSON.parse(getRes);
-      }
-    }
 
-    if (lockDocument && !tmpUser.view) {
-      // Для view не ждем снятия lock-а
-      var sendObject = {
-        type: "waitAuth",
-        lockDocument: lockDocument
-      };
-      sendData(conn, sendObject);//Or 0 if fails
-    } else {
-      if (bIsRestore) {
-        yield* sendAuthInfo(undefined, undefined, conn, participantsMap);
-      } else {
-        var objChangesDocument = yield* getDocumentChanges(docId);
-        yield* sendAuthInfo(objChangesDocument.arrChanges, objChangesDocument.getLength(), conn, participantsMap);
+    if (commonDefines.c_oAscServerCommandErrors.NoError === bindEventsRes) {
+      var lockDocument = null;
+      if (!bIsRestore && 2 === countNoView && !tmpUser.view) {
+        // Ставим lock на документ
+        var isLock = yield utils.promiseRedis(redisClient, redisClient.setnx,
+                                              redisKeyLockDoc + docId, JSON.stringify(firstParticipantNoView));
+        if (isLock) {
+          lockDocument = firstParticipantNoView;
+          yield utils.promiseRedis(redisClient, redisClient.expire, redisKeyLockDoc + docId, cfgExpLockDoc);
+        }
       }
+      if (!lockDocument) {
+        var getRes = yield utils.promiseRedis(redisClient, redisClient.get, redisKeyLockDoc + docId);
+        if (getRes) {
+          lockDocument = JSON.parse(getRes);
+        }
+      }
+
+      if (lockDocument && !tmpUser.view) {
+        // Для view не ждем снятия lock-а
+        var sendObject = {
+          type: "waitAuth",
+          lockDocument: lockDocument
+        };
+        sendData(conn, sendObject);//Or 0 if fails
+      } else {
+        if (bIsRestore) {
+          yield* sendAuthInfo(undefined, undefined, conn, participantsMap);
+        } else {
+          var objChangesDocument = yield* getDocumentChanges(docId);
+          yield* sendAuthInfo(objChangesDocument.arrChanges, objChangesDocument.getLength(), conn, participantsMap);
+        }
+      }
+      yield* publish({type: commonDefines.c_oPublishType.participantsState, docId: docId, user: tmpUser, state: true}, docId, tmpUser.id);
+    } else {
+      sendFileError(conn, 'ip filter');
+      res = false;
     }
-    yield* publish({type: commonDefines.c_oPublishType.participantsState, docId: docId, user: tmpUser, state: true}, docId, tmpUser.id);
+    return res;
   }
 
   function* sendAuthInfo(objChangesDocument, changesIndex, conn, participantsMap) {
diff --git a/DocService/sources/server.js b/DocService/sources/server.js
index 98c0f4f6..21451718 100644
--- a/DocService/sources/server.js
+++ b/DocService/sources/server.js
@@ -104,6 +104,8 @@ if (cluster.isMaster) {
   const path = require('path');
   const bodyParser = require("body-parser");
   const mime = require('mime');
+  const forwarded = require('forwarded');
+  const ipaddr = require('ipaddr.js');
   const docsCoServer = require('./DocsCoServer');
   const canvasService = require('./canvasservice');
   const converterService = require('./converterservice');
@@ -112,6 +114,8 @@ if (cluster.isMaster) {
   const constants = require('./../../Common/sources/constants');
   const utils = require('./../../Common/sources/utils');
   const configStorage = configCommon.get('storage');
+  var configIpFilter = configCommon.get('services.CoAuthoring.ipfilter');
+  var cfgIpFilterEseForRequest = configIpFilter.get('useforrequest');
   const app = express();
   var server = null;
 
@@ -158,7 +162,27 @@ if (cluster.isMaster) {
       }
     });
   }
-
+  function checkClientIp(req, res, next) {
+    var status = 0;
+    if (cfgIpFilterEseForRequest) {
+      var addresses = forwarded(req);
+      var ipString = addresses[addresses.length - 1];
+      //IPv6 -> IPv4
+      if (ipaddr.IPv6.isValid(ipString)) {
+        var ip = ipaddr.IPv6.parse(ipString);
+        if (ip.isIPv4MappedAddress()) {
+          ipString = ip.toIPv4Address().toString();
+        }
+      }
+      console.log('ipString'+ipString);
+      status = utils.checkIpFilter(ipString);
+    }
+    if (status > 0) {
+      res.sendStatus(status);
+    } else {
+      next();
+    }
+  }
   // Если захочется использовать 'development' и 'production',
   // то с помощью app.settings.env (https://github.com/strongloop/express/issues/936)
   // Если нужна обработка ошибок, то теперь она такая https://github.com/expressjs/errorhandler
@@ -171,8 +195,8 @@ if (cluster.isMaster) {
       res.send('Server is functioning normally. Version: ' + docsCoServer.version);
     });
 
-    app.get('/coauthoring/CommandService.ashx', docsCoServer.commandFromServer);
-    app.post('/coauthoring/CommandService.ashx', docsCoServer.commandFromServer);
+    app.get('/coauthoring/CommandService.ashx', checkClientIp, docsCoServer.commandFromServer);
+    app.post('/coauthoring/CommandService.ashx', checkClientIp, docsCoServer.commandFromServer);
 
     if (config.has('server.fonts_route')) {
       var fontsRoute = config.get('server.fonts_route');
@@ -181,12 +205,12 @@ if (cluster.isMaster) {
       app.get('/' + fontsRoute + 'odttf/:fontname', fontService.getFont);
     }
 
-    app.get('/ConvertService.ashx', converterService.convert);
-    app.post('/ConvertService.ashx', converterService.convert);
+    app.get('/ConvertService.ashx', checkClientIp, converterService.convert);
+    app.post('/ConvertService.ashx', checkClientIp, converterService.convert);
 
     var rawFileParser = bodyParser.raw({ inflate: true, limit: config.get('server.limits_tempfile_upload'), type: '*/*' });
-    app.get('/FileUploader.ashx', rawFileParser, fileUploaderService.uploadTempFile);
-    app.post('/FileUploader.ashx', rawFileParser, fileUploaderService.uploadTempFile);
+    app.get('/FileUploader.ashx', checkClientIp, rawFileParser, fileUploaderService.uploadTempFile);
+    app.post('/FileUploader.ashx', checkClientIp, rawFileParser, fileUploaderService.uploadTempFile);
 
     var docIdRegExp = new RegExp("^[" + constants.DOC_ID_PATTERN + "]*$", 'i');
     app.param('docid', (req, res, next, val) => {
@@ -207,7 +231,7 @@ if (cluster.isMaster) {
     app.post('/upload/:docid/:userid/:index/:vkey?', rawFileParser, fileUploaderService.uploadImageFile);
 
     app.post('/downloadas/:docid', rawFileParser, canvasService.downloadAs);
-    app.get('/healthcheck', converterService.convertHealthCheck);
+    app.get('/healthcheck', checkClientIp, converterService.convertHealthCheck);
   });
 
   process.on('message', (msg) => {
diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js
index c5982b51..a712027e 100644
--- a/FileConverter/sources/converter.js
+++ b/FileConverter/sources/converter.js
@@ -166,24 +166,31 @@ function* downloadFile(docId, uri, fileFrom) {
   var res = false;
   var data = null;
   var downloadAttemptCount = 0;
-  while (!res && downloadAttemptCount++ < cfgDownloadAttemptMaxCount) {
-    try {
-      data = yield utils.downloadUrlPromise(uri, cfgDownloadTimeout * 1000, cfgDownloadMaxBytes);
-      res = true;
-    } catch (err) {
-      res = false;
-      logger.error('error downloadFile:url=%s;attempt=%d;code:%s;connect:%s;(id=%s)\r\n%s', uri, downloadAttemptCount, err.code, err.connect, docId, err.stack);
-      //not continue attempts if timeout
-      if (err.code === 'ETIMEDOUT' || err.code === 'EMSGSIZE') {
-        break;
-      } else {
-        yield utils.sleep(cfgDownloadAttemptDelay);
+  var urlParsed = url.parse(uri);
+  var filterStatus = utils.checkIpFilter(urlParsed.hostname);
+  if (0 == filterStatus) {
+    while (!res && downloadAttemptCount++ < cfgDownloadAttemptMaxCount) {
+      try {
+        data = yield utils.downloadUrlPromise(uri, cfgDownloadTimeout * 1000, cfgDownloadMaxBytes);
+        res = true;
+      } catch (err) {
+        res = false;
+        logger.error('error downloadFile:url=%s;attempt=%d;code:%s;connect:%s;(id=%s)\r\n%s', uri, downloadAttemptCount, err.code, err.connect, docId, err.stack);
+        //not continue attempts if timeout
+        if (err.code === 'ETIMEDOUT' || err.code === 'EMSGSIZE') {
+          break;
+        } else {
+          yield utils.sleep(cfgDownloadAttemptDelay);
+        }
       }
     }
-  }
-  if (res) {
-    logger.debug('downloadFile complete(id=%s)', docId);
-    fs.writeFileSync(fileFrom, data);
+    if (res) {
+      logger.debug('downloadFile complete(id=%s)', docId);
+      fs.writeFileSync(fileFrom, data);
+    }
+  } else {
+    logger.error('checkIpFilter error:url=%s;code:%s;(id=%s)', uri, filterStatus, docId);
+    res = false;
   }
   return res;
 }

From f4e2432c5bc0b52716e8a8562fe607ff5265682c Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Fri, 19 Aug 2016 13:31:05 +0300
Subject: [PATCH 21/34] change ipfilter rules format

---
 Common/config/default.json | 2 +-
 Common/sources/utils.js    | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Common/config/default.json b/Common/config/default.json
index acb0159c..71f7f7d8 100644
--- a/Common/config/default.json
+++ b/Common/config/default.json
@@ -101,7 +101,7 @@
 				"filesremovedatonce": 10
 			},
 			"ipfilter": {
-				"rules": [[true, "*"]],
+				"rules": [{"address": "*", "allowed": true}],
 				"useforrequest": false,
 				"errorcode": 401
 			}
diff --git a/Common/sources/utils.js b/Common/sources/utils.js
index 4527d20e..d2a53305 100644
--- a/Common/sources/utils.js
+++ b/Common/sources/utils.js
@@ -49,9 +49,9 @@ var g_oIpFilterRules = function() {
   var res = [];
   for (var i = 0; i < cfgIpFilterRules.length; ++i) {
     var rule = cfgIpFilterRules[i];
-    var regExpStr = rule[1].split('*').map(escapeStringRegexp).join('.*');
-    var exp = new RegExp("^" + regExpStr + "$", 'i');
-    res.push({allow: rule[0], exp: exp});
+    var regExpStr = rule['address'].split('*').map(escapeStringRegexp).join('.*');
+    var exp = new RegExp('^' + regExpStr + '$', 'i');
+    res.push({allow: rule['allowed'], exp: exp});
   }
   return res;
 }();

From 65fc7a02d2afb54c63e66e92bcfcfe5e9e852159 Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Thu, 25 Aug 2016 18:58:13 +0300
Subject: [PATCH 22/34] add postgresql support

---
 Common/config/default.json                    |   7 +-
 Common/config/development-mac.json            |   6 +
 Common/config/development-windows.json        |   6 +
 DocService/package.json                       |   3 +-
 DocService/sources/baseConnector.js           |  32 +----
 DocService/sources/mySqlBaseConnector.js      |   2 +-
 DocService/sources/postgreSqlBaseConnector.js | 123 +++++++++++++++---
 DocService/sources/taskresult.js              |  31 +----
 schema/postgresql/createdb.sql                |  52 ++++++++
 schema/postgresql/removedb.sql                |   1 +
 10 files changed, 177 insertions(+), 86 deletions(-)
 create mode 100644 schema/postgresql/createdb.sql
 create mode 100644 schema/postgresql/removedb.sql

diff --git a/Common/config/default.json b/Common/config/default.json
index 71f7f7d8..b0d1933d 100644
--- a/Common/config/default.json
+++ b/Common/config/default.json
@@ -63,16 +63,17 @@
 				"limits_image_types_copy": "jpg;png;gif;bmp;emf;wmf;svg;txt;bin"
 			},
 			"sql": {
-				"type": "mysql",
+				"type": "postgres",
 				"tableChanges": "doc_changes",
 				"tableCallbacks": "doc_callbacks",
 				"tableResult": "task_result",
 				"dbHost": "localhost",
-				"dbPort": 3306,
+				"dbPort": 5432,
 				"dbName": "onlyoffice",
-				"dbUser": "root",
+				"dbUser": "onlyoffice",
 				"dbPass": "onlyoffice",
 				"charset": "utf8",
+				"connectionlimit": 10,
 				"max_allowed_packet": 1048575
 			},
 			"redis": {
diff --git a/Common/config/development-mac.json b/Common/config/development-mac.json
index 836cd9a4..bf7b6472 100644
--- a/Common/config/development-mac.json
+++ b/Common/config/development-mac.json
@@ -32,6 +32,12 @@
       },
       "utils": {
         "utils_common_fontdir": "/Library/Fonts"
+      },
+      "sql": {
+        "type": "mysql",
+        "dbPort": 3306,
+        "dbUser": "root",
+        "dbPass": "onlyoffice"
       }
     }
   },
diff --git a/Common/config/development-windows.json b/Common/config/development-windows.json
index d799da0a..7e77fff2 100644
--- a/Common/config/development-windows.json
+++ b/Common/config/development-windows.json
@@ -32,6 +32,12 @@
       },
       "utils": {
         "utils_common_fontdir": "C:\\Windows\\Fonts"
+      },
+      "sql": {
+        "type": "mysql",
+        "dbPort": 3306,
+        "dbUser": "root",
+        "dbPass": "onlyoffice"
       }
     }
   },
diff --git a/DocService/package.json b/DocService/package.json
index 041a4217..cfb7d4e9 100644
--- a/DocService/package.json
+++ b/DocService/package.json
@@ -16,7 +16,8 @@
     "mime": "^1.3.4",
     "multiparty": "^4.1.2",
     "mysql": "^2.11.1",
-    "pg": "^6.0.3",
+    "pg": "^6.1.0",
+    "pg-escape": "^0.2.0",
     "redis": "^2.6.2",
     "sockjs": "http://d2ettrnqo7v976.cloudfront.net/npm/sockjs-node/v0.3.15.14.tar.gz",
     "underscore": "^1.8.3"
diff --git a/DocService/sources/baseConnector.js b/DocService/sources/baseConnector.js
index 885c6519..8c7bd774 100644
--- a/DocService/sources/baseConnector.js
+++ b/DocService/sources/baseConnector.js
@@ -81,37 +81,9 @@ exports.loadTable = function (tableId, callbackFunction) {
 	var sqlCommand = "SELECT * FROM " + table + ";";
 	baseConnector.sqlQuery(sqlCommand, callbackFunction);
 };
-exports.upsertInTable = function (tableId, toInsert, toUpdate, callbackFunction) {
-  var table = getTableById(tableId);
-  var sqlCommand = "INSERT INTO " + table + " VALUES (";
-  for (var i = 0, l = toInsert.length; i < l; ++i) {
-    sqlCommand += baseConnector.sqlEscape(toInsert[i]);
-    if (i !== l - 1)
-      sqlCommand += ",";
-  }
-  sqlCommand += ") ON DUPLICATE KEY UPDATE ";
-  for (var i = 0, l = toUpdate.length; i + 1 < l; i += 2) {
-    sqlCommand += toUpdate[i] + "=" + baseConnector.sqlEscape(toUpdate[i+1]);
-    if (i + 1 !== l - 1)
-      sqlCommand += ",";
-  }
-  sqlCommand += ";";
-  baseConnector.sqlQuery(sqlCommand, callbackFunction);
-};
-exports.upsertInTablePromise = function (tableId, toInsert, toUpdate) {
-  return new Promise(function(resolve, reject) {
-    exports.upsertInTable(tableId, toInsert, toUpdate, function(error, result) {
-      if (error) {
-        reject(error);
-      } else {
-        resolve(result);
-      }
-    });
-  });
-};
 exports.insertCallback = function(id, href, baseUrl, callbackFunction) {
-  var sqlCommand = "INSERT IGNORE INTO " + tableCallbacks + " VALUES (" + baseConnector.sqlEscape(id) + "," +
-    baseConnector.sqlEscape(href) + "," + baseConnector.sqlEscape(baseUrl) + ");";
+  var sqlCommand = "INSERT " + baseConnector.ignoreStr + " INTO " + tableCallbacks + " VALUES (" + baseConnector.sqlEscape(id) + "," +
+    baseConnector.sqlEscape(href) + "," + baseConnector.sqlEscape(baseUrl) + ") " + baseConnector.doNothingStr + ";";
 
   baseConnector.sqlQuery(sqlCommand, callbackFunction);
 };
diff --git a/DocService/sources/mySqlBaseConnector.js b/DocService/sources/mySqlBaseConnector.js
index ad576b15..a872b38e 100644
--- a/DocService/sources/mySqlBaseConnector.js
+++ b/DocService/sources/mySqlBaseConnector.js
@@ -1 +1 @@
-/*
 * (c) Copyright Ascensio System SIA 2010-2016
 *
 * This program is a free software product. You can redistribute it and/or
 * modify it under the terms of the GNU Affero General Public License (AGPL)
 * version 3 as published by the Free Software Foundation. In accordance with
 * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect
 * that Ascensio System SIA expressly excludes the warranty of non-infringement
 * of any third-party rights.
 *
 * This program is distributed WITHOUT ANY WARRANTY; without even the implied
 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE. For
 * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
 *
 * You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia,
 * EU, LV-1021.
 *
 * The  interactive user interfaces in modified source and object code versions
 * of the Program must display Appropriate Legal Notices, as required under
 * Section 5 of the GNU AGPL version 3.
 *
 * Pursuant to Section 7(b) of the License you must retain the original Product
 * logo when distributing the program. Pursuant to Section 7(e) we decline to
 * grant you any rights under trademark law for use of our trademarks.
 *
 * All the Product's GUI elements, including illustrations and icon sets, as
 * well as technical writing content are licensed under the terms of the
 * Creative Commons Attribution-ShareAlike 4.0 International. See the License
 * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
 *
 */

var mysql = require('mysql');
var configSql = require('config').get('services.CoAuthoring.sql');
var pool  = mysql.createPool({
	host		: configSql.get('dbHost'),
	port		: configSql.get('dbPort'),
	user		: configSql.get('dbUser'),
	password	: configSql.get('dbPass'),
	database	: configSql.get('dbName'),
	charset		: configSql.get('charset'),
	timezone	: '+0000',
	flags : '-FOUND_ROWS'
});

var logger = require('./../../Common/sources/logger');

exports.sqlQuery = function (sqlCommand, callbackFunction) {
	pool.getConnection(function(err, connection) {
		if (err) {
			logger.error('pool.getConnection error: %s', err);
			if (callbackFunction) callbackFunction(err, null);
			return;
		}

		connection.query(sqlCommand, function (error, result) {
			connection.release();
			if (error) {
				logger.error('________________________error_____________________');
				logger.error('sqlQuery: %s sqlCommand: %s', error.code, sqlCommand);
				logger.error(error);
				logger.error('_____________________end_error_____________________');
			}
			if (callbackFunction) callbackFunction(error, result);
		});
	});
};
exports.sqlEscape = function (value) {
	return pool.escape(value);
};
\ No newline at end of file
+/*
 * (c) Copyright Ascensio System SIA 2010-2016
 *
 * This program is a free software product. You can redistribute it and/or
 * modify it under the terms of the GNU Affero General Public License (AGPL)
 * version 3 as published by the Free Software Foundation. In accordance with
 * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect
 * that Ascensio System SIA expressly excludes the warranty of non-infringement
 * of any third-party rights.
 *
 * This program is distributed WITHOUT ANY WARRANTY; without even the implied
 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE. For
 * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
 *
 * You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia,
 * EU, LV-1021.
 *
 * The  interactive user interfaces in modified source and object code versions
 * of the Program must display Appropriate Legal Notices, as required under
 * Section 5 of the GNU AGPL version 3.
 *
 * Pursuant to Section 7(b) of the License you must retain the original Product
 * logo when distributing the program. Pursuant to Section 7(e) we decline to
 * grant you any rights under trademark law for use of our trademarks.
 *
 * All the Product's GUI elements, including illustrations and icon sets, as
 * well as technical writing content are licensed under the terms of the
 * Creative Commons Attribution-ShareAlike 4.0 International. See the License
 * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
 *
 */

var mysql = require('mysql');
var sqlBase = require('./baseConnector');
var configSql = require('config').get('services.CoAuthoring.sql');
var pool  = mysql.createPool({
	host		: configSql.get('dbHost'),
	port		: configSql.get('dbPort'),
	user		: configSql.get('dbUser'),
	password	: configSql.get('dbPass'),
	database	: configSql.get('dbName'),
	charset		: configSql.get('charset'),
	connectionLimit	: configSql.get('connectionlimit'),
	timezone	: '+0000',
	flags : '-FOUND_ROWS'
});

var logger = require('./../../Common/sources/logger');

exports.sqlQuery = function (sqlCommand, callbackFunction) {
	pool.getConnection(function(err, connection) {
		if (err) {
			logger.error('pool.getConnection error: %s', err);
			if (callbackFunction) callbackFunction(err, null);
			return;
		}

		connection.query(sqlCommand, function (error, result) {
			connection.release();
			if (error) {
				logger.error('________________________error_____________________');
				logger.error('sqlQuery: %s sqlCommand: %s', error.code, sqlCommand);
				logger.error(error);
				logger.error('_____________________end_error_____________________');
			}
			if (callbackFunction) callbackFunction(error, result);
		});
	});
};
exports.sqlEscape = function (value) {
	return pool.escape(value);
};

exports.ignoreStr = 'IGNORE';
exports.doNothingStr = '';

function getUpsertString(task, opt_updateUserIndex) {
	task.completeDefaults();
	var dateNow = sqlBase.getDateTime(new Date());
	var commandArg = [task.key, task.status, task.statusInfo, dateNow, task.title, task.userIndex, task.changeId];
	var commandArgEsc = commandArg.map(function(curVal) {
		return exports.sqlEscape(curVal)
	});
	var sql = 'INSERT INTO task_result ( id, status, status_info, last_open_date, title,' +
		' user_index, change_id  ) VALUES (' + commandArgEsc.join(', ') + ') ON DUPLICATE KEY UPDATE' +
		' last_open_date = ' + exports.sqlEscape(dateNow);
	if (opt_updateUserIndex) {
		sql += ', user_index = LAST_INSERT_ID(user_index + 1);';
	} else {
		sql += ';';
	}
	return sql;
}

exports.upsert = function(task, opt_updateUserIndex) {
	return new Promise(function(resolve, reject) {
		var sqlCommand = getUpsertString(task, opt_updateUserIndex);
		exports.sqlQuery(sqlCommand, function(error, result) {
			if (error) {
				reject(error);
			} else {
				resolve(result);
			}
		});
	});
};
\ No newline at end of file
diff --git a/DocService/sources/postgreSqlBaseConnector.js b/DocService/sources/postgreSqlBaseConnector.js
index 3fbc761b..2757afbf 100644
--- a/DocService/sources/postgreSqlBaseConnector.js
+++ b/DocService/sources/postgreSqlBaseConnector.js
@@ -31,29 +31,110 @@
  */
 
 var pg = require('pg');
+var co = require('co');
+var pgEscape = require('pg-escape');
+var types = require('pg').types;
+var sqlBase = require('./baseConnector');
 var configSql = require('config').get('services.CoAuthoring.sql');
-var connectionString = 'postgres://' + configSql.get('dbUser') + ':' + configSql.get('dbPass') + '@' + configSql.get('dbHost') +
-	(configSql.get('dbPort') ? (':' + configSql.get('dbPort')) : '') + '/' + configSql.get('dbName');
+var pool = new pg.Pool({
+  host: configSql.get('dbHost'),
+  port: configSql.get('dbPort'),
+  user: configSql.get('dbUser'),
+  password: configSql.get('dbPass'),
+  database: configSql.get('dbName'),
+  max: configSql.get('connectionlimit'),
+  min: 0,
+  ssl: false,
+  idleTimeoutMillis: 30000
+});
+//todo datetime timezone
+types.setTypeParser(1114, function(stringValue) {
+  return new Date(stringValue + '+0000');
+});
+types.setTypeParser(1184, function(stringValue) {
+  return new Date(stringValue + '+0000');
+});
 
 var logger = require('./../../Common/sources/logger');
 
-exports.sqlQuery = function (sqlCommand, callbackFunction) {
-	pg.connect(connectionString, function (err, connection, done) {
-		if(err) {
-			logger.error('pool.getConnection error: %s', err);
-			if (callbackFunction) callbackFunction(err, null);
-			return;
-		}
-
-		connection.query(sqlCommand, function (error, result) {
-			//call `done()` to release the client back to the pool
-			done();
-
-			if (error) logger.error('sqlQuery: %s sqlCommand: %s', error.message, sqlCommand.slice(0, 50));
-			if (callbackFunction) callbackFunction(error, result ? result.rows : result);
-		});
-	});
+exports.sqlQuery = function(sqlCommand, callbackFunction, opt_noModifyRes) {
+  co(function *() {
+    var client = null;
+    var result = null;
+    var error = null;
+    try {
+      client = yield pool.connect();
+      result = yield client.query(sqlCommand);
+    } catch (err) {
+      error = err;
+      if (client) {
+        logger.error('sqlQuery error sqlCommand: %s:\r\n%s', sqlCommand.slice(0, 50), err.stack);
+      } else {
+        logger.error('pool.getConnection error: %s', err);
+      }
+    } finally {
+      if (client) {
+        client.release();
+      }
+      if (callbackFunction) {
+        var output = result;
+        if (result && !opt_noModifyRes) {
+          if ('SELECT' === result.command) {
+            output = result.rows;
+          } else {
+            output = {affectedRows: result.rowCount};
+          }
+        }
+        callbackFunction(error, output);
+      }
+    }
+  });
+};
+exports.sqlEscape = function(value) {
+  //todo parameterized queries
+  return undefined !== value ? pgEscape.literal(value.toString()) : 'NULL';
+};
+exports.ignoreStr = '';
+exports.doNothingStr = 'ON CONFLICT DO NOTHING';
+
+function getUpsertString(task, opt_updateUserIndex) {
+  task.completeDefaults();
+  var dateNow = sqlBase.getDateTime(new Date());
+  var commandArg = [task.key, task.status, task.statusInfo, dateNow, task.title, task.userIndex, task.changeId];
+  var commandArgEsc = commandArg.map(function(curVal) {
+    return exports.sqlEscape(curVal)
+  });
+  //http://stackoverflow.com/questions/34762732/how-to-find-out-if-an-upsert-was-an-update-with-postgresql-9-5-upsert
+  var sql = "INSERT INTO task_result (id, status, status_info, last_open_date, title, user_index, change_id) SELECT " +
+    commandArgEsc.join(', ') +
+    " WHERE 'false' = set_config('myapp.isupdate', 'false', true) ON CONFLICT (id) DO UPDATE SET  last_open_date = " +
+    sqlBase.baseConnector.sqlEscape(dateNow);
+  if (opt_updateUserIndex) {
+    sql += ', user_index = task_result.user_index + 1';
+  }
+  sql +=
+    " WHERE 'true' = set_config('myapp.isupdate', 'true', true) RETURNING current_setting('myapp.isupdate') as update";
+  if (opt_updateUserIndex) {
+    sql += ', user_index';
+  }
+  sql += ';';
+  return sql;
+}
+exports.upsert = function(task, opt_updateUserIndex) {
+  return new Promise(function(resolve, reject) {
+    var sqlCommand = getUpsertString(task, opt_updateUserIndex);
+    exports.sqlQuery(sqlCommand, function(error, result) {
+      if (error) {
+        reject(error);
+      } else {
+        if (result && result.rows.length > 0) {
+          var first = result.rows[0];
+          result = {affectedRows: 0, insertId: 0};
+          result.affectedRows = 'true' == first.update ? 2 : 1;
+          result.insertId = opt_updateUserIndex ? first.user_index : 0;
+        }
+        resolve(result);
+      }
+    }, true);
+  });
 };
-exports.sqlEscape = function (value) {
-	return value.replace( /(\')/g, "\\'" );
-};
\ No newline at end of file
diff --git a/DocService/sources/taskresult.js b/DocService/sources/taskresult.js
index b4ce80a7..0f907db2 100644
--- a/DocService/sources/taskresult.js
+++ b/DocService/sources/taskresult.js
@@ -83,36 +83,8 @@ TaskResultData.prototype.completeDefaults = function() {
   }
 };
 
-function getUpsertString(task, opt_updateUserIndex) {
-  task.completeDefaults();
-  var dateNow = sqlBase.getDateTime(new Date());
-  var commandArg = [task.key, task.status, task.statusInfo, dateNow, task.title, task.userIndex, task.changeId];
-  var commandArgEsc = commandArg.map(function(curVal) {
-    return sqlBase.baseConnector.sqlEscape(curVal)
-  });
-  var sql = 'INSERT INTO task_result ( id, status, status_info, last_open_date, title,' +
-    ' user_index, change_id  ) VALUES (' + commandArgEsc.join(', ') + ') ON DUPLICATE KEY UPDATE' +
-    ' last_open_date = ' + sqlBase.baseConnector.sqlEscape(dateNow);
-  if (opt_updateUserIndex) {
-    //todo LAST_INSERT_ID in posgresql - RETURNING
-    sql += ', user_index = LAST_INSERT_ID(user_index + 1);';
-  } else {
-    sql += ';';
-  }
-  return sql;
-}
-
 function upsert(task, opt_updateUserIndex) {
-  return new Promise(function(resolve, reject) {
-    var sqlCommand = getUpsertString(task, opt_updateUserIndex);
-    sqlBase.baseConnector.sqlQuery(sqlCommand, function(error, result) {
-      if (error) {
-        reject(error);
-      } else {
-        resolve(result);
-      }
-    });
-  });
+  return sqlBase.baseConnector.upsert(task, opt_updateUserIndex);
 }
 
 function getSelectString(docId) {
@@ -281,7 +253,6 @@ exports.upsert = upsert;
 exports.select = select;
 exports.update = update;
 exports.updateIf = updateIf;
-exports.addRandomKey = addRandomKey;
 exports.addRandomKeyTask = addRandomKeyTask;
 exports.remove = remove;
 exports.getExpired = getExpired;
diff --git a/schema/postgresql/createdb.sql b/schema/postgresql/createdb.sql
new file mode 100644
index 00000000..99831f0d
--- /dev/null
+++ b/schema/postgresql/createdb.sql
@@ -0,0 +1,52 @@
+--
+-- Create schema onlyoffice
+--
+
+-- CREATE DATABASE onlyoffice ENCODING = 'UTF8' CONNECTION LIMIT = -1;
+
+--
+-- Drop tables
+--
+DROP TABLE IF EXISTS "public"."doc_callbacks";
+DROP TABLE IF EXISTS "public"."doc_changes";
+
+-- ----------------------------
+-- Table structure for doc_callbacks
+-- ----------------------------
+CREATE TABLE IF NOT EXISTS "public"."doc_callbacks" (
+"id" varchar(255) COLLATE "default" NOT NULL,
+"callback" text COLLATE "default" NOT NULL,
+"baseurl" text COLLATE "default" NOT NULL,
+PRIMARY KEY ("id")
+)
+WITH (OIDS=FALSE);
+
+-- ----------------------------
+-- Table structure for doc_changes
+-- ----------------------------
+CREATE TABLE IF NOT EXISTS "public"."doc_changes" (
+"id" varchar(255) COLLATE "default" NOT NULL,
+"change_id" int8 NOT NULL,
+"user_id" varchar(255) COLLATE "default" NOT NULL,
+"user_id_original" varchar(255) COLLATE "default" NOT NULL,
+"user_name" varchar(255) COLLATE "default" NOT NULL,
+"change_data" text COLLATE "default" NOT NULL,
+"change_date" timestamp without time zone NOT NULL,
+PRIMARY KEY ("id", "change_id")
+)
+WITH (OIDS=FALSE);
+
+-- ----------------------------
+-- Table structure for task_result
+-- ----------------------------
+CREATE TABLE IF NOT EXISTS "public"."task_result" (
+"id" varchar(255) COLLATE "default" NOT NULL,
+"status" int2 NOT NULL,
+"status_info" int8 NOT NULL,
+"last_open_date" timestamp without time zone NOT NULL,
+"title" varchar(255) COLLATE "default" NOT NULL,
+"user_index" int8 NOT NULL DEFAULT 1,
+"change_id" int8 NOT NULL DEFAULT 0,
+PRIMARY KEY ("id")
+)
+WITH (OIDS=FALSE);
\ No newline at end of file
diff --git a/schema/postgresql/removedb.sql b/schema/postgresql/removedb.sql
new file mode 100644
index 00000000..b2aa9b4d
--- /dev/null
+++ b/schema/postgresql/removedb.sql
@@ -0,0 +1 @@
+DROP DATABASE IF EXISTS onlyoffice;
\ No newline at end of file

From 39ecb8835b39e576dca919b7d8d3c9991d4fbed5 Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Fri, 26 Aug 2016 14:37:46 +0300
Subject: [PATCH 23/34] ConvertService.ashx with empty outputtype, remove
 console.log

---
 DocService/sources/converterservice.js | 13 ++++++++++---
 DocService/sources/server.js           |  1 -
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/DocService/sources/converterservice.js b/DocService/sources/converterservice.js
index b27d6c0f..bc679727 100644
--- a/DocService/sources/converterservice.js
+++ b/DocService/sources/converterservice.js
@@ -32,6 +32,7 @@
 
 var config = require('config');
 var co = require('co');
+const forwarded = require('forwarded');
 var taskResult = require('./taskresult');
 var logger = require('./../../Common/sources/logger');
 var utils = require('./../../Common/sources/utils');
@@ -201,7 +202,7 @@ function convertRequest(req, res) {
       cmd.setUrl(req.query['url']);
       cmd.setEmbeddedFonts(false);//req.query['embeddedfonts'];
       cmd.setFormat(req.query['filetype']);
-      var outputtype = req.query['outputtype'];
+      var outputtype = req.query['outputtype'] || '';
       docId = 'conv_' + req.query['key'] + '_' + outputtype;
       cmd.setDocId(docId);
       cmd.setTitle(constants.OUTPUT_NAME + '.' + outputtype);
@@ -212,8 +213,14 @@ function convertRequest(req, res) {
       cmd.setPassword(req.query['password']);
       var async = 'true' == req.query['async'];
 
-      var status = yield* convertByCmd(cmd, async, utils.getBaseUrlByRequest(req));
-      utils.fillXmlResponse(res, status.url, status.err);
+      if (constants.AVS_OFFICESTUDIO_FILE_UNKNOWN !== cmd.getOutputFormat()) {
+        var status = yield* convertByCmd(cmd, async, utils.getBaseUrlByRequest(req));
+        utils.fillXmlResponse(res, status.url, status.err);
+      } else {
+        var addresses = forwarded(req);
+        logger.error('Error convert unknown outputtype: query = %s from = %s docId = %s', JSON.stringify(req.query), addresses, docId);
+        utils.fillXmlResponse(res, undefined, constants.UNKNOWN);
+      }
     }
     catch (e) {
       logger.error('Error convert: docId = %s\r\n%s', docId, e.stack);
diff --git a/DocService/sources/server.js b/DocService/sources/server.js
index 21451718..ffa0383c 100644
--- a/DocService/sources/server.js
+++ b/DocService/sources/server.js
@@ -174,7 +174,6 @@ if (cluster.isMaster) {
           ipString = ip.toIPv4Address().toString();
         }
       }
-      console.log('ipString'+ipString);
       status = utils.checkIpFilter(ipString);
     }
     if (status > 0) {

From 56c02016ff034967c804a6687573438e0206d4e1 Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Fri, 26 Aug 2016 20:09:54 +0300
Subject: [PATCH 24/34] getChanges return changes in undefined order

---
 DocService/sources/baseConnector.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/DocService/sources/baseConnector.js b/DocService/sources/baseConnector.js
index 8c7bd774..3789f691 100644
--- a/DocService/sources/baseConnector.js
+++ b/DocService/sources/baseConnector.js
@@ -238,6 +238,7 @@ exports.getChangesPromise = function (docId, optStartIndex, optEndIndex) {
     if (null != optStartIndex && null != optEndIndex) {
       getCondition += ' AND change_id>=' + optStartIndex + ' AND change_id<' + optEndIndex;
     }
+    getCondition += ' ORDER BY change_id ASC';
     getDataFromTable(c_oTableId.changes, "*", getCondition, function(error, result) {
       if (error) {
         reject(error);

From a0de6683aec354379980b5aec155a1dde8357b3d Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Fri, 26 Aug 2016 20:22:32 +0300
Subject: [PATCH 25/34] add to 'getChanges return changes in undefined order'

---
 DocService/sources/baseConnector.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/DocService/sources/baseConnector.js b/DocService/sources/baseConnector.js
index 3789f691..964fba31 100644
--- a/DocService/sources/baseConnector.js
+++ b/DocService/sources/baseConnector.js
@@ -252,7 +252,7 @@ exports.getChanges = function (docId, callback) {
 	lockCriticalSection(docId, function () {_getChanges(docId, callback);});
 };
 function _getChanges (docId, callback) {
-	getDataFromTable(c_oTableId.changes, "*", "id='" + docId + "'",
+	getDataFromTable(c_oTableId.changes, "*", "id='" + docId + "' ORDER BY change_id ASC",
 		function (error, result) {unLockCriticalSection(docId); if (callback) callback(error, result);});
 }
 

From cb91e2a74b3d17273dc0d2d3cf604581b9c64c89 Mon Sep 17 00:00:00 2001
From: "Alexander.Trofimov" <Alexander.Trofimov@onlyoffice.com>
Date: Mon, 29 Aug 2016 11:42:31 +0300
Subject: [PATCH 26/34] add ExpiredTrial

---
 Common/sources/constants.js        |  3 ++-
 Common/sources/license.js          | 20 +++++++++++---------
 DocService/sources/DocsCoServer.js |  9 +++++----
 3 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/Common/sources/constants.js b/Common/sources/constants.js
index ed2a6de7..9d1d1774 100644
--- a/Common/sources/constants.js
+++ b/Common/sources/constants.js
@@ -41,7 +41,8 @@ exports.LICENSE_RESULT = {
   Expired     : 2,
   Success     : 3,
   UnknownUser : 4,
-  Connections : 5
+  Connections : 5,
+  ExpiredTrial: 6
 };
 
 exports.LICENSE_CONNECTIONS = 21;
diff --git a/Common/sources/license.js b/Common/sources/license.js
index b43da416..b7c6b64f 100644
--- a/Common/sources/license.js
+++ b/Common/sources/license.js
@@ -49,8 +49,9 @@ const buildDate = '6/29/2016';
 const oBuildDate = new Date(buildDate);
 
 exports.readLicense = function*() {
-	const resMax = {count: 999999, type: constants.LICENSE_RESULT.Success};
-	var res = {count: 1, type: constants.LICENSE_RESULT.Error, light: false};
+	const c_LR = constants.LICENSE_RESULT;
+	const resMax = {count: 999999, type: c_LR.Success};
+	var res = {count: 1, type: c_LR.Error, light: false};
 	var checkFile = false;
 	try {
 		var oFile = fs.readFileSync(configL.get('license_file')).toString();
@@ -64,26 +65,27 @@ exports.readLicense = function*() {
 		const publicKey = '-----BEGIN PUBLIC KEY-----\nMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDRhGF7X4A0ZVlEg594WmODVVUI\niiPQs04aLmvfg8SborHss5gQXu0aIdUT6nb5rTh5hD2yfpF2WIW6M8z0WxRhwicg\nXwi80H1aLPf6lEPPLvN29EhQNjBpkFkAJUbS8uuhJEeKw0cE49g80eBBF4BCqSL6\nPFQbP9/rByxdxEoAIQIDAQAB\n-----END PUBLIC KEY-----\n';
 		if (verify.verify(publicKey, sign, 'hex')) {
 			const endDate = new Date(oLicense['end_date']);
-			const checkDate = (true === oLicense['trial'] || 'true' === oLicense['trial']) ? new Date() : oBuildDate; // Someone who likes to put json string instead of bool
+			const isTrial = (true === oLicense['trial'] || 'true' === oLicense['trial']);
+			const checkDate = isTrial ? new Date() : oBuildDate; // Someone who likes to put json string instead of bool
 			if (endDate >= checkDate && 2 <= oLicense['version']) {
 				res.count = Math.min(Math.max(res.count, oLicense['process'] >> 0), resMax.count);
-				res.type = constants.LICENSE_RESULT.Success;
+				res.type = c_LR.Success;
 			} else {
-				res.type = constants.LICENSE_RESULT.Expired;
+				res.type = isTrial ? c_LR.ExpiredTrial : c_LR.Expired;
 			}
 
 			res.light = (true === oLicense['light'] || 'true' === oLicense['light']); // Someone who likes to put json string instead of bool
 		}
 	} catch (e) {
 		res.count = 1;
-		res.type = constants.LICENSE_RESULT.Error;
+		res.type = c_LR.Error;
 
 		if (checkFile || (yield* _getFileState())) {
-			res.type = constants.LICENSE_RESULT.Expired;
+			res.type = c_LR.ExpiredTrial;
 		}
 	}
-	if (res.type === constants.LICENSE_RESULT.Expired) {
-		res.count = 0;
+	if (res.type === c_LR.Expired || res.type === c_LR.ExpiredTrial) {
+		res.count = 1;
 		logger.error('License Expired!!!');
 	}
 
diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js
index 4e6edc6a..6da06bf8 100644
--- a/DocService/sources/DocsCoServer.js
+++ b/DocService/sources/DocsCoServer.js
@@ -1881,9 +1881,10 @@ exports.install = function(server, callbackFunction) {
   function _checkLicense(conn) {
     return co(function* () {
       try {
+        const c_LR = constants.LICENSE_RESULT;
         var licenseType = licenseInfo.type;
-        if (constants.LICENSE_RESULT.Success !== licenseType) {
-          licenseType = constants.LICENSE_RESULT.Success;
+        if (c_LR.Error === licenseType) {
+          licenseType = c_LR.Success;
 
           var count = constants.LICENSE_CONNECTIONS;
           var cursor = '0', sum = 0, scanRes, tmp, length, i, users;
@@ -1894,7 +1895,7 @@ exports.install = function(server, callbackFunction) {
 
             for (i = 0; i < length; ++i) {
               if (sum >= count) {
-                licenseType = constants.LICENSE_RESULT.Connections;
+                licenseType = c_LR.Connections;
                 break;
               }
 
@@ -1903,7 +1904,7 @@ exports.install = function(server, callbackFunction) {
             }
 
             if (sum >= count) {
-              licenseType = constants.LICENSE_RESULT.Connections;
+              licenseType = c_LR.Connections;
               break;
             }
 

From c212143443efa1fe1c05f93134138d88889d9720 Mon Sep 17 00:00:00 2001
From: "Alexander.Trofimov" <Alexander.Trofimov@onlyoffice.com>
Date: Tue, 30 Aug 2016 13:16:46 +0300
Subject: [PATCH 27/34] add integration params

---
 Common/sources/constants.js        |  4 +++
 Common/sources/license.js          | 40 ++++++++++++++++++++++++------
 DocService/sources/DocsCoServer.js |  2 +-
 3 files changed, 38 insertions(+), 8 deletions(-)

diff --git a/Common/sources/constants.js b/Common/sources/constants.js
index 9d1d1774..221bb5f6 100644
--- a/Common/sources/constants.js
+++ b/Common/sources/constants.js
@@ -156,6 +156,9 @@ exports.EDITOR_TYPE_SPREADSHEET = 1;
 exports.EDITOR_TYPE_PRESENTATION = 2;
 exports.EDITOR_TYPE_CONVERTATION = 3;
 
+exports.PACKAGE_TYPE_OS = 0;
+exports.PACKAGE_TYPE_I = 1;
+
 exports.REDIS_KEY_PUBSUB = 'pubsub';
 exports.REDIS_KEY_SAVE_LOCK = 'savelock:';
 exports.REDIS_KEY_PRESENCE_HASH = 'presence:hash:';
@@ -170,6 +173,7 @@ exports.REDIS_KEY_FORCE_SAVE = 'forcesave:';
 exports.REDIS_KEY_SAVED = 'saved:';
 exports.REDIS_KEY_SHUTDOWN = 'shutdown';
 exports.REDIS_KEY_LICENSE = 'license';
+exports.REDIS_KEY_LICENSE_T = 'licenseT';
 
 exports.SHUTDOWN_CODE = 4001;
 exports.SHUTDOWN_REASON = 'server shutdown';
diff --git a/Common/sources/license.js b/Common/sources/license.js
index b7c6b64f..13417722 100644
--- a/Common/sources/license.js
+++ b/Common/sources/license.js
@@ -40,18 +40,20 @@ const utils = require('./utils');
 const pubsubRedis = require('./../../DocService/sources/pubsubRedis');
 const redisClient = pubsubRedis.getClientRedis();
 
-const cfgRedisPrefix = config.get('services.CoAuthoring.redis.prefix');
-const redisKeyLicense = cfgRedisPrefix + constants.REDIS_KEY_LICENSE;
-
 const buildVersion = '4.0.0';
 const buildNumber = 19;
 const buildDate = '6/29/2016';
 const oBuildDate = new Date(buildDate);
+const oPackageType = constants.PACKAGE_TYPE_OS;
+
+const cfgRedisPrefix = config.get('services.CoAuthoring.redis.prefix');
+const redisKeyLicense = cfgRedisPrefix + ((constants.PACKAGE_TYPE_OS === oPackageType) ? constants.REDIS_KEY_LICENSE :
+	constants.REDIS_KEY_LICENSE_T);
 
 exports.readLicense = function*() {
 	const c_LR = constants.LICENSE_RESULT;
 	const resMax = {count: 999999, type: c_LR.Success};
-	var res = {count: 1, type: c_LR.Error, light: false};
+	var res = {count: 1, type: c_LR.Error, light: false, packageType: oPackageType};
 	var checkFile = false;
 	try {
 		var oFile = fs.readFileSync(configL.get('license_file')).toString();
@@ -80,8 +82,19 @@ exports.readLicense = function*() {
 		res.count = 1;
 		res.type = c_LR.Error;
 
-		if (checkFile || (yield* _getFileState())) {
+		if (checkFile) {
 			res.type = c_LR.ExpiredTrial;
+		} else {
+			if (constants.PACKAGE_TYPE_OS === oPackageType) {
+				if (yield* _getFileState()) {
+					res.type = c_LR.ExpiredTrial;
+				}
+			} else {
+				res.type = (yield* _getFileState()) ? c_LR.Success : c_LR.ExpiredTrial;
+				if (res.type === c_LR.Success) {
+					return res;
+				}
+			}
 		}
 	}
 	if (res.type === c_LR.Expired || res.type === c_LR.ExpiredTrial) {
@@ -97,8 +110,21 @@ exports.readLicense = function*() {
 };
 
 function* _getFileState() {
-	return yield utils.promiseRedis(redisClient, redisClient.hget, redisKeyLicense, redisKeyLicense);
+	const val = yield utils.promiseRedis(redisClient, redisClient.hget, redisKeyLicense, redisKeyLicense);
+	if (constants.PACKAGE_TYPE_OS === oPackageType) {
+		return val;
+	}
+
+	if (null === val) {
+		yield* _updateFileState();
+		return true;
+	}
+
+	var now = new Date();
+	now.setMonth(now.getMonth() - 1);
+	return (0 >= (now - new Date(val)));
 }
 function* _updateFileState() {
-	yield utils.promiseRedis(redisClient, redisClient.hset, redisKeyLicense, redisKeyLicense, redisKeyLicense);
+	const val = constants.PACKAGE_TYPE_OS === oPackageType ? redisKeyLicense : new Date();
+	yield utils.promiseRedis(redisClient, redisClient.hset, redisKeyLicense, redisKeyLicense, val);
 }
diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js
index 6da06bf8..df07b25c 100644
--- a/DocService/sources/DocsCoServer.js
+++ b/DocService/sources/DocsCoServer.js
@@ -1883,7 +1883,7 @@ exports.install = function(server, callbackFunction) {
       try {
         const c_LR = constants.LICENSE_RESULT;
         var licenseType = licenseInfo.type;
-        if (c_LR.Error === licenseType) {
+        if (constants.PACKAGE_TYPE_OS === licenseInfo.packageType && c_LR.Error === licenseType) {
           licenseType = c_LR.Success;
 
           var count = constants.LICENSE_CONNECTIONS;

From 6cd612fa7cdbe5da9f33f61ad44432ca555dd1e7 Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Tue, 30 Aug 2016 15:28:07 +0300
Subject: [PATCH 28/34] restart if cron job stop

---
 DocService/sources/DocsCoServer.js |  9 +++++++--
 DocService/sources/gc.js           | 19 ++++++++++++++-----
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js
index df07b25c..c52f0c55 100644
--- a/DocService/sources/DocsCoServer.js
+++ b/DocService/sources/DocsCoServer.js
@@ -2079,8 +2079,13 @@ exports.install = function(server, callbackFunction) {
       }
     });
   }
-  var innerPintJob = new cron.CronJob(cfgExpDocumentsCron, expireDoc);
-  innerPintJob.start();
+  var innerPingJob = function(opt_isStart) {
+    if (!opt_isStart) {
+      logger.warn('expireDoc restart');
+    }
+    new cron.CronJob(cfgExpDocumentsCron, expireDoc, innerPingJob, true);
+  };
+  innerPingJob(true);
 
   pubsub = new pubsubService();
   pubsub.on('message', pubsubOnMessage);
diff --git a/DocService/sources/gc.js b/DocService/sources/gc.js
index c119fdc2..6f9a5852 100644
--- a/DocService/sources/gc.js
+++ b/DocService/sources/gc.js
@@ -129,9 +129,18 @@ var checkDocumentExpire = function() {
     }
   });
 };
+var documentExpireJob = function(opt_isStart) {
+  if (!opt_isStart) {
+    logger.warn('checkDocumentExpire restart');
+  }
+  new cron.CronJob(cfgExpDocumentsCron, checkDocumentExpire, documentExpireJob, true);
+};
+documentExpireJob(true);
 
-var documentExpireJob = new cron.CronJob(cfgExpDocumentsCron, checkDocumentExpire);
-documentExpireJob.start();
-
-var fileExpireJob = new cron.CronJob(cfgExpFilesCron, checkFileExpire);
-fileExpireJob.start();
+var fileExpireJob = function(opt_isStart) {
+  if (!opt_isStart) {
+    logger.warn('checkFileExpire restart');
+  }
+  new cron.CronJob(cfgExpFilesCron, checkFileExpire, fileExpireJob, true);
+};
+fileExpireJob(true);

From 732049cd9d96bb2663c45bb2171757a32f97a5e3 Mon Sep 17 00:00:00 2001
From: alexandervnuchkov <alexander.vnuchkov@teamlab.com>
Date: Tue, 30 Aug 2016 16:45:01 +0300
Subject: [PATCH 29/34] Update README.md

Added link to Stack Overflow
---
 Readme.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Readme.md b/Readme.md
index 68b0136e..f2991a3a 100644
--- a/Readme.md
+++ b/Readme.md
@@ -75,10 +75,11 @@ In case it is necessary to temporarily edit the config files, create the local.j
 
 ## User Feedback and Support
 
-If you have any problems with or questions about [ONLYOFFICE Document Server][2], please visit our official forum to find answers to your questions: [dev.onlyoffice.org][1].
+If you have any problems with or questions about [ONLYOFFICE Document Server][2], please visit our official forum to find answers to your questions: [dev.onlyoffice.org][1] or you can ask and answer ONLYOFFICE development questions on [Stack Overflow][3].
 
   [1]: http://dev.onlyoffice.org
   [2]: https://github.com/ONLYOFFICE/DocumentServer
+  [3]: http://stackoverflow.com/questions/tagged/onlyoffice
 
 ## License
 

From bf221129246029526c14029ce2d70b9867d952e4 Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Tue, 30 Aug 2016 18:09:03 +0300
Subject: [PATCH 30/34] connection with critical error can not save changes

---
 DocService/sources/DocsCoServer.js | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js
index c52f0c55..d6ff6278 100644
--- a/DocService/sources/DocsCoServer.js
+++ b/DocService/sources/DocsCoServer.js
@@ -830,6 +830,10 @@ exports.install = function(server, callbackFunction) {
           logger.debug('Server shutdown receive data');
           return;
         }
+        if (conn.isCiriticalError && ('message' == data.type || 'getLock' == data.type || 'saveChanges' == data.type)) {
+          logger.warn("conn.isCiriticalError send command: docId = %s type = %s", docId, data.type);
+          return;
+        }
         switch (data.type) {
           case 'auth'          :
             yield* auth(conn, data);
@@ -1090,6 +1094,7 @@ exports.install = function(server, callbackFunction) {
 
   function sendFileError(conn, errorId) {
     logger.error('error description: docId = %s errorId = %s', conn.docId, errorId);
+    conn.isCiriticalError = true;
     sendData(conn, {type: 'error', description: errorId});
   }
 

From 68108e6f059575590c8e993beaae381d9e96f202 Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Wed, 31 Aug 2016 17:14:05 +0300
Subject: [PATCH 31/34] support postgre before 9.5

---
 DocService/sources/baseConnector.js           |  5 +-
 DocService/sources/mySqlBaseConnector.js      |  2 +-
 DocService/sources/postgreSqlBaseConnector.js | 87 +++++++++++++------
 schema/postgresql/createdb.sql                | 31 ++++++-
 4 files changed, 93 insertions(+), 32 deletions(-)

diff --git a/DocService/sources/baseConnector.js b/DocService/sources/baseConnector.js
index 964fba31..cc6f3db1 100644
--- a/DocService/sources/baseConnector.js
+++ b/DocService/sources/baseConnector.js
@@ -82,10 +82,7 @@ exports.loadTable = function (tableId, callbackFunction) {
 	baseConnector.sqlQuery(sqlCommand, callbackFunction);
 };
 exports.insertCallback = function(id, href, baseUrl, callbackFunction) {
-  var sqlCommand = "INSERT " + baseConnector.ignoreStr + " INTO " + tableCallbacks + " VALUES (" + baseConnector.sqlEscape(id) + "," +
-    baseConnector.sqlEscape(href) + "," + baseConnector.sqlEscape(baseUrl) + ") " + baseConnector.doNothingStr + ";";
-
-  baseConnector.sqlQuery(sqlCommand, callbackFunction);
+  baseConnector.insertCallback(id, href, baseUrl, callbackFunction);
 };
 exports.insertCallbackPromise = function(id, href, baseUrl) {
   return new Promise(function(resolve, reject) {
diff --git a/DocService/sources/mySqlBaseConnector.js b/DocService/sources/mySqlBaseConnector.js
index a872b38e..ead210ac 100644
--- a/DocService/sources/mySqlBaseConnector.js
+++ b/DocService/sources/mySqlBaseConnector.js
@@ -1 +1 @@
-/*
 * (c) Copyright Ascensio System SIA 2010-2016
 *
 * This program is a free software product. You can redistribute it and/or
 * modify it under the terms of the GNU Affero General Public License (AGPL)
 * version 3 as published by the Free Software Foundation. In accordance with
 * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect
 * that Ascensio System SIA expressly excludes the warranty of non-infringement
 * of any third-party rights.
 *
 * This program is distributed WITHOUT ANY WARRANTY; without even the implied
 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE. For
 * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
 *
 * You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia,
 * EU, LV-1021.
 *
 * The  interactive user interfaces in modified source and object code versions
 * of the Program must display Appropriate Legal Notices, as required under
 * Section 5 of the GNU AGPL version 3.
 *
 * Pursuant to Section 7(b) of the License you must retain the original Product
 * logo when distributing the program. Pursuant to Section 7(e) we decline to
 * grant you any rights under trademark law for use of our trademarks.
 *
 * All the Product's GUI elements, including illustrations and icon sets, as
 * well as technical writing content are licensed under the terms of the
 * Creative Commons Attribution-ShareAlike 4.0 International. See the License
 * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
 *
 */

var mysql = require('mysql');
var sqlBase = require('./baseConnector');
var configSql = require('config').get('services.CoAuthoring.sql');
var pool  = mysql.createPool({
	host		: configSql.get('dbHost'),
	port		: configSql.get('dbPort'),
	user		: configSql.get('dbUser'),
	password	: configSql.get('dbPass'),
	database	: configSql.get('dbName'),
	charset		: configSql.get('charset'),
	connectionLimit	: configSql.get('connectionlimit'),
	timezone	: '+0000',
	flags : '-FOUND_ROWS'
});

var logger = require('./../../Common/sources/logger');

exports.sqlQuery = function (sqlCommand, callbackFunction) {
	pool.getConnection(function(err, connection) {
		if (err) {
			logger.error('pool.getConnection error: %s', err);
			if (callbackFunction) callbackFunction(err, null);
			return;
		}

		connection.query(sqlCommand, function (error, result) {
			connection.release();
			if (error) {
				logger.error('________________________error_____________________');
				logger.error('sqlQuery: %s sqlCommand: %s', error.code, sqlCommand);
				logger.error(error);
				logger.error('_____________________end_error_____________________');
			}
			if (callbackFunction) callbackFunction(error, result);
		});
	});
};
exports.sqlEscape = function (value) {
	return pool.escape(value);
};

exports.ignoreStr = 'IGNORE';
exports.doNothingStr = '';

function getUpsertString(task, opt_updateUserIndex) {
	task.completeDefaults();
	var dateNow = sqlBase.getDateTime(new Date());
	var commandArg = [task.key, task.status, task.statusInfo, dateNow, task.title, task.userIndex, task.changeId];
	var commandArgEsc = commandArg.map(function(curVal) {
		return exports.sqlEscape(curVal)
	});
	var sql = 'INSERT INTO task_result ( id, status, status_info, last_open_date, title,' +
		' user_index, change_id  ) VALUES (' + commandArgEsc.join(', ') + ') ON DUPLICATE KEY UPDATE' +
		' last_open_date = ' + exports.sqlEscape(dateNow);
	if (opt_updateUserIndex) {
		sql += ', user_index = LAST_INSERT_ID(user_index + 1);';
	} else {
		sql += ';';
	}
	return sql;
}

exports.upsert = function(task, opt_updateUserIndex) {
	return new Promise(function(resolve, reject) {
		var sqlCommand = getUpsertString(task, opt_updateUserIndex);
		exports.sqlQuery(sqlCommand, function(error, result) {
			if (error) {
				reject(error);
			} else {
				resolve(result);
			}
		});
	});
};
\ No newline at end of file
+/*
 * (c) Copyright Ascensio System SIA 2010-2016
 *
 * This program is a free software product. You can redistribute it and/or
 * modify it under the terms of the GNU Affero General Public License (AGPL)
 * version 3 as published by the Free Software Foundation. In accordance with
 * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect
 * that Ascensio System SIA expressly excludes the warranty of non-infringement
 * of any third-party rights.
 *
 * This program is distributed WITHOUT ANY WARRANTY; without even the implied
 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE. For
 * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
 *
 * You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia,
 * EU, LV-1021.
 *
 * The  interactive user interfaces in modified source and object code versions
 * of the Program must display Appropriate Legal Notices, as required under
 * Section 5 of the GNU AGPL version 3.
 *
 * Pursuant to Section 7(b) of the License you must retain the original Product
 * logo when distributing the program. Pursuant to Section 7(e) we decline to
 * grant you any rights under trademark law for use of our trademarks.
 *
 * All the Product's GUI elements, including illustrations and icon sets, as
 * well as technical writing content are licensed under the terms of the
 * Creative Commons Attribution-ShareAlike 4.0 International. See the License
 * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
 *
 */

var mysql = require('mysql');
var sqlBase = require('./baseConnector');
var configSql = require('config').get('services.CoAuthoring.sql');
var pool  = mysql.createPool({
	host		: configSql.get('dbHost'),
	port		: configSql.get('dbPort'),
	user		: configSql.get('dbUser'),
	password	: configSql.get('dbPass'),
	database	: configSql.get('dbName'),
	charset		: configSql.get('charset'),
	connectionLimit	: configSql.get('connectionlimit'),
	timezone	: '+0000',
	flags : '-FOUND_ROWS'
});
var cfgTableCallbacks = configSql.get('tableCallbacks');
var logger = require('./../../Common/sources/logger');

exports.sqlQuery = function (sqlCommand, callbackFunction) {
	pool.getConnection(function(err, connection) {
		if (err) {
			logger.error('pool.getConnection error: %s', err);
			if (callbackFunction) callbackFunction(err, null);
			return;
		}

		connection.query(sqlCommand, function (error, result) {
			connection.release();
			if (error) {
				logger.error('________________________error_____________________');
				logger.error('sqlQuery: %s sqlCommand: %s', error.code, sqlCommand);
				logger.error(error);
				logger.error('_____________________end_error_____________________');
			}
			if (callbackFunction) callbackFunction(error, result);
		});
	});
};
exports.sqlEscape = function (value) {
	return pool.escape(value);
};

exports.insertCallback = function(id, href, baseUrl, callbackFunction) {
	var sqlCommand = "INSERT IGNORE INTO " + cfgTableCallbacks + " VALUES (" + exports.sqlEscape(id) + "," +
		exports.sqlEscape(href) + "," + exports.sqlEscape(baseUrl) + ");";

	exports.sqlQuery(sqlCommand, callbackFunction);
};

function getUpsertString(task, opt_updateUserIndex) {
	task.completeDefaults();
	var dateNow = sqlBase.getDateTime(new Date());
	var commandArg = [task.key, task.status, task.statusInfo, dateNow, task.title, task.userIndex, task.changeId];
	var commandArgEsc = commandArg.map(function(curVal) {
		return exports.sqlEscape(curVal)
	});
	var sql = 'INSERT INTO task_result ( id, status, status_info, last_open_date, title,' +
		' user_index, change_id  ) VALUES (' + commandArgEsc.join(', ') + ') ON DUPLICATE KEY UPDATE' +
		' last_open_date = ' + exports.sqlEscape(dateNow);
	if (opt_updateUserIndex) {
		sql += ', user_index = LAST_INSERT_ID(user_index + 1);';
	} else {
		sql += ';';
	}
	return sql;
}

exports.upsert = function(task, opt_updateUserIndex) {
	return new Promise(function(resolve, reject) {
		var sqlCommand = getUpsertString(task, opt_updateUserIndex);
		exports.sqlQuery(sqlCommand, function(error, result) {
			if (error) {
				reject(error);
			} else {
				resolve(result);
			}
		});
	});
};
\ No newline at end of file
diff --git a/DocService/sources/postgreSqlBaseConnector.js b/DocService/sources/postgreSqlBaseConnector.js
index 2757afbf..0a62ac3b 100644
--- a/DocService/sources/postgreSqlBaseConnector.js
+++ b/DocService/sources/postgreSqlBaseConnector.js
@@ -47,6 +47,7 @@ var pool = new pg.Pool({
   ssl: false,
   idleTimeoutMillis: 30000
 });
+var cfgTableCallbacks = configSql.get('tableCallbacks');
 //todo datetime timezone
 types.setTypeParser(1114, function(stringValue) {
   return new Date(stringValue + '+0000');
@@ -57,7 +58,7 @@ types.setTypeParser(1184, function(stringValue) {
 
 var logger = require('./../../Common/sources/logger');
 
-exports.sqlQuery = function(sqlCommand, callbackFunction, opt_noModifyRes) {
+exports.sqlQuery = function(sqlCommand, callbackFunction, opt_noModifyRes, opt_noLog) {
   co(function *() {
     var client = null;
     var result = null;
@@ -67,10 +68,12 @@ exports.sqlQuery = function(sqlCommand, callbackFunction, opt_noModifyRes) {
       result = yield client.query(sqlCommand);
     } catch (err) {
       error = err;
-      if (client) {
-        logger.error('sqlQuery error sqlCommand: %s:\r\n%s', sqlCommand.slice(0, 50), err.stack);
-      } else {
-        logger.error('pool.getConnection error: %s', err);
+      if (!opt_noLog) {
+        if (client) {
+          logger.error('sqlQuery error sqlCommand: %s:\r\n%s', sqlCommand.slice(0, 50), err.stack);
+        } else {
+          logger.error('pool.getConnection error: %s', err);
+        }
       }
     } finally {
       if (client) {
@@ -94,35 +97,67 @@ exports.sqlEscape = function(value) {
   //todo parameterized queries
   return undefined !== value ? pgEscape.literal(value.toString()) : 'NULL';
 };
-exports.ignoreStr = '';
-exports.doNothingStr = 'ON CONFLICT DO NOTHING';
+var isSupportOnConflict = false;
+(function checkIsSupportOnConflict() {
+  var sqlCommand = 'INSERT INTO checkIsSupportOnConflict (id) VALUES(1) ON CONFLICT DO NOTHING;';
+  exports.sqlQuery(sqlCommand, function(error, result) {
+    if (error) {
+      if ('42601' == error.code) {
+        //SYNTAX ERROR
+        isSupportOnConflict = false;
+        logger.debug('checkIsSupportOnConflict false');
+      } else if ('42P01' == error.code) {
+        //	UNDEFINED TABLE
+        isSupportOnConflict = true;
+        logger.debug('checkIsSupportOnConflict true');
+      } else {
+        logger.error('checkIsSupportOnConflict unexpected error code:\r\n%s', error.stack);
+      }
+    }
+  }, true, true);
+})();
 
-function getUpsertString(task, opt_updateUserIndex) {
+exports.insertCallback = function(id, href, baseUrl, callbackFunction) {
+  var sqlCommand = "INSERT INTO " + cfgTableCallbacks + " VALUES (" + exports.sqlEscape(id) + "," +
+    exports.sqlEscape(href) + "," + exports.sqlEscape(baseUrl) + ")";
+  if (isSupportOnConflict) {
+    sqlCommand += ' ON CONFLICT DO NOTHING;';
+    exports.sqlQuery(sqlCommand, callbackFunction);
+  } else {
+    sqlCommand += ';';
+    exports.sqlQuery(sqlCommand, function(error, result) {
+      if (error && error.code == '23505') {
+        //UNIQUE VIOLATION
+        callbackFunction(null, result);
+      } else {
+        callbackFunction(error, result);
+      }
+    });
+  }
+};
+
+function getUpsertString(task) {
   task.completeDefaults();
   var dateNow = sqlBase.getDateTime(new Date());
   var commandArg = [task.key, task.status, task.statusInfo, dateNow, task.title, task.userIndex, task.changeId];
   var commandArgEsc = commandArg.map(function(curVal) {
     return exports.sqlEscape(curVal)
   });
-  //http://stackoverflow.com/questions/34762732/how-to-find-out-if-an-upsert-was-an-update-with-postgresql-9-5-upsert
-  var sql = "INSERT INTO task_result (id, status, status_info, last_open_date, title, user_index, change_id) SELECT " +
-    commandArgEsc.join(', ') +
-    " WHERE 'false' = set_config('myapp.isupdate', 'false', true) ON CONFLICT (id) DO UPDATE SET  last_open_date = " +
-    sqlBase.baseConnector.sqlEscape(dateNow);
-  if (opt_updateUserIndex) {
-    sql += ', user_index = task_result.user_index + 1';
+  if (isSupportOnConflict) {
+    //http://stackoverflow.com/questions/34762732/how-to-find-out-if-an-upsert-was-an-update-with-postgresql-9-5-upsert
+    return "INSERT INTO task_result (id, status, status_info, last_open_date, title, user_index, change_id) SELECT " +
+      commandArgEsc.join(', ') +
+      " WHERE 'false' = set_config('myapp.isupdate', 'false', true) ON CONFLICT (id) DO UPDATE SET  last_open_date = " +
+      sqlBase.baseConnector.sqlEscape(dateNow) +
+      ", user_index = task_result.user_index + 1 WHERE 'true' = set_config('myapp.isupdate', 'true', true) RETURNING" +
+      " current_setting('myapp.isupdate') as isupdate, user_index as userindex;";
+  } else {
+    return "SELECT * FROM merge_db(" + commandArgEsc.join(', ') + ");";
   }
-  sql +=
-    " WHERE 'true' = set_config('myapp.isupdate', 'true', true) RETURNING current_setting('myapp.isupdate') as update";
-  if (opt_updateUserIndex) {
-    sql += ', user_index';
-  }
-  sql += ';';
-  return sql;
 }
-exports.upsert = function(task, opt_updateUserIndex) {
+exports.upsert = function(task) {
   return new Promise(function(resolve, reject) {
-    var sqlCommand = getUpsertString(task, opt_updateUserIndex);
+    var sqlCommand = getUpsertString(task);
     exports.sqlQuery(sqlCommand, function(error, result) {
       if (error) {
         reject(error);
@@ -130,8 +165,8 @@ exports.upsert = function(task, opt_updateUserIndex) {
         if (result && result.rows.length > 0) {
           var first = result.rows[0];
           result = {affectedRows: 0, insertId: 0};
-          result.affectedRows = 'true' == first.update ? 2 : 1;
-          result.insertId = opt_updateUserIndex ? first.user_index : 0;
+          result.affectedRows = 'true' == first.isupdate ? 2 : 1;
+          result.insertId = first.userindex;
         }
         resolve(result);
       }
diff --git a/schema/postgresql/createdb.sql b/schema/postgresql/createdb.sql
index 99831f0d..2d05cab8 100644
--- a/schema/postgresql/createdb.sql
+++ b/schema/postgresql/createdb.sql
@@ -49,4 +49,33 @@ CREATE TABLE IF NOT EXISTS "public"."task_result" (
 "change_id" int8 NOT NULL DEFAULT 0,
 PRIMARY KEY ("id")
 )
-WITH (OIDS=FALSE);
\ No newline at end of file
+WITH (OIDS=FALSE);
+
+--https://www.postgresql.org/docs/current/static/plpgsql-control-structures.html#PLPGSQL-UPSERT-EXAMPLE
+CREATE OR REPLACE FUNCTION merge_db(_id varchar(255), _status int2, _status_info int8, _last_open_date timestamp without time zone, _title varchar(255), _user_index int8, _change_id int8, OUT isupdate char(5), OUT userindex int8) AS
+$$
+DECLARE
+	t_var "public"."task_result"."user_index"%TYPE;
+BEGIN
+	LOOP
+		-- first try to update the key
+		-- note that "a" must be unique
+		UPDATE "public"."task_result" SET last_open_date=_last_open_date, user_index=user_index+1 WHERE id = _id RETURNING user_index into userindex;
+		IF found THEN
+			isupdate := 'true';
+			RETURN;
+		END IF;
+		-- not there, so try to insert the key
+		-- if someone else inserts the same key concurrently,
+		-- we could get a unique-key failure
+		BEGIN
+			INSERT INTO "public"."task_result"(id, status, status_info, last_open_date, title, user_index, change_id) VALUES(_id, _status, _status_info, _last_open_date, _title, _user_index, _change_id) RETURNING user_index into userindex;
+			isupdate := 'false';
+			RETURN;
+		EXCEPTION WHEN unique_violation THEN
+			-- do nothing, and loop to try the UPDATE again
+		END;
+	END LOOP;
+END;
+$$
+LANGUAGE plpgsql;

From cbc896514667a2949963ca455fd9b5ad95bf99f4 Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Thu, 1 Sep 2016 13:01:46 +0300
Subject: [PATCH 32/34] remove console.log

---
 Common/sources/rabbitMQCore.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Common/sources/rabbitMQCore.js b/Common/sources/rabbitMQCore.js
index 19e88d26..78fc6347 100644
--- a/Common/sources/rabbitMQCore.js
+++ b/Common/sources/rabbitMQCore.js
@@ -72,7 +72,7 @@ function connetPromise(closeCallback) {
           var closeEventCallback = function() {
             //in some case receive multiple close events
             conn.removeListener('close', closeEventCallback);
-            console.debug("[AMQP] conn close");
+            logger.debug('[AMQP] conn close');
             closeCallback();
           };
           conn.on('close', closeEventCallback);

From 8e53081bb84c205ff01ab78965cfe0d67a71a400 Mon Sep 17 00:00:00 2001
From: konovalovsergey <Sergey.Konovalov@onlyoffice.com>
Date: Thu, 1 Sep 2016 13:36:23 +0300
Subject: [PATCH 33/34] at close connection callback could mistakenly saved in
 database

---
 DocService/sources/DocsCoServer.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js
index d6ff6278..66d4ebfe 100644
--- a/DocService/sources/DocsCoServer.js
+++ b/DocService/sources/DocsCoServer.js
@@ -610,6 +610,9 @@ function* sendStatusDocument(docId, bChangeBase, userAction, callback, baseUrl,
     var getRes = yield* getCallback(docId);
     if (getRes) {
       callback = getRes.server;
+      if (!baseUrl) {
+        baseUrl = getRes.baseUrl;
+      }
     }
   }
   if (null == callback) {
@@ -747,7 +750,7 @@ function* cleanDocumentOnExit(docId, deleteChanges) {
 function* cleanDocumentOnExitNoChanges(docId, opt_userId) {
   var userAction = opt_userId ? new commonDefines.OutputAction(commonDefines.c_oAscUserAction.Out, opt_userId) : null;
   // Отправляем, что все ушли и нет изменений (чтобы выставить статус на сервере об окончании редактирования)
-  yield* sendStatusDocument(docId, c_oAscChangeBase.All, userAction);
+  yield* sendStatusDocument(docId, c_oAscChangeBase.No, userAction);
   //если пользователь зашел в документ, соединение порвалось, на сервере удалилась вся информация,
   //при восстановлении соединения userIndex сохранится и он совпадет с userIndex следующего пользователя
   yield* cleanDocumentOnExit(docId, false);

From cf4e98a4dbb013bc530b2b4c712a483bd40d5879 Mon Sep 17 00:00:00 2001
From: "Alexander.Trofimov" <Alexander.Trofimov@onlyoffice.com>
Date: Fri, 2 Sep 2016 10:48:45 +0300
Subject: [PATCH 34/34] update state

---
 Common/sources/license.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Common/sources/license.js b/Common/sources/license.js
index 13417722..e195f9f1 100644
--- a/Common/sources/license.js
+++ b/Common/sources/license.js
@@ -103,7 +103,7 @@ exports.readLicense = function*() {
 	}
 
 	if (checkFile) {
-		yield* _updateFileState();
+		yield* _updateFileState(true);
 	}
 
 	return res;
@@ -116,7 +116,7 @@ function* _getFileState() {
 	}
 
 	if (null === val) {
-		yield* _updateFileState();
+		yield* _updateFileState(false);
 		return true;
 	}
 
@@ -124,7 +124,7 @@ function* _getFileState() {
 	now.setMonth(now.getMonth() - 1);
 	return (0 >= (now - new Date(val)));
 }
-function* _updateFileState() {
-	const val = constants.PACKAGE_TYPE_OS === oPackageType ? redisKeyLicense : new Date();
+function* _updateFileState(state) {
+	const val = constants.PACKAGE_TYPE_OS === oPackageType ? redisKeyLicense : (state ? new Date(1) : new Date());
 	yield utils.promiseRedis(redisClient, redisClient.hset, redisKeyLicense, redisKeyLicense, val);
 }