diff --git a/Common/config/default.json b/Common/config/default.json index e86f0526..f7b3b872 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -103,7 +103,8 @@ "documents": 300, "documentsCron": "0 5 * * * *", "files": 604800, - "filesCron": "00 59 23 * * *" + "filesCron": "00 00 */1 * * *", + "filesremovedatonce": 10 } } }, diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 821b3e82..e3530fba 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -57,8 +57,8 @@ var commonDefines = require('./../../Common/sources/commondefines'); var statsDClient = require('./../../Common/sources/statsdclient'); var config = require('config').get('services.CoAuthoring'); var sqlBase = require('./baseConnector'); -var taskResult = require('./taskresult'); var canvasService = require('./canvasservice'); +var checkExpire = require('./checkexpire'); var redis = require(config.get('redis.name')); var pubsubService = require('./' + config.get('pubsub.name')); var queueService = require('./../../Common/sources/' + configCommon.get('queue.name')); @@ -1811,39 +1811,10 @@ exports.install = function(server, callbackFunction) { } }); }; - var checkFileExpire = function () { - utils.spawn(function*() { - try { - logger.debug('checkFileExpire start'); - var removedCount = 0; - //если ввести LIMIT, тогда могут всегда отдавать одни и теже данные, которые не удаляются - var expired = yield taskResult.getExpired(cfgExpFiles); - for (var i = 0; i < expired.length; ++i) { - var docId = expired[i].tr_key; - //delete if no changes, participants - var participantCount = yield* getParticipantCount(docId); - if (!(participantCount > 0)) { - var puckerIndex = yield* getChangesIndex(docId); - if (!(puckerIndex > 0)) { - var removeRes = yield taskResult.remove(docId); - //если ничего не удалилось, значит это сделал другой процесс - if (removeRes.affectedRows > 0) { - removedCount++; - yield storage.deletePath(docId); - } - } - } - } - logger.debug('checkFileExpire end: expired = %d removedFiles = %d', expired.length, removedCount); - } catch (e) { - logger.error('checkFileExpire error:\r\n%s', e.stack); - } - }); - }; //удаление файлов от которых не приходит heartbeat var documentExpireJob = new cron.CronJob(cfgExpDocumentsCron, checkDocumentExpire); documentExpireJob.start(); - var fileExpireJob = new cron.CronJob(cfgExpFilesCron, checkFileExpire); + var fileExpireJob = new cron.CronJob(cfgExpFilesCron, checkExpire.checkFileExpire); fileExpireJob.start(); //cache diff --git a/DocService/sources/checkexpire.js b/DocService/sources/checkexpire.js new file mode 100644 index 00000000..77b4aa8c --- /dev/null +++ b/DocService/sources/checkexpire.js @@ -0,0 +1,40 @@ +var config = require('config').get('services.CoAuthoring'); +var taskResult = require('./taskresult'); +var storage = require('./../../Common/sources/storage-base'); +var utils = require('./../../Common/sources/utils'); +var logger = require('./../../Common/sources/logger'); + +var cfgExpFiles = config.get('expire.files'); +var cfgExpFilesRemovedAtOnce = config.get('expire.filesremovedatonce'); + +//todo checkDocumentExpire +var checkFileExpire = function() { + utils.spawn(function*() { + try { + logger.debug('checkFileExpire start'); + var expired; + var removedCount = 0; + var currentRemovedCount; + do { + currentRemovedCount = 0; + expired = yield taskResult.getExpired(cfgExpFilesRemovedAtOnce, cfgExpFiles); + console.log('expired.length' + expired.length); + for (var i = 0; i < expired.length; ++i) { + var docId = expired[i].tr_key; + //todo drop user + var removeRes = yield taskResult.remove(docId); + //если ничего не удалилось, значит это сделал другой процесс + if (removeRes.affectedRows > 0) { + currentRemovedCount++; + yield storage.deletePath(docId); + } + } + removedCount += currentRemovedCount; + } while (currentRemovedCount > 0); + logger.debug('checkFileExpire end: removedCount = %d', removedCount); + } catch (e) { + logger.error('checkFileExpire error:\r\n%s', e.stack); + } + }); +}; +exports.checkFileExpire = checkFileExpire; diff --git a/DocService/sources/taskresult.js b/DocService/sources/taskresult.js index 6619d934..36b6a6b0 100644 --- a/DocService/sources/taskresult.js +++ b/DocService/sources/taskresult.js @@ -166,15 +166,16 @@ function remove(docId) { }); }); } -function getExpiredString(expireSeconds) { +function getExpiredString(maxCount, expireSeconds) { var expireDate = new Date(); utils.addSeconds(expireDate, -expireSeconds); var expireDateStr = sqlBase.baseConnector.sqlEscape(sqlBase.getDateTime(expireDate)); - return 'SELECT * FROM task_result WHERE tr_last_open_date <= ' + expireDateStr + ';'; + return 'SELECT * FROM task_result WHERE tr_last_open_date <= ' + expireDateStr + + ' AND NOT EXISTS(SELECT dc_key FROM doc_changes WHERE dc_key = tr_key LIMIT 1) LIMIT ' + maxCount + ';'; } -function getExpired(expireSeconds) { +function getExpired(maxCount, expireSeconds) { return new Promise(function(resolve, reject) { - var sqlCommand = getExpiredString(expireSeconds); + var sqlCommand = getExpiredString(maxCount, expireSeconds); sqlBase.baseConnector.sqlQuery(sqlCommand, function(error, result) { if (error) { reject(error);