freifunk/ffffng
0
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Fix: More asynchronous glob operations and graceful-fs.

Browse source 
Hope this fixes the "Error: EMFILE, too many open files" in
NodeInformationRetrievalJob.

See: https://github.com/freifunkhamburg/ffffng/issues/13
This commit is contained in:
baldo 2016-06-21 22:59:37 +02:00
parent bb8d958ad5
commit 2dc7ad0c23
6 changed files with 70 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
package.json
View file

@ -32,6 +32,7 @@
"deep-extend": "0.4.1",
"express": "4.14.0",
"glob": "7.0.5",
"graceful-fs": "4.1.4",
"http-auth": "2.4.4",
"http-errors": "1.5.0",
"lodash": "4.13.1",

2
server/config.js
View file

@ -28,7 +28,7 @@ if (commandLineOptions.help || !commandLineOptions.config) {
}
var fs = require('fs');
var fs = require('graceful-fs');
var deepExtend = require('deep-extend');
var defaultConfig = {

2
server/db/database.js
View file

@ -1,7 +1,7 @@
'use strict';
var async = require('async');
var fs = require('fs');
var fs = require('graceful-fs');
var glob = require('glob');
var path = require('path');

2
server/libs.js
View file

@ -17,7 +17,7 @@
lib('async');
lib('crypto');
lib('deepExtend', 'deep-extend');
lib('fs');
lib('fs', 'graceful-fs');
lib('glob');
lib('moment');
lib('request');

7
server/main.js
View file

@ -7,6 +7,13 @@ global.angular = require('ng-di');
angular.module('ffffng', []);
(function () {
// Use graceful-fs instead of fs also in all libraries to have more robust fs handling.
var realFs = require('fs');
var gracefulFs = require('graceful-fs');
gracefulFs.gracefulify(realFs);
})();
require('./config');
require('./logger').tag('main', 'startup').info('Server starting up...');

95
server/services/nodeService.js
View file

@ -33,7 +33,7 @@ angular.module('ffffng')
return crypto.randomBytes(8).toString('hex');
}
function findNodeFiles(filter) {
function toNodeFilesPattern(filter) {
var pattern = _.join(
_.map(filenameParts, function (field) {
return filter.hasOwnProperty(field) ? filter[field] : '*';
@ -41,7 +41,15 @@ angular.module('ffffng')
'@'
);
return glob.sync(config.server.peersPath + '/' + pattern.toLowerCase());
return config.server.peersPath + '/' + pattern.toLowerCase();
}
function findNodeFiles(filter, callback) {
glob(toNodeFilesPattern(filter), callback);
}
function findNodeFilesSync(filter) {
return glob.sync(toNodeFilesPattern(filter));
}
function findFilesInPeersPath(callback) {
@ -76,7 +84,7 @@ angular.module('ffffng')
}
function isDuplicate(filter, token) {
var files = findNodeFiles(filter);
var files = findNodeFilesSync(filter);
if (files.length === 0) {
return false;
}
@ -159,7 +167,7 @@ angular.module('ffffng')
var error;
if (isUpdate) {
var files = findNodeFiles({ token: token });
var files = findNodeFilesSync({ token: token });
if (files.length !== 1) {
return callback({data: 'Node not found.', type: ErrorTypes.notFound});
}
@ -196,20 +204,26 @@ angular.module('ffffng')
}
function deleteNodeFile(token, callback) {
var files = findNodeFiles({ token: token });
if (files.length !== 1) {
return callback({data: 'Node not found.', type: ErrorTypes.notFound});
}
findNodeFiles({ token: token }, function (err, files) {
if (err) {
Logger.tag('node', 'delete').error('Could not find node file: ' + file, error);
return callback({data: 'Could not delete node.', type: ErrorTypes.internalError});
}
try {
fs.unlinkSync(files[0]);
}
catch (error) {
Logger.tag('node', 'delete').error('Could not delete node file: ' + file, error);
return callback({data: 'Could not delete node.', type: ErrorTypes.internalError});
}
if (files.length !== 1) {
return callback({data: 'Node not found.', type: ErrorTypes.notFound});
}
return callback(null);
try {
fs.unlinkSync(files[0]);
}
catch (error) {
Logger.tag('node', 'delete').error('Could not delete node file: ' + file, error);
return callback({data: 'Could not delete node.', type: ErrorTypes.internalError});
}
return callback(null);
});
}
function parseNodeFile(file, callback) {
@ -263,14 +277,18 @@ angular.module('ffffng')
}
function findNodeDataByFilePattern(filter, callback) {
var files = findNodeFiles(filter);
findNodeFiles(filter, function (err, files) {
if (err) {
return callback(err);
}
if (files.length !== 1) {
return callback(null);
}
if (files.length !== 1) {
return callback(null);
}
var file = files[0];
return parseNodeFile(file, callback);
var file = files[0];
return parseNodeFile(file, callback);
});
}
function getNodeDataByFilePattern(filter, callback) {
@ -404,21 +422,26 @@ angular.module('ffffng')
},
getAllNodes: function (callback) {
var files = findNodeFiles({});
async.mapLimit(
files,
MAX_PARALLEL_NODES_PARSING,
parseNodeFile,
function (err, nodes) {
if (err) {
Logger.tag('nodes').error('Error getting all nodes:', error);
return callback({data: 'Internal error.', type: ErrorTypes.internalError});
}
return callback(null, nodes);
findNodeFiles({}, function (err, files) {
if (err) {
Logger.tag('nodes').error('Error getting all nodes:', error);
return callback({data: 'Internal error.', type: ErrorTypes.internalError});
}
);
async.mapLimit(
files,
MAX_PARALLEL_NODES_PARSING,
parseNodeFile,
function (err, nodes) {
if (err) {
Logger.tag('nodes').error('Error getting all nodes:', error);
return callback({data: 'Internal error.', type: ErrorTypes.internalError});
}
return callback(null, nodes);
}
);
});
},
findNodeDataByMac: function (mac, callback) {