Fix: More asynchronous glob operations and graceful-fs.

Hope this fixes the "Error: EMFILE, too many open files" in
NodeInformationRetrievalJob.

See: https://github.com/freifunkhamburg/ffffng/issues/13

package.json

@@ -32,6 +32,7 @@
         "deep-extend": "0.4.1",
         "express": "4.14.0",
         "glob": "7.0.5",
+        "graceful-fs": "4.1.4",
         "http-auth": "2.4.4",
         "http-errors": "1.5.0",
         "lodash": "4.13.1",

server/config.js

@@ -28,7 +28,7 @@ if (commandLineOptions.help || !commandLineOptions.config) {
 }
 
 
-var fs = require('fs');
+var fs = require('graceful-fs');
 var deepExtend = require('deep-extend');
 
 var defaultConfig = {

server/db/database.js

@@ -1,7 +1,7 @@
 'use strict';
 
 var async = require('async');
-var fs = require('fs');
+var fs = require('graceful-fs');
 var glob = require('glob');
 var path = require('path');
 

server/libs.js

@@ -17,7 +17,7 @@
     lib('async');
     lib('crypto');
     lib('deepExtend', 'deep-extend');
-    lib('fs');
+    lib('fs', 'graceful-fs');
     lib('glob');
     lib('moment');
     lib('request');

server/main.js

@@ -7,6 +7,13 @@ global.angular = require('ng-di');
 
 angular.module('ffffng', []);
 
+(function () {
+    // Use graceful-fs instead of fs also in all libraries to have more robust fs handling.
+    var realFs = require('fs');
+    var gracefulFs = require('graceful-fs');
+    gracefulFs.gracefulify(realFs);
+})();
+
 require('./config');
 
 require('./logger').tag('main', 'startup').info('Server starting up...');

server/services/nodeService.js

@@ -33,7 +33,7 @@ angular.module('ffffng')
         return crypto.randomBytes(8).toString('hex');
     }
 
-    function findNodeFiles(filter) {
+    function toNodeFilesPattern(filter) {
         var pattern = _.join(
             _.map(filenameParts, function (field) {
                 return filter.hasOwnProperty(field) ? filter[field] : '*';
@@ -41,7 +41,15 @@ angular.module('ffffng')
             '@'
         );
 
-        return glob.sync(config.server.peersPath + '/' + pattern.toLowerCase());
+        return config.server.peersPath + '/' + pattern.toLowerCase();
+    }
+
+    function findNodeFiles(filter, callback) {
+        glob(toNodeFilesPattern(filter), callback);
+    }
+
+    function findNodeFilesSync(filter) {
+        return glob.sync(toNodeFilesPattern(filter));
     }
 
     function findFilesInPeersPath(callback) {
@@ -76,7 +84,7 @@ angular.module('ffffng')
     }
 
     function isDuplicate(filter, token) {
-        var files = findNodeFiles(filter);
+        var files = findNodeFilesSync(filter);
         if (files.length === 0) {
             return false;
         }
@@ -159,7 +167,7 @@ angular.module('ffffng')
         var error;
 
         if (isUpdate) {
-            var files = findNodeFiles({ token: token });
+            var files = findNodeFilesSync({ token: token });
             if (files.length !== 1) {
                 return callback({data: 'Node not found.', type: ErrorTypes.notFound});
             }
@@ -196,20 +204,26 @@ angular.module('ffffng')
     }
 
     function deleteNodeFile(token, callback) {
-        var files = findNodeFiles({ token: token });
+        findNodeFiles({ token: token }, function (err, files) {
-        if (files.length !== 1) {
+            if (err) {
-            return callback({data: 'Node not found.', type: ErrorTypes.notFound});
+                Logger.tag('node', 'delete').error('Could not find node file: ' + file, error);
-        }
+                return callback({data: 'Could not delete node.', type: ErrorTypes.internalError});
+            }
 
-        try {
+            if (files.length !== 1) {
-            fs.unlinkSync(files[0]);
+                return callback({data: 'Node not found.', type: ErrorTypes.notFound});
-        }
+            }
-        catch (error) {
-            Logger.tag('node', 'delete').error('Could not delete node file: ' + file, error);
-            return callback({data: 'Could not delete node.', type: ErrorTypes.internalError});
-        }
 
-        return callback(null);
+            try {
+                fs.unlinkSync(files[0]);
+            }
+            catch (error) {
+                Logger.tag('node', 'delete').error('Could not delete node file: ' + file, error);
+                return callback({data: 'Could not delete node.', type: ErrorTypes.internalError});
+            }
+
+            return callback(null);
+        });
     }
 
     function parseNodeFile(file, callback) {
@@ -263,14 +277,18 @@ angular.module('ffffng')
     }
 
     function findNodeDataByFilePattern(filter, callback) {
-        var files = findNodeFiles(filter);
+        findNodeFiles(filter, function (err, files) {
+            if (err) {
+                return callback(err);
+            }
 
-        if (files.length !== 1) {
+            if (files.length !== 1) {
-            return callback(null);
+                return callback(null);
-        }
+            }
 
-        var file = files[0];
+            var file = files[0];
-        return parseNodeFile(file, callback);
+            return parseNodeFile(file, callback);
+        });
     }
 
     function getNodeDataByFilePattern(filter, callback) {
@@ -404,21 +422,26 @@ angular.module('ffffng')
         },
 
         getAllNodes: function (callback) {
-            var files = findNodeFiles({});
+            findNodeFiles({}, function (err, files) {
-
+                if (err) {
-            async.mapLimit(
+                    Logger.tag('nodes').error('Error getting all nodes:', error);
-                files,
+                    return callback({data: 'Internal error.', type: ErrorTypes.internalError});
-                MAX_PARALLEL_NODES_PARSING,
-                parseNodeFile,
-                function (err, nodes) {
-                    if (err) {
-                        Logger.tag('nodes').error('Error getting all nodes:', error);
-                        return callback({data: 'Internal error.', type: ErrorTypes.internalError});
-                    }
-
-                    return callback(null, nodes);
                 }
-            );
+
+                async.mapLimit(
+                    files,
+                    MAX_PARALLEL_NODES_PARSING,
+                    parseNodeFile,
+                    function (err, nodes) {
+                        if (err) {
+                            Logger.tag('nodes').error('Error getting all nodes:', error);
+                            return callback({data: 'Internal error.', type: ErrorTypes.internalError});
+                        }
+
+                        return callback(null, nodes);
+                    }
+                );
+            });
         },
 
         findNodeDataByMac: function (mac, callback) {