diff --git a/README.md b/README.md index 9a82730..4d36909 100644 --- a/README.md +++ b/README.md @@ -153,7 +153,7 @@ Dann die `config.json` anpassen nach belieben. Es gibt die folgenden Konfigurati } ``` -* **`server.map.nodesJsonUrl`** URL der `nodes.json` des meshviewers, z. B.: `"http://musterstadt.freifunk.net/nodes.json"` +* **`server.map.nodesJsonUrl`** URL der `nodes.json` des meshviewers (kann eine URL oder eine Liste von URLs sein), z. B.: `["http://musterstadt.freifunk.net/nodes.json"]` * **`client.community.name`** Name der Freifunk-Community, z. B.: `"Freifunk Musterstadt"` * **`client.community.domain`** Domain der Freifunk-Community, z. B.: `"musterstadt.freifunk.net"` diff --git a/config.json.example b/config.json.example index ce2c693..5b5a5f4 100644 --- a/config.json.example +++ b/config.json.example @@ -34,7 +34,7 @@ }, "map": { - "nodesJsonUrl": "http://musterstadt.freifunk.net/nodes.json" + "nodesJsonUrl": ["http://musterstadt.freifunk.net/nodes.json"] } }, "client": { diff --git a/server/config.js b/server/config.js index 5cae933..ad70823 100644 --- a/server/config.js +++ b/server/config.js @@ -89,7 +89,7 @@ var defaultConfig = { }, map: { - nodesJsonUrl: 'http://map.musterstadt.freifunk.net/nodes.json' + nodesJsonUrl: ['http://map.musterstadt.freifunk.net/nodes.json'] } }, client: { diff --git a/server/services/monitoringService.js b/server/services/monitoringService.js index 8465088..046f0f0 100644 --- a/server/services/monitoringService.js +++ b/server/services/monitoringService.js @@ -418,6 +418,87 @@ angular.module('ffffng') }); } + function retrieveNodeInformationForUrl(url, callback) { + Logger.tag('monitoring', 'information-retrieval').debug('Retrieving nodes.json: %s', url); + request(url, function (err, response, body) { + if (err) { + return callback(err); + } + + if (response.statusCode !== 200) { + return callback(new Error( + 'Could not download nodes.json from ' + url + ': ' + + response.statusCode + ' - ' + response.statusMessage + )); + } + + parseNodesJson(body, function (err, data) { + if (err) { + return callback(err); + } + + if (previousImportTimestamp !== null && !data.importTimestamp.isAfter(previousImportTimestamp)) { + Logger + .tag('monitoring', 'information-retrieval') + .debug( + 'No new data, skipping. Current timestamp: %s, previous timestamp: %s', + data.importTimestamp.format(), + previousImportTimestamp.format() + ); + return callback(); + } + previousImportTimestamp = data.importTimestamp; + + // We do not parallelize here as the sqlite will start slowing down and blocking with too many + // parallel queries. This has resulted in blocking other requests too and thus in a major slowdonw. + async.eachSeries( + data.nodes, + function (nodeData, nodeCallback) { + Logger.tag('monitoring', 'information-retrieval').debug('Importing: %s', nodeData.mac); + + NodeService.getNodeDataByMac(nodeData.mac, function (err, node) { + if (err) { + Logger + .tag('monitoring', 'information-retrieval') + .error('Error importing: ' + nodeData.mac, err); + return nodeCallback(err); + } + + if (!node) { + Logger + .tag('monitoring', 'information-retrieval') + .debug('Unknown node, skipping: %s', nodeData.mac); + return nodeCallback(null); + } + + storeNodeInformation(nodeData, node, function (err) { + if (err) { + Logger + .tag('monitoring', 'information-retrieval') + .debug('Could not update / deleting node data: %s', nodeData.mac, err); + return nodeCallback(err); + } + + Logger + .tag('monitoring', 'information-retrieval') + .debug('Updating / deleting node data done: %s', nodeData.mac); + + nodeCallback(); + }); + }); + }, + function (err) { + if (err) { + return callback(err); + } + + markMissingNodesAsOffline(data.nodes, callback); + } + ); + }); + }); + } + return { getAll: function (restParams, callback) { var sortFields = [ @@ -554,85 +635,12 @@ angular.module('ffffng') }, retrieveNodeInformation: function (callback) { - var url = config.server.map.nodesJsonUrl; - Logger.tag('monitoring', 'information-retrieval').debug('Retrieving nodes.json: %s', url); - request(url, function (err, response, body) { - if (err) { - return callback(err); - } + var urls = config.server.map.nodesJsonUrl; + if (_.isString(urls)) { + urls = [urls]; + } - if (response.statusCode !== 200) { - return callback(new Error( - 'Could not download nodes.json from ' + url + ': ' + - response.statusCode + ' - ' + response.statusMessage - )); - } - - parseNodesJson(body, function (err, data) { - if (err) { - return callback(err); - } - - if (previousImportTimestamp !== null && !data.importTimestamp.isAfter(previousImportTimestamp)) { - Logger - .tag('monitoring', 'information-retrieval') - .debug( - 'No new data, skipping. Current timestamp: %s, previous timestamp: %s', - data.importTimestamp.format(), - previousImportTimestamp.format() - ); - return callback(); - } - previousImportTimestamp = data.importTimestamp; - - // We do not parallelize here as the sqlite will start slowing down and blocking with too many - // parallel queries. This has resulted in blocking other requests too and thus in a major slowdonw. - async.eachSeries( - data.nodes, - function (nodeData, nodeCallback) { - Logger.tag('monitoring', 'information-retrieval').debug('Importing: %s', nodeData.mac); - - NodeService.getNodeDataByMac(nodeData.mac, function (err, node) { - if (err) { - Logger - .tag('monitoring', 'information-retrieval') - .error('Error importing: ' + nodeData.mac, err); - return nodeCallback(err); - } - - if (!node) { - Logger - .tag('monitoring', 'information-retrieval') - .debug('Unknown node, skipping: %s', nodeData.mac); - return nodeCallback(null); - } - - storeNodeInformation(nodeData, node, function (err) { - if (err) { - Logger - .tag('monitoring', 'information-retrieval') - .debug('Could not update / deleting node data: %s', nodeData.mac, err); - return nodeCallback(err); - } - - Logger - .tag('monitoring', 'information-retrieval') - .debug('Updating / deleting node data done: %s', nodeData.mac); - - nodeCallback(); - }); - }); - }, - function (err) { - if (err) { - return callback(err); - } - - markMissingNodesAsOffline(data.nodes, callback); - } - ); - }); - }); + async.eachSeries(urls, retrieveNodeInformationForUrl, callback); }, sendMonitoringMails: function (callback) {