freifunk/ffffng
0
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Fix: Correct support for multiple nodes.json files.

Browse source
This commit is contained in:
baldo 2017-03-06 20:32:52 +01:00
parent 966a714f77
commit f12ef87567
1 changed files with 80 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

141
server/services/monitoringService.js
View file

@ -418,84 +418,100 @@ angular.module('ffffng')
}); });
} }
function retrieveNodeInformationForUrl(url, callback) { function withUrlsData(urls, callback) {
Logger.tag('monitoring', 'information-retrieval').debug('Retrieving nodes.json: %s', url); async.map(urls, function (url, urlCallback) {
request(url, function (err, response, body) { Logger.tag('monitoring', 'information-retrieval').debug('Retrieving nodes.json: %s', url);
request(url, function (err, response, body) {
if (err) {
return urlCallback(err);
}
if (response.statusCode !== 200) {
return urlCallback(new Error(
'Could not download nodes.json from ' + url + ': ' +
response.statusCode + ' - ' + response.statusMessage
));
}
parseNodesJson(body, urlCallback);
});
}, callback);
}
function retrieveNodeInformationForUrls(urls, callback) {
withUrlsData(urls, function (err, datas) {
if (err) { if (err) {
return callback(err); return callback(err);
} }
if (response.statusCode !== 200) { var maxTimestamp = datas[0].importTimestamp;
return callback(new Error( _.each(datas, function (data) {
'Could not download nodes.json from ' + url + ': ' + if (data.importTimestamp.isAfter(maxTimestamp)) {
response.statusCode + ' - ' + response.statusMessage maxTimestamp = data.importTimestamp;
)); }
});
if (previousImportTimestamp !== null && !maxTimestamp.isAfter(previousImportTimestamp)) {
Logger
.tag('monitoring', 'information-retrieval')
.debug(
'No new data, skipping. Current timestamp: %s, previous timestamp: %s',
maxTimestamp.format(),
previousImportTimestamp.format()
);
return callback();
} }
previousImportTimestamp = maxTimestamp;
parseNodesJson(body, function (err, data) { // We do not parallelize here as the sqlite will start slowing down and blocking with too many
if (err) { // parallel queries. This has resulted in blocking other requests too and thus in a major slowdonw.
return callback(err); var nodes = _.flatMap(datas, function (data) {
} return data.nodes;
});
async.eachSeries(
nodes,
function (nodeData, nodeCallback) {
Logger.tag('monitoring', 'information-retrieval').debug('Importing: %s', nodeData.mac);
if (previousImportTimestamp !== null && !data.importTimestamp.isAfter(previousImportTimestamp)) { NodeService.getNodeDataByMac(nodeData.mac, function (err, node) {
Logger if (err) {
.tag('monitoring', 'information-retrieval') Logger
.debug( .tag('monitoring', 'information-retrieval')
'No new data, skipping. Current timestamp: %s, previous timestamp: %s', .error('Error importing: ' + nodeData.mac, err);
data.importTimestamp.format(), return nodeCallback(err);
previousImportTimestamp.format() }
);
return callback();
}
previousImportTimestamp = data.importTimestamp;
// We do not parallelize here as the sqlite will start slowing down and blocking with too many if (!node) {
// parallel queries. This has resulted in blocking other requests too and thus in a major slowdonw. Logger
async.eachSeries( .tag('monitoring', 'information-retrieval')
data.nodes, .debug('Unknown node, skipping: %s', nodeData.mac);
function (nodeData, nodeCallback) { return nodeCallback(null);
Logger.tag('monitoring', 'information-retrieval').debug('Importing: %s', nodeData.mac); }
NodeService.getNodeDataByMac(nodeData.mac, function (err, node) { storeNodeInformation(nodeData, node, function (err) {
if (err) { if (err) {
Logger Logger
.tag('monitoring', 'information-retrieval') .tag('monitoring', 'information-retrieval')
.error('Error importing: ' + nodeData.mac, err); .debug('Could not update / deleting node data: %s', nodeData.mac, err);
return nodeCallback(err); return nodeCallback(err);
} }
if (!node) { Logger
Logger .tag('monitoring', 'information-retrieval')
.tag('monitoring', 'information-retrieval') .debug('Updating / deleting node data done: %s', nodeData.mac);
.debug('Unknown node, skipping: %s', nodeData.mac);
return nodeCallback(null);
}
storeNodeInformation(nodeData, node, function (err) { nodeCallback();
if (err) {
Logger
.tag('monitoring', 'information-retrieval')
.debug('Could not update / deleting node data: %s', nodeData.mac, err);
return nodeCallback(err);
}
Logger
.tag('monitoring', 'information-retrieval')
.debug('Updating / deleting node data done: %s', nodeData.mac);
nodeCallback();
});
}); });
}, });
function (err) { },
if (err) { function (err) {
return callback(err); if (err) {
} return callback(err);
markMissingNodesAsOffline(data.nodes, callback);
} }
);
}); markMissingNodesAsOffline(nodes, callback);
}
);
}); });
} }
@ -636,11 +652,14 @@ angular.module('ffffng')
retrieveNodeInformation: function (callback) { retrieveNodeInformation: function (callback) {
var urls = config.server.map.nodesJsonUrl; var urls = config.server.map.nodesJsonUrl;
if (_.isEmpty(urls)) {
return callback(null);
}
if (_.isString(urls)) { if (_.isString(urls)) {
urls = [urls]; urls = [urls];
} }
async.eachSeries(urls, retrieveNodeInformationForUrl, callback); retrieveNodeInformationForUrls(urls, callback);
}, },
sendMonitoringMails: function (callback) { sendMonitoringMails: function (callback) {