From ad3f075d930613b41164dfe5f72083c1e296b939 Mon Sep 17 00:00:00 2001 From: baldo Date: Tue, 24 May 2016 14:42:25 +0200 Subject: [PATCH] WIP: Job to retrieve node information for monitoring. --- app/views/deleteNodeForm.html | 4 +- app/views/newNodeForm.html | 3 +- app/views/updateNodeForm.html | 4 +- config.json.example | 7 +- package.json | 1 + server/config.js | 7 +- .../db/patches/002_add-node-status-table.sql | 15 ++ server/jobs/mailQueueJob.js | 2 +- server/jobs/nodeInformationRetrievalJob.js | 13 ++ server/jobs/scheduler.js | 3 + server/libs.js | 1 + server/services/monitoringService.js | 203 +++++++++++++++++- server/services/nodeService.js | 22 +- server/validation/validator.js | 4 + 14 files changed, 274 insertions(+), 15 deletions(-) create mode 100644 server/db/patches/002_add-node-status-table.sql create mode 100644 server/jobs/nodeInformationRetrievalJob.js diff --git a/app/views/deleteNodeForm.html b/app/views/deleteNodeForm.html index 9cf734a..f422af1 100644 --- a/app/views/deleteNodeForm.html +++ b/app/views/deleteNodeForm.html @@ -30,8 +30,8 @@

Erledigt!

Die Daten Deines Freifunk-Knotens sind gelöscht worden. Es kann jetzt noch bis zu 20 Minuten dauern, - bis die Änderungen überall wirksam werden und sich im Knotengraph - und in der Knotenkarte auswirken. + bis die Änderungen überall wirksam werden und sich in der + Knotenkarte auswirken.

diff --git a/app/views/newNodeForm.html b/app/views/newNodeForm.html index cfb4013..c82af5c 100644 --- a/app/views/newNodeForm.html +++ b/app/views/newNodeForm.html @@ -17,8 +17,7 @@

Geschafft!

Dein Freifunk-Knoten ist erfolgreich angemeldet worden. Es kann jetzt noch bis zu 20 Minuten dauern, bis - Dein Knoten funktioniert und im Knotengraph - und in der Knotenkarte auftaucht. + Dein Knoten funktioniert und in der Knotenkarte auftaucht.

diff --git a/app/views/updateNodeForm.html b/app/views/updateNodeForm.html index f4e1e6b..1ca5fdd 100644 --- a/app/views/updateNodeForm.html +++ b/app/views/updateNodeForm.html @@ -30,8 +30,8 @@

Geschafft!

Die Daten Deines Freifunk-Knotens sind erfolgreich geändert worden. Es kann jetzt noch bis zu 20 Minuten dauern, - bis die Änderungen überall wirksam werden und sich im Knotengraph - und in der Knotenkarte auswirken. + bis die Änderungen überall wirksam werden und sich in der + Knotenkarte auswirken.

diff --git a/config.json.example b/config.json.example index 86cc6a3..9a87c81 100644 --- a/config.json.example +++ b/config.json.example @@ -18,6 +18,10 @@ "pass": "pass" } } + }, + + "map": { + "nodesJsonUrl": "https://map.hamburg.freifunk.net/nodes.json" } }, "client": { @@ -27,8 +31,7 @@ "contactEmail": "kontakt@hamburg.freifunk.net" }, "map": { - "graphUrl": "http://graph.hamburg.freifunk.net/graph.html", - "mapUrl": "http://graph.hamburg.freifunk.net/geomap.html" + "mapUrl": "http://map.hamburg.freifunk.net" }, "coordsSelector": { "lat": 53.565278, diff --git a/package.json b/package.json index 546532c..f9f14ae 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "node-cron": "~1.1.1", "nodemailer": "~2.4.1", "nodemailer-html-to-text": "~2.1.0", + "request": "~2.72.0", "serve-static": "~1.10.2", "sqlite3": "~3.1.4", "time-grunt": "~1.3.0" diff --git a/server/config.js b/server/config.js index b75e99b..b40b95f 100644 --- a/server/config.js +++ b/server/config.js @@ -24,6 +24,10 @@ var defaultConfig = { pass: 'pass' } } + }, + + map: { + nodesJsonUrl: 'http://map.musterstadt.freifunk.net/nodes.json' } }, client: { @@ -33,8 +37,7 @@ var defaultConfig = { contactEmail: 'kontakt@musterstadt.freifunk.net' }, map: { - graphUrl: 'http://graph.musterstadt.freifunk.net/graph.html', - mapUrl: 'http://graph.musterstadt.freifunk.net/geomap.html' + mapUrl: 'http://map.musterstadt.freifunk.net' }, monitoring: { enabled: true diff --git a/server/db/patches/002_add-node-status-table.sql b/server/db/patches/002_add-node-status-table.sql new file mode 100644 index 0000000..970e562 --- /dev/null +++ b/server/db/patches/002_add-node-status-table.sql @@ -0,0 +1,15 @@ +CREATE TABLE node_state ( + id INTEGER PRIMARY KEY, + + mac VARCHAR(17) NOT NULL UNIQUE, + + state VARCHAR(10) NOT NULL, + last_seen DATETIME NOT NULL, + + import_timestamp DATETIME NOT NULL, + + last_status_mail_send DATETIME, + + created_at DATETIME DEFAULT (strftime('%s','now')) NOT NULL, + modified_at DATETIME DEFAULT (strftime('%s','now')) NOT NULL +); diff --git a/server/jobs/mailQueueJob.js b/server/jobs/mailQueueJob.js index 39878cb..2846ba0 100644 --- a/server/jobs/mailQueueJob.js +++ b/server/jobs/mailQueueJob.js @@ -1,6 +1,6 @@ 'use strict'; -angular.module('ffffng').factory('MailQueueJob', function (Database, MailService) { +angular.module('ffffng').factory('MailQueueJob', function (MailService) { return { run: function () { MailService.sendPendingMails(function (err) { diff --git a/server/jobs/nodeInformationRetrievalJob.js b/server/jobs/nodeInformationRetrievalJob.js new file mode 100644 index 0000000..1cbe9e9 --- /dev/null +++ b/server/jobs/nodeInformationRetrievalJob.js @@ -0,0 +1,13 @@ +'use strict'; + +angular.module('ffffng').factory('NodeInformationRetrievalJob', function (MonitoringService) { + return { + run: function () { + MonitoringService.retrieveNodeInformation(function (err) { + if (err) { + console.error(err); + } + }); + } + }; +}); diff --git a/server/jobs/scheduler.js b/server/jobs/scheduler.js index 5a72515..24da90c 100644 --- a/server/jobs/scheduler.js +++ b/server/jobs/scheduler.js @@ -24,6 +24,9 @@ angular.module('ffffng').factory('Scheduler', function ($injector) { return { init: function () { schedule('*/5 * * * * *', 'MailQueueJob'); + // schedule('0 */1 * * * *', 'NodeInformationRetrievalJob'); + schedule('*/10 * * * * *', 'NodeInformationRetrievalJob'); + // schedule('0 */1 * * * *', 'NodeInformationCleanupJob'); } }; }); diff --git a/server/libs.js b/server/libs.js index 9c3b047..2889df8 100644 --- a/server/libs.js +++ b/server/libs.js @@ -20,4 +20,5 @@ lib('fs'); lib('glob'); lib('moment'); + lib('request'); })(); diff --git a/server/services/monitoringService.js b/server/services/monitoringService.js index 3cd8010..21c1dc1 100644 --- a/server/services/monitoringService.js +++ b/server/services/monitoringService.js @@ -1,8 +1,169 @@ 'use strict'; angular.module('ffffng') -.service('MonitoringService', function (NodeService, ErrorTypes) { - return { +.service('MonitoringService', function ( + _, + async, + config, + Database, + ErrorTypes, + moment, + NodeService, + request, + Strings, + Validator, + Constraints +) { + var previousImportTimestamp = null; + + function insertNodeInformation(nodeData, node, callback) { + return Database.run( + 'INSERT INTO node_state ' + + '(mac, state, last_seen, import_timestamp, last_status_mail_send) ' + + 'VALUES (?, ?, ?, ?, ?)', + [ + node.mac, + nodeData.state, + nodeData.lastSeen.unix(), + nodeData.importTimestamp.unix(), + null // new node so we haven't send a mail yet + ], + callback + ); + } + + function updateNodeInformation(nodeData, node, row, callback) { + debugger; + if (!moment(row.import_timestamp).isBefore(nodeData.importTimestamp)) { + return callback(); + } + + return Database.run( + 'UPDATE node_state ' + + 'WHERE id = ? AND mac = ?' + + 'SET state = ?, last_seen = ?, import_timestamp = ?, modified_at = ?', + [ + row.id, node.mac, + nodeData.state, nodeData.lastSeen.unix(), nodeData.importTimestamp.unix(), moment.unix() + ], + callback + ); + } + + function deleteNodeInformation(nodeData, node, callback) { + return Database.run( + 'DELETE FROM node_state WHERE mac = ? AND import_timestamp < ?', + [node.mac, nodeData.importTimestamp.unix()], + callback + ); + } + + function storeNodeInformation(nodeData, node, callback) { + if (node.monitoring && node.monitoringConfirmed) { + return Database.get('SELECT * FROM node_state WHERE mac = ?', [node.mac], function (err, row) { + if (err) { + return callback(err); + } + + if (_.isUndefined(row)) { + return insertNodeInformation(nodeData, node, callback); + } else { + return updateNodeInformation(nodeData, node, row, callback); + } + }); + } else { + return deleteNodeInformation(node, callback); + } + } + + var isValidMac = Validator.forConstraint(Constraints.node.mac); + + function parseNodesJson(body, callback) { + function parseTimestamp(timestamp) { + if (!_.isString(json.timestamp)) { + return moment.invalid(); + } + return moment(timestamp); + } + + var data = {}; + try { + var json = JSON.parse(body); + + if (json.version !== 1) { + return callback(new Error('Unexpected nodes.json version: ' + json.version)); + } + + data.importTimestamp = parseTimestamp(json.timestamp); + if (!data.importTimestamp.isValid()) { + return callback(new Error('Invalid timestamp: ' + json.timestamp)); + } + + if (!_.isPlainObject(json.nodes)) { + return callback(new Error('Invalid nodes object type: ' + (typeof json.nodes))); + } + + data.nodes = _.values(_.map(json.nodes, function (nodeData, nodeId) { + if (!_.isPlainObject(nodeData)) { + throw new Error( + 'Node ' + nodeId + ': Unexpected node type: ' + (typeof nodeData) + ); + } + + if (!_.isPlainObject(nodeData.nodeinfo)) { + throw new Error( + 'Node ' + nodeId + ': Unexpected nodeinfo type: ' + (typeof nodeData.nodeinfo) + ); + } + if (!_.isPlainObject(nodeData.nodeinfo.network)) { + throw new Error( + 'Node ' + nodeId + ': Unexpected nodeinfo.network type: ' + + (typeof nodeData.nodeinfo.network) + ); + } + + if (!isValidMac(nodeData.nodeinfo.network.mac)) { + throw new Error( + 'Node ' + nodeId + ': Invalid MAC: ' + nodeData.nodeinfo.network.mac + ); + } + var mac = Strings.normalizeMac(nodeData.nodeinfo.network.mac); + + if (!_.isPlainObject(nodeData.flags)) { + throw new Error( + 'Node ' + nodeId + ': Unexpected flags type: ' + (typeof nodeData.flags) + ); + } + if (!_.isBoolean(nodeData.flags.online)) { + throw new Error( + 'Node ' + nodeId + ': Unexpected flags.online type: ' + (typeof nodeData.flags.online) + ); + } + var isOnline = nodeData.flags.online; + + var lastSeen = parseTimestamp(nodeData.lastseen); + if (!lastSeen.isValid()) { + throw new Error( + 'Node ' + nodeId + ': Invalid lastseen timestamp: ' + nodeData.lastseen + ); + } + + return { + mac: mac, + importTimestamp: data.importTimestamp, + state: isOnline ? 'ONLINE' : 'OFFLINE', + lastSeen: lastSeen + }; + })); + } + catch (error) { + return callback(error); + } + + callback(null, data); + } + + return { confirm: function (token, callback) { NodeService.getNodeDataByMonitoringToken(token, function (err, node, nodeSecrets) { if (err) { @@ -48,6 +209,44 @@ angular.module('ffffng') callback(null, node); }); }); + }, + + retrieveNodeInformation: function (callback) { + console.info(); + request(config.server.map.nodesJsonUrl, function (err, response, body) { + if (err) { + return callback(err); + } + + parseNodesJson(body, function (err, data) { + if (err) { + return callback(err); + } + + if (previousImportTimestamp !== null && !data.importTimestamp.isAfter(previousImportTimestamp)) { + return callback(); + } + previousImportTimestamp = data.importTimestamp; + + async.each( + data.nodes, + function (nodeData, nodeCallback) { + NodeService.findNodeDataByMac(nodeData.mac, function (err, node) { + if (err) { + return nodeCallback(err); + } + + if (!node) { + return nodeCallback(null); + } + + storeNodeInformation(nodeData, node, nodeCallback); + }); + }, + callback + ); + }); + }); } }; }); diff --git a/server/services/nodeService.js b/server/services/nodeService.js index 09493e5..afa7af3 100644 --- a/server/services/nodeService.js +++ b/server/services/nodeService.js @@ -224,17 +224,31 @@ angular.module('ffffng') callback(null, node, nodeSecrets); } - function getNodeDataByFilePattern(filter, callback) { + function findNodeDataByFilePattern(filter, callback) { var files = findNodeFiles(filter); if (files.length !== 1) { - return callback({data: 'Node not found.', type: ErrorTypes.notFound}); + return callback(null); } var file = files[0]; return parseNodeFile(file, callback); } + function getNodeDataByFilePattern(filter, callback) { + findNodeDataByFilePattern(filter, function (err, node, nodeSecrets) { + if (err) { + return callback(err); + } + + if (!node) { + return callback({data: 'Node not found.', type: ErrorTypes.notFound}); + } + + callback(null, node, nodeSecrets); + }); + } + function sendMonitoringConfirmationMail(node, nodeSecrets, callback) { var confirmUrl = UrlBuilder.monitoringConfirmUrl(nodeSecrets); var disableUrl = UrlBuilder.monitoringDisableUrl(nodeSecrets); @@ -351,6 +365,10 @@ angular.module('ffffng') deleteNodeFile(token, callback); }, + findNodeDataByMac: function (mac, callback) { + return findNodeDataByFilePattern({ mac: mac }, callback); + }, + getNodeDataByToken: function (token, callback) { return getNodeDataByFilePattern({ token: token }, callback); }, diff --git a/server/validation/validator.js b/server/validation/validator.js index 287d7bd..ad814b6 100644 --- a/server/validation/validator.js +++ b/server/validation/validator.js @@ -6,6 +6,10 @@ angular.module('ffffng').factory('Validator', function (_) { return acceptUndefined || constraint.optional; } + if (!_.isString(value)) { + return false; + } + var trimmed = value.trim(); return (trimmed === '' && constraint.optional) || trimmed.match(constraint.regex); };