WIP: Job to retrieve node information for monitoring.

app/views/deleteNodeForm.html

@@ -30,8 +30,8 @@
     <h1>Erledigt!</h1>
     <p>
         Die Daten Deines Freifunk-Knotens sind gelöscht worden. Es kann jetzt noch bis zu 20 Minuten dauern,
-        bis die Änderungen überall wirksam werden und sich im <a href="{{ config.map.graphUrl }}" target="_blank">Knotengraph</a>
+        bis die Änderungen überall wirksam werden und sich in der
-        und in der <a href="{{ config.map.mapUrl }}" target="_blank">Knotenkarte</a> auswirken.
+        <a href="{{ config.map.mapUrl }}" target="_blank">Knotenkarte</a> auswirken.
     </p>
 
     <div class="summary">

app/views/newNodeForm.html

@@ -17,8 +17,7 @@
     <h1>Geschafft!</h1>
     <p>
         Dein Freifunk-Knoten ist erfolgreich angemeldet worden. Es kann jetzt noch bis zu 20 Minuten dauern, bis
-        Dein Knoten funktioniert und im <a href="{{ config.map.graphUrl }}" target="_blank">Knotengraph</a>
+        Dein Knoten funktioniert und in der <a href="{{ config.map.mapUrl }}" target="_blank">Knotenkarte</a> auftaucht.
-        und in der <a href="{{ config.map.mapUrl }}" target="_blank">Knotenkarte</a> auftaucht.
     </p>
 
     <f-node-saved f-node="node" f-token="token"></f-node-saved>

app/views/updateNodeForm.html

@@ -30,8 +30,8 @@
     <h1>Geschafft!</h1>
     <p>
         Die Daten Deines Freifunk-Knotens sind erfolgreich geändert worden. Es kann jetzt noch bis zu 20 Minuten dauern,
-        bis die Änderungen überall wirksam werden und sich im <a href="{{ config.map.graphUrl }}" target="_blank">Knotengraph</a>
+        bis die Änderungen überall wirksam werden und sich in der
-        und in der <a href="{{ config.map.mapUrl }}" target="_blank">Knotenkarte</a> auswirken.
+        <a href="{{ config.map.mapUrl }}" target="_blank">Knotenkarte</a> auswirken.
     </p>
 
     <f-node-saved f-node="node" f-token="token"></f-node-saved>

config.json.example

@@ -18,6 +18,10 @@
                     "pass": "pass"
                 }
             }
+        },
+
+        "map": {
+            "nodesJsonUrl": "https://map.hamburg.freifunk.net/nodes.json"
         }
     },
     "client": {
@@ -27,8 +31,7 @@
             "contactEmail": "kontakt@hamburg.freifunk.net"
         },
         "map": {
-            "graphUrl": "http://graph.hamburg.freifunk.net/graph.html",
+            "mapUrl": "http://map.hamburg.freifunk.net"
-            "mapUrl": "http://graph.hamburg.freifunk.net/geomap.html"
         },
         "coordsSelector": {
             "lat": 53.565278,

package.json

@@ -44,6 +44,7 @@
         "node-cron": "~1.1.1",
         "nodemailer": "~2.4.1",
         "nodemailer-html-to-text": "~2.1.0",
+        "request": "~2.72.0",
         "serve-static": "~1.10.2",
         "sqlite3": "~3.1.4",
         "time-grunt": "~1.3.0"

server/config.js

@@ -24,6 +24,10 @@ var defaultConfig = {
                     pass: 'pass'
                 }
             }
+        },
+
+        map: {
+            nodesJsonUrl: 'http://map.musterstadt.freifunk.net/nodes.json'
         }
     },
     client: {
@@ -33,8 +37,7 @@ var defaultConfig = {
             contactEmail: 'kontakt@musterstadt.freifunk.net'
         },
         map: {
-            graphUrl: 'http://graph.musterstadt.freifunk.net/graph.html',
+            mapUrl: 'http://map.musterstadt.freifunk.net'
-            mapUrl: 'http://graph.musterstadt.freifunk.net/geomap.html'
         },
         monitoring: {
             enabled: true

server/db/patches/002_add-node-status-table.sql

@@ -0,0 +1,15 @@
+CREATE TABLE node_state (
+    id INTEGER PRIMARY KEY,
+
+    mac VARCHAR(17) NOT NULL UNIQUE,
+
+    state VARCHAR(10) NOT NULL,
+    last_seen DATETIME NOT NULL,
+
+    import_timestamp DATETIME NOT NULL,
+
+    last_status_mail_send DATETIME,
+
+    created_at DATETIME DEFAULT (strftime('%s','now')) NOT NULL,
+    modified_at DATETIME DEFAULT (strftime('%s','now')) NOT NULL
+);

server/jobs/mailQueueJob.js

@@ -1,6 +1,6 @@
 'use strict';
 
-angular.module('ffffng').factory('MailQueueJob', function (Database, MailService) {
+angular.module('ffffng').factory('MailQueueJob', function (MailService) {
     return {
         run: function () {
             MailService.sendPendingMails(function (err) {

server/jobs/nodeInformationRetrievalJob.js

@@ -0,0 +1,13 @@
+'use strict';
+
+angular.module('ffffng').factory('NodeInformationRetrievalJob', function (MonitoringService) {
+    return {
+        run: function () {
+            MonitoringService.retrieveNodeInformation(function (err) {
+                if (err) {
+                    console.error(err);
+                }
+            });
+        }
+    };
+});

server/jobs/scheduler.js

@@ -24,6 +24,9 @@ angular.module('ffffng').factory('Scheduler', function ($injector) {
     return {
         init: function () {
             schedule('*/5 * * * * *', 'MailQueueJob');
+            // schedule('0 */1 * * * *', 'NodeInformationRetrievalJob');
+            schedule('*/10 * * * * *', 'NodeInformationRetrievalJob');
+            // schedule('0 */1 * * * *', 'NodeInformationCleanupJob');
         }
     };
 });

server/libs.js

@@ -20,4 +20,5 @@
     lib('fs');
     lib('glob');
     lib('moment');
+    lib('request');
 })();

server/services/monitoringService.js

@@ -1,8 +1,169 @@
 'use strict';
 
 angular.module('ffffng')
-.service('MonitoringService', function (NodeService, ErrorTypes) {
+.service('MonitoringService', function (
-    return {
+        _,
+        async,
+        config,
+        Database,
+        ErrorTypes,
+        moment,
+        NodeService,
+        request,
+        Strings,
+        Validator,
+        Constraints
+) {
+        var previousImportTimestamp = null;
+
+        function insertNodeInformation(nodeData, node, callback) {
+            return Database.run(
+                'INSERT INTO node_state ' +
+                '(mac, state, last_seen, import_timestamp, last_status_mail_send) ' +
+                'VALUES (?, ?, ?, ?, ?)',
+                [
+                    node.mac,
+                    nodeData.state,
+                    nodeData.lastSeen.unix(),
+                    nodeData.importTimestamp.unix(),
+                    null // new node so we haven't send a mail yet
+                ],
+                callback
+            );
+        }
+
+        function updateNodeInformation(nodeData, node, row, callback) {
+            debugger;
+            if (!moment(row.import_timestamp).isBefore(nodeData.importTimestamp)) {
+                return callback();
+            }
+
+            return Database.run(
+                'UPDATE node_state ' +
+                'WHERE id = ? AND mac = ?' +
+                'SET state = ?, last_seen = ?, import_timestamp = ?, modified_at = ?',
+                [
+                    row.id, node.mac,
+                    nodeData.state, nodeData.lastSeen.unix(), nodeData.importTimestamp.unix(), moment.unix()
+                ],
+                callback
+            );
+        }
+
+        function deleteNodeInformation(nodeData, node, callback) {
+            return Database.run(
+                'DELETE FROM node_state WHERE mac = ? AND import_timestamp < ?',
+                [node.mac, nodeData.importTimestamp.unix()],
+                callback
+            );
+        }
+
+        function storeNodeInformation(nodeData, node, callback) {
+            if (node.monitoring && node.monitoringConfirmed) {
+                return Database.get('SELECT * FROM node_state WHERE mac = ?', [node.mac], function (err, row) {
+                    if (err) {
+                        return callback(err);
+                    }
+
+                    if (_.isUndefined(row)) {
+                        return insertNodeInformation(nodeData, node, callback);
+                    } else {
+                        return updateNodeInformation(nodeData, node, row, callback);
+                    }
+                });
+            } else {
+                return deleteNodeInformation(node, callback);
+            }
+        }
+
+        var isValidMac = Validator.forConstraint(Constraints.node.mac);
+
+        function parseNodesJson(body, callback) {
+            function parseTimestamp(timestamp) {
+                if (!_.isString(json.timestamp)) {
+                    return moment.invalid();
+                }
+                return moment(timestamp);
+            }
+
+            var data = {};
+            try {
+                var json = JSON.parse(body);
+
+                if (json.version !== 1) {
+                    return callback(new Error('Unexpected nodes.json version: ' + json.version));
+                }
+
+                data.importTimestamp = parseTimestamp(json.timestamp);
+                if (!data.importTimestamp.isValid()) {
+                    return callback(new Error('Invalid timestamp: ' + json.timestamp));
+                }
+
+                if (!_.isPlainObject(json.nodes)) {
+                    return callback(new Error('Invalid nodes object type: ' + (typeof json.nodes)));
+                }
+
+                data.nodes = _.values(_.map(json.nodes, function (nodeData, nodeId) {
+                    if (!_.isPlainObject(nodeData)) {
+                        throw new Error(
+                            'Node ' + nodeId + ': Unexpected node type: ' + (typeof nodeData)
+                        );
+                    }
+
+                    if (!_.isPlainObject(nodeData.nodeinfo)) {
+                        throw new Error(
+                            'Node ' + nodeId + ': Unexpected nodeinfo type: ' + (typeof nodeData.nodeinfo)
+                        );
+                    }
+                    if (!_.isPlainObject(nodeData.nodeinfo.network)) {
+                        throw new Error(
+                            'Node ' + nodeId + ': Unexpected nodeinfo.network type: ' +
+                            (typeof nodeData.nodeinfo.network)
+                        );
+                    }
+
+                    if (!isValidMac(nodeData.nodeinfo.network.mac)) {
+                        throw new Error(
+                            'Node ' + nodeId + ': Invalid MAC: ' + nodeData.nodeinfo.network.mac
+                        );
+                    }
+                    var mac = Strings.normalizeMac(nodeData.nodeinfo.network.mac);
+
+                    if (!_.isPlainObject(nodeData.flags)) {
+                        throw new Error(
+                            'Node ' + nodeId + ': Unexpected flags type: ' + (typeof nodeData.flags)
+                        );
+                    }
+                    if (!_.isBoolean(nodeData.flags.online)) {
+                        throw new Error(
+                            'Node ' + nodeId + ': Unexpected flags.online type: ' + (typeof nodeData.flags.online)
+                        );
+                    }
+                    var isOnline = nodeData.flags.online;
+
+                    var lastSeen = parseTimestamp(nodeData.lastseen);
+                    if (!lastSeen.isValid()) {
+                        throw new Error(
+                            'Node ' + nodeId + ': Invalid lastseen timestamp: ' + nodeData.lastseen
+                        );
+                    }
+
+                    return {
+                        mac: mac,
+                        importTimestamp: data.importTimestamp,
+                        state: isOnline ? 'ONLINE' : 'OFFLINE',
+                        lastSeen: lastSeen
+                    };
+                }));
+            }
+            catch (error) {
+                return callback(error);
+            }
+
+            callback(null, data);
+        }
+
+        return {
         confirm: function (token, callback) {
             NodeService.getNodeDataByMonitoringToken(token, function (err, node, nodeSecrets) {
                 if (err) {
@@ -48,6 +209,44 @@ angular.module('ffffng')
                     callback(null, node);
                 });
             });
+        },
+
+        retrieveNodeInformation: function (callback) {
+            console.info();
+            request(config.server.map.nodesJsonUrl, function (err, response, body) {
+                if (err) {
+                    return callback(err);
+                }
+
+                parseNodesJson(body, function (err, data) {
+                    if (err) {
+                        return callback(err);
+                    }
+
+                    if (previousImportTimestamp !== null && !data.importTimestamp.isAfter(previousImportTimestamp)) {
+                        return callback();
+                    }
+                    previousImportTimestamp = data.importTimestamp;
+
+                    async.each(
+                        data.nodes,
+                        function (nodeData, nodeCallback) {
+                            NodeService.findNodeDataByMac(nodeData.mac, function (err, node) {
+                                if (err) {
+                                    return nodeCallback(err);
+                                }
+
+                                if (!node) {
+                                    return nodeCallback(null);
+                                }
+
+                                storeNodeInformation(nodeData, node, nodeCallback);
+                            });
+                        },
+                        callback
+                    );
+                });
+            });
         }
     };
 });

server/services/nodeService.js

@@ -224,17 +224,31 @@ angular.module('ffffng')
         callback(null, node, nodeSecrets);
     }
 
-    function getNodeDataByFilePattern(filter, callback) {
+    function findNodeDataByFilePattern(filter, callback) {
         var files = findNodeFiles(filter);
 
         if (files.length !== 1) {
-            return callback({data: 'Node not found.', type: ErrorTypes.notFound});
+            return callback(null);
         }
 
         var file = files[0];
         return parseNodeFile(file, callback);
     }
 
+    function getNodeDataByFilePattern(filter, callback) {
+        findNodeDataByFilePattern(filter, function (err, node, nodeSecrets) {
+            if (err) {
+                return callback(err);
+            }
+
+            if (!node) {
+                return callback({data: 'Node not found.', type: ErrorTypes.notFound});
+            }
+
+            callback(null, node, nodeSecrets);
+        });
+    }
+
     function sendMonitoringConfirmationMail(node, nodeSecrets, callback) {
         var confirmUrl = UrlBuilder.monitoringConfirmUrl(nodeSecrets);
         var disableUrl = UrlBuilder.monitoringDisableUrl(nodeSecrets);
@@ -351,6 +365,10 @@ angular.module('ffffng')
             deleteNodeFile(token, callback);
         },
 
+        findNodeDataByMac: function (mac, callback) {
+            return findNodeDataByFilePattern({ mac: mac }, callback);
+        },
+
         getNodeDataByToken: function (token, callback) {
             return getNodeDataByFilePattern({ token: token }, callback);
         },

server/validation/validator.js

@@ -6,6 +6,10 @@ angular.module('ffffng').factory('Validator', function (_) {
             return acceptUndefined || constraint.optional;
         }
 
+        if (!_.isString(value)) {
+            return false;
+        }
+
         var trimmed = value.trim();
         return (trimmed === '' && constraint.optional) || trimmed.match(constraint.regex);
     };