NodeRRD: add many more DS and reduce retention duration

alfred_merge.py

@@ -1,42 +0,0 @@
-#!/usr/bin/env python3
-import subprocess
-import json
-
-from collections import MutableMapping
-
-def rec_merge(d1, d2):
-    '''
-    Update two dicts of dicts recursively, 
-    if either mapping has leaves that are non-dicts, 
-    the second's leaf overwrites the first's.
-    '''
-    for k, v in d1.items(): # in Python 2, use .iteritems()!
-        if k in d2:
-            # this next check is the only difference!
-            if all(isinstance(e, MutableMapping) for e in (v, d2[k])):
-                d2[k] = rec_merge(v, d2[k])
-            # we could further check types and merge as appropriate here.
-    d3 = d1.copy()
-    d3.update(d2)
-    return d3
-
-
-class alfred_merge:
-  def __init__(self,request_data_type_1 = 158, request_data_type_2 = 159):
-    self.request_data_type_1 = request_data_type_1
-    self.request_data_type_2 = request_data_type_2
-
-  def aliases(self):
-    output = subprocess.check_output(["/usr/local/bin/alfred-json","-z", "-r",str(self.request_data_type_1),"-f","json"])
-    alfred_data_1 = json.loads(output.decode("utf-8"))
-    output = subprocess.check_output(["/usr/local/bin/alfred-json","-z", "-r",str(self.request_data_type_2),"-f","json"])
-    alfred_data_2 = json.loads(output.decode("utf-8"))
-    
-    return json.dumps(rec_merge(alfred_data_1, alfred_data_2))
-
-
-if __name__ == "__main__":
-  ad = alfred_merge()
-  al = ad.aliases()
-  print(al)
-

backend.py

@@ -27,13 +27,8 @@ def main(params):
     os.makedirs(params['dest_dir'], exist_ok=True)
 
     nodes_fn = os.path.join(params['dest_dir'], 'nodes.json')
-    tmp_nodes_fn = os.path.join(params['dest_dir'], 'nodes.json.tmp')
-
     graph_fn = os.path.join(params['dest_dir'], 'graph.json')
-    tmp_graph_fn = os.path.join(params['dest_dir'], 'graph.json.tmp')
-
     nodelist_fn = os.path.join(params['dest_dir'], 'nodelist.json')
-    tmp_nodelist_fn = os.path.join(params['dest_dir'], 'nodelist.json.tmp')
 
     now = datetime.utcnow().replace(microsecond=0)
 
@@ -89,9 +84,9 @@ def main(params):
     # integrate static aliases data
     for aliases in params['aliases']:
         with open(aliases, 'r') as f:
-#           nodeinfo = validate_nodeinfos(json.load(f))
+            nodeinfo = validate_nodeinfos(json.load(f))
-            nodes.import_nodeinfo(nodedb['nodes'], json.load(f),
+            nodes.import_nodeinfo(nodedb['nodes'], nodeinfo,
-                                  now, assume_online=False, statics=True)
+                                  now, assume_online=False)
 
     nodes.reset_statistics(nodedb['nodes'])
     for alfred in alfred_instances:
@@ -102,6 +97,7 @@ def main(params):
     for batman in batman_instances:
         vd = batman.vis_data()
         gwl = batman.gateway_list()
+
         mesh_info.append((vd, gwl))
 
     # update nodedb from batman-adv data
@@ -130,8 +126,6 @@ def main(params):
             try:
                 for mac in node["nodeinfo"]["network"]["mesh"]["bat0"]["interfaces"]["tunnel"]:
                     macs.add(mac)
-                for mac in node["nodeinfo"]["network"]["mesh"]["bat-ffhh"]["interfaces"]["tunnel"]:
-                    macs.add(mac)
             except KeyError:
                 pass
 
@@ -143,30 +137,27 @@ def main(params):
     batadv_graph = graph.to_undirected(batadv_graph)
 
     # write processed data to dest dir
-    with open(tmp_nodes_fn, 'w') as f:
+    with open(nodes_fn, 'w') as f:
         json.dump(nodedb, f)
 
     graph_out = {'batadv': json_graph.node_link_data(batadv_graph),
                  'version': GRAPH_VERSION}
 
-    with open(tmp_graph_fn, 'w') as f:
+    with open(graph_fn, 'w') as f:
         json.dump(graph_out, f)
 
-    with open(tmp_nodelist_fn, 'w') as f:
+    with open(nodelist_fn, 'w') as f:
         json.dump(export_nodelist(now, nodedb), f)
 
-    os.rename(tmp_nodes_fn, nodes_fn)
-    os.rename(tmp_graph_fn, graph_fn)
-    os.rename(tmp_nodelist_fn, nodelist_fn)
-
     # optional rrd graphs (trigger with --rrd)
     if params['rrd']:
         script_directory = os.path.dirname(os.path.realpath(__file__))
         rrd = RRD(os.path.join(script_directory, 'nodedb'),
                   os.path.join(params['dest_dir'], 'nodes'))
-        rrd.update_database(nodedb['nodes'])
+        rrd.update_database(nodedb['nodes'], batadv_graph)
         rrd.update_images()
 
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
 

gateway.json

@@ -1,29 +0,0 @@
-[
-  {
-    "node_id": "deadbfff0101",
-    "hostname": "gw01"
-  },
-  {
-    "node_id": "deadbeef0505",
-    "hostname": "gw02.hamburg.freifunk.net",
-    "network": {
-        "mac": "de:ad:be:ef:05:05",
-        "mesh": {
-            "bat0": {
-                "interfaces": {
-                    "tunnel": [
-                        "de:ad:be:ff:05:05",
-                        "de:ad:be:fc:05:05",
-                        "de:ad:bf:ff:05:05"
-                    ]
-                }
-            }
-        }
-    }
-  },
-  {
-    "node_id": "00163efb9d8d",
-    "hostname": "gw03"
-  }
-]
-

generate_aliases.py

@@ -1,110 +0,0 @@
-#!/usr/bin/env python2
-
-from __future__ import print_function
-
-import json
-import os
-import sys
-
-if len(sys.argv) != 2:
-    print('usage: ' + sys.argv[0] + ' /path/to/peers')
-    sys.exit(1)
-
-peersDir = sys.argv[1]
-
-def normalizeMac(mac):
-    mac = mac.lower()
-    normalized = ''
-
-    n = 0
-
-    for c in mac:
-        if c != ':':
-            if n > 0 and n % 2 == 0:
-                normalized = normalized + ':'
-            normalized = normalized + c
-            n += 1
-
-    return normalized
-
-def toAlias(peer):
-    alias = {}
-
-    if not (peer.has_key('name') and peer.has_key('mac')):
-        return None
-
-    name = peer['name']
-    mac = peer['mac']
-
-    alias['node_id'] = mac.replace(':', '')
-    alias['hostname'] = name
-
-    if peer.has_key('geo'):
-        geo = peer['geo']
-
-        location = {}
-
-        if geo.has_key('lon'): location['longitude'] = geo['lon']
-        if geo.has_key('lat'): location['latitude'] = geo['lat']
-
-        alias['location'] = location
-
-    #alias['network'] = {}
-    #alias['network']['mesh_interfaces'] = [mac]
-
-    return alias
-
-aliases = []
-
-for filename in os.listdir(peersDir):
-    if len(filename) == 0 or filename[0] == '.':
-        continue
-
-    isGateway = False
-
-    absFilename = peersDir + '/' + filename
-    if os.path.isfile(absFilename):
-        peerFile = open(absFilename, 'r')
-        try:
-            peerLines = peerFile.readlines()
-            peer = {}
-
-            for line in peerLines:
-                parts = line.split()
-
-                if len(parts) > 2:
-                    if parts[1] == 'Knotenname:':
-                        peer['name'] = parts[2]
-
-                    elif parts[0] == 'remote':
-                        isGateway = True
-
-                    elif parts[1] == 'MAC:':
-                        peer['mac'] = normalizeMac(parts[2])
-
-                    elif parts[1] == 'Koordinaten:' and len(parts) > 3:
-                        try:
-                            peer['geo'] = {'lat': float(parts[2]), 'lon': float(parts[3])}
-
-                        except ValueError:
-                            print('Error in %s: Invalid coordinates: %s' % (absFilename, parts[2:4]), file = sys.stderr)
-
-                elif len(parts) == 2 and parts[0] == 'key':
-                    keyParts = parts[1].split('"')
-                    if len(keyParts) > 1:
-                        peer['vpn'] = keyParts[1].lower()
-
-            if isGateway:
-                continue
-
-            alias = toAlias(peer)
-            if alias:
-                aliases.append(alias)
-
-        except Exception as e:
-            print('Error in %s, ignoring peer: %s' % (absFilename, e), file = sys.stderr)
-
-        finally:
-            peerFile.close()
-
-print(json.dumps(aliases))

generate_aliases_v2.py

@@ -1,112 +0,0 @@
-#!/usr/bin/env python2
-
-from __future__ import print_function
-
-import json
-import os
-import sys
-
-if len(sys.argv) != 2:
-    print('usage: ' + sys.argv[0] + ' /path/to/peers')
-    sys.exit(1)
-
-peersDir = sys.argv[1]
-
-def normalizeMac(mac):
-    mac = mac.lower()
-    normalized = ''
-
-    n = 0
-
-    for c in mac:
-        if c != ':':
-            if n > 0 and n % 2 == 0:
-                normalized = normalized + ':'
-            normalized = normalized + c
-            n += 1
-
-    return normalized
-
-def toAlias(peer):
-    alias = {}
-
-    if not (peer.has_key('name') and peer.has_key('mac')):
-        return None
-
-    name = peer['name']
-    mac = peer['mac']
-
-    alias['node_id'] = mac.replace(':', '')
-    alias['hostname'] = name
-
-    if peer.has_key('geo'):
-        geo = peer['geo']
-
-        location = {}
-
-        if geo.has_key('lon'): location['longitude'] = geo['lon']
-        if geo.has_key('lat'): location['latitude'] = geo['lat']
-
-        alias['location'] = location
-
-    #alias['network'] = {}
-    #alias['network']['mesh_interfaces'] = [mac]
-
-    return {'nodeinfo':alias}
-
-aliases = {}
-
-for filename in os.listdir(peersDir):
-    if len(filename) == 0 or filename[0] == '.':
-        continue
-
-    isGateway = False
-
-    absFilename = peersDir + '/' + filename
-    if os.path.isfile(absFilename):
-        peerFile = open(absFilename, 'r')
-        try:
-            peerLines = peerFile.readlines()
-            peer = {}
-
-            for line in peerLines:
-                parts = line.split()
-
-                if len(parts) > 2:
-                    if parts[1] == 'Knotenname:':
-                        peer['name'] = parts[2]
-
-                    elif parts[0] == 'remote':
-                        isGateway = True
-
-                    elif parts[1] == 'MAC:':
-                        peer['mac'] = normalizeMac(parts[2])
-
-                    elif parts[1] == 'Koordinaten:' and len(parts) > 3:
-                        try:
-                            peer['geo'] = {'lat': float(parts[2]), 'lon': float(parts[3])}
-
-                        except ValueError:
-                            print('Error in %s: Invalid coordinates: %s' % (absFilename, parts[2:4]), file = sys.stderr)
-
-                elif len(parts) == 2 and parts[0] == 'key':
-                    keyParts = parts[1].split('"')
-                    if len(keyParts) > 1:
-                        peer['vpn'] = keyParts[1].lower()
-
-            if isGateway:
-                continue
-
-            alias = toAlias(peer)
-            if alias:
-                tmpid = alias['nodeinfo']['node_id']
-#                alias['nodeinfo'].pop('node_id')
-                aliases[tmpid] = alias
-
-        except Exception as e:
-            print('Error in %s, ignoring peer: %s' % (absFilename, e), file = sys.stderr)
-
-        finally:
-            peerFile.close()
-
-print(json.dumps(aliases))

lib/NodeRRD.py

@@ -8,25 +8,35 @@ class NodeRRD(RRD):
     ds_list = [
         DS('upstate', 'GAUGE', 120, 0, 1),
         DS('clients', 'GAUGE', 120, 0, float('NaN')),
+        DS('neighbors', 'GAUGE', 120, 0, float('NaN')),
+        DS('vpn_neighbors', 'GAUGE', 120, 0, float('NaN')),
+        DS('loadavg', 'GAUGE', 120, 0, float('NaN')),
+        DS('rx_bytes', 'DERIVE', 120, 0, float('NaN')),
+        DS('rx_packets', 'DERIVE', 120, 0, float('NaN')),
+        DS('tx_bytes', 'DERIVE', 120, 0, float('NaN')),
+        DS('tx_packets', 'DERIVE', 120, 0, float('NaN')),
+        DS('mgmt_rx_bytes', 'DERIVE', 120, 0, float('NaN')),
+        DS('mgmt_rx_packets', 'DERIVE', 120, 0, float('NaN')),
+        DS('mgmt_tx_bytes', 'DERIVE', 120, 0, float('NaN')),
+        DS('mgmt_tx_packets', 'DERIVE', 120, 0, float('NaN')),
+        DS('forward_bytes', 'DERIVE', 120, 0, float('NaN')),
+        DS('forward_packets', 'DERIVE', 120, 0, float('NaN')),
     ]
     rra_list = [
         # 2 hours of  1 minute samples
         RRA('AVERAGE', 0.5, 1, 120),
-        #  5 days  of  5 minute samples
+        # 7 days  of  15 minute samples
-        RRA('AVERAGE', 0.5, 5, 1440),
+        RRA('AVERAGE', 0.5, 15, 672),
-        # 30 days  of  1 hour   samples
-        RRA('AVERAGE', 0.5, 60, 720),
-        #  1 year  of 12 hour   samples
-        RRA('AVERAGE', 0.5, 720, 730),
     ]
 
-    def __init__(self, filename, node=None):
+    def __init__(self, filename, node=None, graph=None):
         """
         Create a new RRD for a given node.
 
         If the RRD isn't supposed to be updated, the node can be omitted.
         """
         self.node = node
+        self.node_graph = graph
         super().__init__(filename)
         self.ensure_sanity(self.ds_list, self.rra_list, step=60)
 
@@ -37,8 +47,28 @@ class NodeRRD(RRD):
 
     # TODO: fix this, python does not support function overloading
     def update(self):
-        super().update({'upstate': int(self.node['flags']['online']),
+        values = {
-                        'clients': self.node['statistics']['clients']})
+            'upstate': int(self.node['flags']['online']),
+            'clients': float(self.node['statistics']['clients']),
+            'loadavg': float(self.node['statistics'].get('loadavg', 0)),
+        }
+        for item in ('rx', 'tx', 'mgmt_rx', 'mgmt_tx', 'forward'):
+            try:
+                values.update({
+                    ('%s_bytes' % item): int(self.node['statistics'].get('traffic', {}).get(item, {}).get('bytes', 0)),
+                    ('%s_packets' % item): int(self.node['statistics'].get('traffic', {}).get(item, {}).get('packets', 0)),
+                })
+            except TypeError:
+                pass
+        try:
+            graph_node = next(key for key, node in self.node_graph.nodes(data=True) if node.get('node_id') == self.node['nodeinfo']['node_id'])
+            values.update({
+                'neighbors': float(len(self.node_graph[graph_node])),
+                'vpn_neighbors': float(len(list(filter(lambda edge: edge.get('vpn', False), self.node_graph[graph_node].values())))),
+            })
+        except StopIteration:
+            pass
+        super().update(values)
 
     def graph(self, directory, timeframe):
         """

lib/alfred.py

@@ -13,7 +13,7 @@ class Alfred(object):
             raise RuntimeError('alfred: invalid unix socket path given')
 
     def _fetch(self, data_type):
-        cmd = ['/usr/local/bin/alfred-json',
+        cmd = ['alfred-json',
                '-z',
                '-f', 'json',
                '-r', str(data_type)]

lib/batman.py

@@ -96,3 +96,7 @@ if __name__ == "__main__":
     bc = Batman()
     vd = bc.vis_data()
     gw = bc.gateway_list()
+    for x in vd:
+        print(x)
+    print(gw)
+    print(bc.gateway_mode())

lib/graph.py

@@ -25,6 +25,7 @@ def mark_vpn(graph, vpn_macs):
     components = map(frozenset, nx.weakly_connected_components(graph))
     components = filter(vpn_macs.intersection, components)
     nodes = reduce(lambda a, b: a | b, components, set())
+
     for node in nodes:
         for k, v in graph[node].items():
             v['vpn'] = True

lib/nodelist.py

@@ -13,9 +13,6 @@ def export_nodelist(now, nodedb):
         node_out["status"] = dict()
         node_out["status"]["online"] = node["flags"]["online"]
 
-        if "firstseen" in node:
-            node_out["status"]["firstcontact"] = node["firstseen"]
-
         if "lastseen" in node:
             node_out["status"]["lastcontact"] = node["lastseen"]
 

lib/nodes.py

@@ -6,10 +6,6 @@ from functools import reduce
 def build_mac_table(nodes):
     macs = dict()
     for node_id, node in nodes.items():
-        try:
-            macs[node['network']['mac']] = node_id
-        except KeyError:
-            pass
         try:
             for mac in node['nodeinfo']['network']['mesh_interfaces']:
                 macs[mac] = node_id
@@ -27,11 +23,6 @@ def build_mac_table(nodes):
                 macs[mac] = node_id
         except KeyError:
             pass
-        try:
-            for mac in node['nodeinfo']['network']['mesh']['bat-ffhh']['interfaces']['tunnel']:
-                macs[mac] = node_id
-        except KeyError:
-            pass
 
         try:
             for mac in node['nodeinfo']['network']['mesh']['bat0']['interfaces']['other']:
@@ -65,23 +56,12 @@ def mark_online(node, now):
     node['flags']['online'] = True
 
 
-def overrideFields(dest, src, fields):
+def import_nodeinfo(nodes, nodeinfos, now, assume_online=False):
-    for field in fields:
-        if field in src:
-            dest[field] = src[field]
-        else:
-            dest.pop(field, None)
-
-
-def import_nodeinfo(nodes, nodeinfos, now, assume_online=False, statics=False):
     for nodeinfo in filter(lambda d: 'node_id' in d, nodeinfos):
-        node = nodes.setdefault(nodeinfo['node_id'], {'flags': {'online': False, 'gateway': False}})
+        node = nodes.setdefault(nodeinfo['node_id'], {'flags': dict()})
-
+        node['nodeinfo'] = nodeinfo
-        if statics:
+        node['flags']['online'] = False
-            node['nodeinfo'] = node.setdefault('nodeinfo', {})
+        node['flags']['gateway'] = False
-            overrideFields(node['nodeinfo'], nodeinfo, ['hostname', 'location', 'node_id'])
-        else:
-            node['nodeinfo'] = nodeinfo
 
         if assume_online:
             mark_online(node, now)
@@ -154,11 +134,6 @@ def import_mesh_ifs_vis_data(nodes, vis_data):
         except KeyError:
             pass
 
-        try:
-            ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat-ffhh']['interfaces']['tunnel']))
-        except KeyError:
-            pass
-
         try:
             ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat0']['interfaces']['other']))
         except KeyError:
@@ -180,6 +155,7 @@ def import_vis_clientcount(nodes, vis_data):
 def mark_gateways(nodes, gateways):
     macs = build_mac_table(nodes)
     gateways = filter(lambda d: d in macs, gateways)
+
     for node in map(lambda d: nodes[macs[d]], gateways):
         node['flags']['gateway'] = True
 

lib/rrddb.py

@@ -27,7 +27,7 @@ class RRD(object):
         except OSError:
             os.mkdir(self.imagePath)
 
-    def update_database(self, nodes):
+    def update_database(self, nodes, graph):
         online_nodes = dict(filter(
             lambda d: d[1]['flags']['online'], nodes.items()))
         client_count = sum(map(
@@ -35,7 +35,7 @@ class RRD(object):
 
         self.globalDb.update(len(online_nodes), client_count)
         for node_id, node in online_nodes.items():
-            rrd = NodeRRD(os.path.join(self.dbPath, node_id + '.rrd'), node)
+            rrd = NodeRRD(os.path.join(self.dbPath, node_id + '.rrd'), node, graph)
             rrd.update()
 
     def update_images(self):

mkmap.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-FFMAPPATH='/opt/ffmap-backend/'
-PEERS="/etc/fastd/ffhh-mesh-vpn/peers"
-
-python2 $FFMAPPATH/generate_aliases.py $PEERS > $FFMAPPATH/aliases.json
-#python3 $FFMAPPATH/backend.py -d /var/www/meshviewer/ --aliases $FFMAPPATH/aliases.json $FFMAPPATH/gateway.json -m bat0:/var/run/alfred.sock -p 30 --vpn de:ad:be:ff:01:01 --vpn de:ad:be:ff:05:05 --vpn de:ad:be:ff:05:06 --vpn de:ad:be:ff:03:03 --vpn de:ad:be:ff:22:22 --vpn de:ad:be:ff:22:23 --vpn de:ad:be:ff:88:88 --vpn de:ad:be:ff:88:89 --vpn de:ad:bf:ff:88:88 --vpn de:ad:bf:ff:22:22 --vpn de:ad:bf:ff:03:03 --vpn de:ad:bf:ff:05:05 --vpn de:ad:bf:ff:01:01 --vpn de:ad:be:fc:03:03 --vpn 00:16:3e:53:75:0d --vpn de:ad:be:fc:05:05 --vpn de:ad:be:fc:01:01 --vpn de:ad:be:ef:03:03 --vpn de:ad:be:ef:01:01 --vpn de:ad:be:ef:05:05 --vpn 00:16:3e:fb:9d:8d --vpn 00:16:3e:fb:9d:9d
-python3 $FFMAPPATH/backend.py -d /var/www/meshviewer/ --aliases $FFMAPPATH/aliases.json $FFMAPPATH/gateway.json -m bat0:/var/run/alfred.sock -p 30 --vpn de:ad:be:ff:01:01 de:ad:be:ff:05:05 de:ad:be:ff:05:06 de:ad:be:ff:03:03 de:ad:be:ff:22:22 de:ad:be:ff:22:23 de:ad:be:ff:88:88 de:ad:be:ff:88:89 de:ad:bf:ff:88:88 de:ad:bf:ff:22:22 de:ad:bf:ff:03:03 de:ad:bf:ff:05:05 de:ad:bf:ff:01:01 de:ad:be:fc:03:03 00:16:3e:53:75:0d de:ad:be:fc:05:05 de:ad:be:fc:01:01 de:ad:be:ef:03:03 de:ad:be:ef:01:01 de:ad:be:ef:05:05 00:16:3e:fb:9d:8d 00:16:3e:fb:9d:9d

node_number.py

@@ -1,32 +0,0 @@
-#!/usr/bin/env python
-#Bibliotheken importieren
-import time
-import datetime
-import json
-import urllib2
-
-#Datei oeffnen
-Datei = urllib2.urlopen('https://map.hamburg.freifunk.net/nodes.json')
-Datei_Sued = urllib2.urlopen('https://map.hamburg.freifunk.net/hhsued/mv1/nodes.json')
-
-Text = Datei.read()
-Knotenzahl = Text.count('"online": true')
-Text = Datei_Sued.read()
-Knotenzahl = Knotenzahl + Text.count('"online":true')
-
-#Zeit holen
-thetime = datetime.datetime.now().isoformat()
-
-ffhh = None
-
-#Freifunk API-Datei einladen und JSON lesen
-with open('/var/www/meta/ffhh.json', 'r') as fp:
-        ffhh = json.load(fp)
-
-#Attribute Zeitstempel und Knotenanzahl setzen
-ffhh['state']['lastchange'] = thetime
-ffhh['state']['nodes'] = Knotenzahl
-
-#Freifunk API-Datein mit geaenderten werten schreiben
-with open('/var/www/meta/ffhh.json', 'w') as fp:
-        json.dump(ffhh, fp, indent=2, separators=(',', ': '))