diff --git a/README.md b/README.md index c345bb3..6cbd238 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,9 @@ ffmap-backend gathers information on the batman network by invoking : * alfred-json and * batadv-vis +In order to use alfred-json and batadv-vis make sure the user running this +backend is allowed to access alfred's socket. + The output will be written to a directory (`-d output`). Run `backend.py --help` for a quick overview of all available options. @@ -15,42 +18,9 @@ Run `backend.py --help` for a quick overview of all available options. For the script's regular execution add the following to the crontab:
-* * * * * backend.py -d /path/to/output -a /path/to/aliases.json --vpn ae:7f:58:7d:6c:2a d2:d0:93:63:f7:da +* * * * * /path/to/ffmap-backend/backend.py -d /path/to/output -a /path/to/aliases.json --vpn ae:7f:58:7d:6c:2a --vpn d2:d0:93:63:f7:da-# Dependencies - -- Python 3 -- Python 3 Package [Networkx](https://networkx.github.io/) -- [alfred-json](https://github.com/tcatm/alfred-json) -- rrdtool (if run with `--with-rrd`) - -# Running as unprivileged user - -Some information collected by ffmap-backend requires access to specific system resources. - -Make sure the user you are running this under is part of the group that owns the alfred socket, so -alfred-json can access the alfred daemon. - - # ls -al /var/run/alfred.sock - srw-rw---- 1 root alfred 0 Mar 19 22:00 /var/run/alfred.sock= - # adduser map alfred - Adding user `map' to group `alfred' ... - Adding user map to group alfred - Done. - $ groups - map alfred - -Running batctl requires passwordless sudo access, because it needs to access the debugfs to retrive -the gateway list. - - # echo 'map ALL = NOPASSWD: /usr/sbin/batctl' | tee /etc/sudoers.d/map - map ALL = NOPASSWD: /usr/sbin/batctl - # chmod 0440 /etc/sudoers.d/map - -That should be everything. The script automatically detects if it is run in unprivileged mode and -will prefix `sudo` where necessary. - # Data format ## nodes.json @@ -79,21 +49,6 @@ will prefix `sudo` where necessary. - online - gateway -## Old data format - -If you want to still use the old [ffmap-d3](https://github.com/ffnord/ffmap-d3) -front end, you can use the file `ffmap-d3.jq` to convert the new output to the -old one: - -``` -jq -n -f ffmap-d3.jq \ - --argfile nodes nodedb/nodes.json \ - --argfile graph nodedb/graph.json \ - > nodedb/ffmap-d3.json -``` - -Then point your ffmap-d3 instance to the `ffmap-d3.json` file. - # Removing owner information If you'd like to redact information about the node owner from `nodes.json`, diff --git a/alfred_merge.py b/alfred_merge.py deleted file mode 100755 index ee1143f..0000000 --- a/alfred_merge.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python3 -import subprocess -import json - -from collections import MutableMapping - -def rec_merge(d1, d2): - ''' - Update two dicts of dicts recursively, - if either mapping has leaves that are non-dicts, - the second's leaf overwrites the first's. - ''' - for k, v in d1.items(): # in Python 2, use .iteritems()! - if k in d2: - # this next check is the only difference! - if all(isinstance(e, MutableMapping) for e in (v, d2[k])): - d2[k] = rec_merge(v, d2[k]) - # we could further check types and merge as appropriate here. - d3 = d1.copy() - d3.update(d2) - return d3 - - -class alfred_merge: - def __init__(self,request_data_type_1 = 158, request_data_type_2 = 159): - self.request_data_type_1 = request_data_type_1 - self.request_data_type_2 = request_data_type_2 - - def aliases(self): - output = subprocess.check_output(["/usr/local/bin/alfred-json","-z", "-r",str(self.request_data_type_1),"-f","json"]) - alfred_data_1 = json.loads(output.decode("utf-8")) - output = subprocess.check_output(["/usr/local/bin/alfred-json","-z", "-r",str(self.request_data_type_2),"-f","json"]) - alfred_data_2 = json.loads(output.decode("utf-8")) - - return json.dumps(rec_merge(alfred_data_1, alfred_data_2)) - - -if __name__ == "__main__": - ad = alfred_merge() - al = ad.aliases() - print(al) - diff --git a/aliases.json_sample b/aliases.json_sample index db35900..ca1eb6b 100644 --- a/aliases.json_sample +++ b/aliases.json_sample @@ -7,30 +7,18 @@ "latitude": 53.86 }, "network": { - "mesh": { - "bat0": { - "interfaces": { - "tunnel": [ - "00:25:86:e6:f1:bf" - ] - } - } - } + "mesh_interfaces": [ + "00:25:86:e6:f1:bf" + ] } }, { "node_id": "gw1", "hostname": "burgtor", "network": { - "mesh": { - "bat0": { - "interfaces": { - "tunnel": [ - "52:54:00:f3:62:d9" - ] - } - } - } + "mesh_interfaces": [ + "52:54:00:f3:62:d9" + ] } } ] diff --git a/backend.py b/backend.py index 8b9d9a6..4bf9058 100755 --- a/backend.py +++ b/backend.py @@ -16,8 +16,6 @@ from lib import graph, nodes from lib.alfred import Alfred from lib.batman import Batman from lib.rrddb import RRD -from lib.nodelist import export_nodelist -from lib.validate import validate_nodeinfos NODES_VERSION = 1 GRAPH_VERSION = 1 @@ -27,13 +25,7 @@ def main(params): os.makedirs(params['dest_dir'], exist_ok=True) nodes_fn = os.path.join(params['dest_dir'], 'nodes.json') - tmp_nodes_fn = os.path.join(params['dest_dir'], 'nodes.json.tmp') - graph_fn = os.path.join(params['dest_dir'], 'graph.json') - tmp_graph_fn = os.path.join(params['dest_dir'], 'graph.json.tmp') - - nodelist_fn = os.path.join(params['dest_dir'], 'nodelist.json') - tmp_nodelist_fn = os.path.join(params['dest_dir'], 'nodelist.json.tmp') now = datetime.utcnow().replace(microsecond=0) @@ -82,16 +74,14 @@ def main(params): # integrate alfred nodeinfo for alfred in alfred_instances: - nodeinfo = validate_nodeinfos(alfred.nodeinfo()) - nodes.import_nodeinfo(nodedb['nodes'], nodeinfo, + nodes.import_nodeinfo(nodedb['nodes'], alfred.nodeinfo(), now, assume_online=True) # integrate static aliases data for aliases in params['aliases']: with open(aliases, 'r') as f: -# nodeinfo = validate_nodeinfos(json.load(f)) nodes.import_nodeinfo(nodedb['nodes'], json.load(f), - now, assume_online=False, statics=True) + now, assume_online=False) nodes.reset_statistics(nodedb['nodes']) for alfred in alfred_instances: @@ -102,6 +92,7 @@ def main(params): for batman in batman_instances: vd = batman.vis_data() gwl = batman.gateway_list() + mesh_info.append((vd, gwl)) # update nodedb from batman-adv data @@ -124,41 +115,19 @@ def main(params): if params['vpn']: graph.mark_vpn(batadv_graph, frozenset(params['vpn'])) - def extract_tunnel(nodes): - macs = set() - for id, node in nodes.items(): - try: - for mac in node["nodeinfo"]["network"]["mesh"]["bat0"]["interfaces"]["tunnel"]: - macs.add(mac) - for mac in node["nodeinfo"]["network"]["mesh"]["bat-ffhh"]["interfaces"]["tunnel"]: - macs.add(mac) - except KeyError: - pass - - return macs - - graph.mark_vpn(batadv_graph, extract_tunnel(nodedb['nodes'])) - batadv_graph = graph.merge_nodes(batadv_graph) batadv_graph = graph.to_undirected(batadv_graph) # write processed data to dest dir - with open(tmp_nodes_fn, 'w') as f: + with open(nodes_fn, 'w') as f: json.dump(nodedb, f) graph_out = {'batadv': json_graph.node_link_data(batadv_graph), 'version': GRAPH_VERSION} - with open(tmp_graph_fn, 'w') as f: + with open(graph_fn, 'w') as f: json.dump(graph_out, f) - with open(tmp_nodelist_fn, 'w') as f: - json.dump(export_nodelist(now, nodedb), f) - - os.rename(tmp_nodes_fn, nodes_fn) - os.rename(tmp_graph_fn, graph_fn) - os.rename(tmp_nodelist_fn, nodelist_fn) - # optional rrd graphs (trigger with --rrd) if params['rrd']: script_directory = os.path.dirname(os.path.realpath(__file__)) @@ -167,12 +136,14 @@ def main(params): rrd.update_database(nodedb['nodes']) rrd.update_images() + if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('-a', '--aliases', help='Read aliases from FILE', - nargs='+', default=[], metavar='FILE') + default=[], action='append', + metavar='FILE') parser.add_argument('-m', '--mesh', default=['bat0'], nargs='+', help='Use given batman-adv mesh interface(s) (defaults' diff --git a/ffmap-d3.jq b/ffmap-d3.jq deleted file mode 100644 index ebeece1..0000000 --- a/ffmap-d3.jq +++ /dev/null @@ -1,52 +0,0 @@ -{ - "meta": { - "timestamp": $nodes.timestamp - }, - "nodes": ( - $graph.batadv.nodes - | map( - if has("node_id") and .node_id - then ( - $nodes.nodes[.node_id] as $node - | { - "id": .id, - "uptime": $node.statistics.uptime, - "flags": ($node.flags + {"client": false}), - "name": $node.nodeinfo.hostname, - "clientcount": (if $node.statistics.clients >= 0 then $node.statistics.clients else 0 end), - "hardware": $node.nodeinfo.hardware.model, - "firmware": $node.nodeinfo.software.firmware.release, - "geo": (if $node.nodeinfo.location then [$node.nodeinfo.location.latitude, $node.nodeinfo.location.longitude] else null end), - #"lastseen": $node.lastseen, - "network": $node.nodeinfo.network - } - ) - else - { - "flags": {}, - "id": .id, - "geo": null, - "clientcount": 0 - } - end - ) - ), - "links": ( - $graph.batadv.links - | map( - $graph.batadv.nodes[.source].node_id as $source_id - | $graph.batadv.nodes[.target].node_id as $target_id - | select( - $source_id and $target_id and - ($nodes.nodes | (has($source_id) and has($target_id))) - ) - | { - "target": .target, - "source": .source, - "quality": "\(.tq), \(.tq)", - "id": ($source_id + "-" + $target_id), - "type": (if .vpn then "vpn" else null end) - } - ) - ) -} diff --git a/gateway.json b/gateway.json deleted file mode 100644 index 4a72859..0000000 --- a/gateway.json +++ /dev/null @@ -1,29 +0,0 @@ -[ - { - "node_id": "deadbfff0101", - "hostname": "gw01" - }, - { - "node_id": "deadbeef0505", - "hostname": "gw02.hamburg.freifunk.net", - "network": { - "mac": "de:ad:be:ef:05:05", - "mesh": { - "bat0": { - "interfaces": { - "tunnel": [ - "de:ad:be:ff:05:05", - "de:ad:be:fc:05:05", - "de:ad:bf:ff:05:05" - ] - } - } - } - } - }, - { - "node_id": "00163efb9d8d", - "hostname": "gw03" - } -] - diff --git a/generate_aliases.py b/generate_aliases.py deleted file mode 100755 index 98fe407..0000000 --- a/generate_aliases.py +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env python2 - -from __future__ import print_function - -import json -import os -import sys - -if len(sys.argv) != 2: - print('usage: ' + sys.argv[0] + ' /path/to/peers') - sys.exit(1) - -peersDir = sys.argv[1] - -def normalizeMac(mac): - mac = mac.lower() - normalized = '' - - n = 0 - - for c in mac: - if c != ':': - if n > 0 and n % 2 == 0: - normalized = normalized + ':' - normalized = normalized + c - n += 1 - - return normalized - -def toAlias(peer): - alias = {} - - if not (peer.has_key('name') and peer.has_key('mac')): - return None - - name = peer['name'] - mac = peer['mac'] - - alias['node_id'] = mac.replace(':', '') - alias['hostname'] = name - - if peer.has_key('geo'): - geo = peer['geo'] - - location = {} - - if geo.has_key('lon'): location['longitude'] = geo['lon'] - if geo.has_key('lat'): location['latitude'] = geo['lat'] - - alias['location'] = location - - #alias['network'] = {} - #alias['network']['mesh_interfaces'] = [mac] - - return alias - -aliases = [] - -for filename in os.listdir(peersDir): - if len(filename) == 0 or filename[0] == '.': - continue - - isGateway = False - - absFilename = peersDir + '/' + filename - if os.path.isfile(absFilename): - peerFile = open(absFilename, 'r') - try: - peerLines = peerFile.readlines() - peer = {} - - for line in peerLines: - parts = line.split() - - if len(parts) > 2: - if parts[1] == 'Knotenname:': - peer['name'] = parts[2] - - elif parts[0] == 'remote': - isGateway = True - - elif parts[1] == 'MAC:': - peer['mac'] = normalizeMac(parts[2]) - - elif parts[1] == 'Koordinaten:' and len(parts) > 3: - try: - peer['geo'] = {'lat': float(parts[2]), 'lon': float(parts[3])} - - except ValueError: - print('Error in %s: Invalid coordinates: %s' % (absFilename, parts[2:4]), file = sys.stderr) - - elif len(parts) == 2 and parts[0] == 'key': - keyParts = parts[1].split('"') - if len(keyParts) > 1: - peer['vpn'] = keyParts[1].lower() - - if isGateway: - continue - - alias = toAlias(peer) - if alias: - aliases.append(alias) - - except Exception as e: - print('Error in %s, ignoring peer: %s' % (absFilename, e), file = sys.stderr) - - finally: - peerFile.close() - -print(json.dumps(aliases)) diff --git a/generate_aliases_v2.py b/generate_aliases_v2.py deleted file mode 100755 index 7a04c7c..0000000 --- a/generate_aliases_v2.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python2 - -from __future__ import print_function - -import json -import os -import sys - -if len(sys.argv) != 2: - print('usage: ' + sys.argv[0] + ' /path/to/peers') - sys.exit(1) - -peersDir = sys.argv[1] - -def normalizeMac(mac): - mac = mac.lower() - normalized = '' - - n = 0 - - for c in mac: - if c != ':': - if n > 0 and n % 2 == 0: - normalized = normalized + ':' - normalized = normalized + c - n += 1 - - return normalized - -def toAlias(peer): - alias = {} - - if not (peer.has_key('name') and peer.has_key('mac')): - return None - - name = peer['name'] - mac = peer['mac'] - - alias['node_id'] = mac.replace(':', '') - alias['hostname'] = name - - if peer.has_key('geo'): - geo = peer['geo'] - - location = {} - - if geo.has_key('lon'): location['longitude'] = geo['lon'] - if geo.has_key('lat'): location['latitude'] = geo['lat'] - - alias['location'] = location - - #alias['network'] = {} - #alias['network']['mesh_interfaces'] = [mac] - - return {'nodeinfo':alias} - -aliases = {} - -for filename in os.listdir(peersDir): - if len(filename) == 0 or filename[0] == '.': - continue - - isGateway = False - - absFilename = peersDir + '/' + filename - if os.path.isfile(absFilename): - peerFile = open(absFilename, 'r') - try: - peerLines = peerFile.readlines() - peer = {} - - for line in peerLines: - parts = line.split() - - if len(parts) > 2: - if parts[1] == 'Knotenname:': - peer['name'] = parts[2] - - elif parts[0] == 'remote': - isGateway = True - - elif parts[1] == 'MAC:': - peer['mac'] = normalizeMac(parts[2]) - - elif parts[1] == 'Koordinaten:' and len(parts) > 3: - try: - peer['geo'] = {'lat': float(parts[2]), 'lon': float(parts[3])} - - except ValueError: - print('Error in %s: Invalid coordinates: %s' % (absFilename, parts[2:4]), file = sys.stderr) - - elif len(parts) == 2 and parts[0] == 'key': - keyParts = parts[1].split('"') - if len(keyParts) > 1: - peer['vpn'] = keyParts[1].lower() - - if isGateway: - continue - - alias = toAlias(peer) - if alias: - tmpid = alias['nodeinfo']['node_id'] -# alias['nodeinfo'].pop('node_id') - aliases[tmpid] = alias - - except Exception as e: - print('Error in %s, ignoring peer: %s' % (absFilename, e), file = sys.stderr) - - finally: - peerFile.close() - -print(json.dumps(aliases)) diff --git a/lib/alfred.py b/lib/alfred.py index 531eaea..4353874 100644 --- a/lib/alfred.py +++ b/lib/alfred.py @@ -13,7 +13,7 @@ class Alfred(object): raise RuntimeError('alfred: invalid unix socket path given') def _fetch(self, data_type): - cmd = ['/usr/local/bin/alfred-json', + cmd = ['alfred-json', '-z', '-f', 'json', '-r', str(data_type)] diff --git a/lib/batman.py b/lib/batman.py index ee0198b..5c48740 100644 --- a/lib/batman.py +++ b/lib/batman.py @@ -1,6 +1,5 @@ import subprocess import json -import os import re @@ -13,14 +12,6 @@ class Batman(object): self.mesh_interface = mesh_interface self.alfred_sock = alfred_sockpath - # ensure /usr/sbin and /usr/local/sbin are in PATH - env = os.environ - path = set(env['PATH'].split(':')) - path.add('/usr/sbin/') - path.add('/usr/local/sbin') - env['PATH'] = ':'.join(path) - self.environ = env - # compile regular expressions only once on startup self.mac_addr_pattern = re.compile(r'(([a-z0-9]{2}:){5}[a-z0-9]{2})') @@ -46,7 +37,7 @@ class Batman(object): cmd = ['batadv-vis', '-i', self.mesh_interface, '-f', 'json'] if self.alfred_sock: cmd.extend(['-u', self.alfred_sock]) - output = subprocess.check_output(cmd, env=self.environ) + output = subprocess.check_output(cmd) lines = output.splitlines() return self.vis_data_helper(lines) @@ -55,10 +46,8 @@ class Batman(object): Parse "batctl -m