diff --git a/.gitignore b/.gitignore
index e7425cf..0d20b64 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,8 +1 @@
-# script-generated
-aliases*.json
-nodedb/
-
-# python bytecode / cache
*.pyc
-pycache/
-__pycache__/
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index ee88058..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,6 +0,0 @@
-sudo: false
-language: python
-python:
- - "3.4"
-install: "pip install pep8"
-script: "pep8 --ignore=E501 *.py lib/*.py"
diff --git a/README.md b/README.md
index c345bb3..8e4abd8 100644
--- a/README.md
+++ b/README.md
@@ -1,117 +1,53 @@
# Data for Freifunk Map, Graph and Node List
-[](https://travis-ci.org/ffnord/ffmap-backend)
+ffmap-backend gathers information on the batman network by invoking
+ batctl
+and
+ batadv-vis
+as root (via sudo) and has this information placed into a target directory
+as the file "nodes.json" and also updates the directory "nodes" with graphical
+representations of uptimes and the number of clients connecting.
-ffmap-backend gathers information on the batman network by invoking :
-
- * batctl (might require root),
- * alfred-json and
- * batadv-vis
-
-The output will be written to a directory (`-d output`).
-
-Run `backend.py --help` for a quick overview of all available options.
-
-For the script's regular execution add the following to the crontab:
+The target directory is suggested to host all information for interpreting those
+node descriptions, e.g. as provided by https://github.com/ffnord/ffmap-d3.git .
+When executed without root privileges, we suggest to grant sudo permissions
+within wrappers of those binaries, so no further changes are required in other
+scripts:
-* * * * * backend.py -d /path/to/output -a /path/to/aliases.json --vpn ae:7f:58:7d:6c:2a d2:d0:93:63:f7:da
+$ cat < $HOME/batctl
+#!/bin/sh
+exec sudo /usr/sbin/batctl $*
+EOCAT
-# Dependencies
+and analogously for batadv-vis. The entry for /etc/sudoers could be
+whateveruser ALL=(ALL:ALL) NOPASSWD: /usr/sbin/batctl,/usr/sbin/batadv-vis,/usr/sbin/alfred-json
-- Python 3
-- Python 3 Package [Networkx](https://networkx.github.io/)
-- [alfred-json](https://github.com/tcatm/alfred-json)
-- rrdtool (if run with `--with-rrd`)
+The destination directory can be made directly available through apache:
+
+$ cat /etc/apache2/site-enabled/000-default
+...
+
+ Options Indexes FollowSymLinks MultiViews
+ AllowOverride None
+ Order allow,deny
+ allow from all
+
+...
-# Running as unprivileged user
+$ cat /etc/apache2/conf.d/freifunk
+Alias /map /home/ffmap/www/
+Alias /firmware /home/freifunk/autoupdates/
+
-Some information collected by ffmap-backend requires access to specific system resources.
+To execute, run
+ python3 -mffmap.run --input-alfred --input-badadv --output-d3json ../www/nodes.json
+The script expects above described sudo-wrappers in the $HOME directory of the user executing
+the script. If those are not available, an error will occurr if not executed as root. Also,
+the tool realpath optionally allows to execute the script from anywhere in the directory tree.
-Make sure the user you are running this under is part of the group that owns the alfred socket, so
-alfred-json can access the alfred daemon.
-
- # ls -al /var/run/alfred.sock
- srw-rw---- 1 root alfred 0 Mar 19 22:00 /var/run/alfred.sock=
- # adduser map alfred
- Adding user `map' to group `alfred' ...
- Adding user map to group alfred
- Done.
- $ groups
- map alfred
-
-Running batctl requires passwordless sudo access, because it needs to access the debugfs to retrive
-the gateway list.
-
- # echo 'map ALL = NOPASSWD: /usr/sbin/batctl' | tee /etc/sudoers.d/map
- map ALL = NOPASSWD: /usr/sbin/batctl
- # chmod 0440 /etc/sudoers.d/map
-
-That should be everything. The script automatically detects if it is run in unprivileged mode and
-will prefix `sudo` where necessary.
-
-# Data format
-
-## nodes.json
-
- { 'nodes': {
- node_id: { 'flags': { flags },
- 'firstseen': isoformat,
- 'lastseen': isoformat,
- 'nodeinfo': {...}, # copied from alfred type 158
- 'statistics': {
- 'uptime': double, # seconds
- 'memory_usage': double, # 0..1
- 'clients': double,
- 'rootfs_usage': double, # 0..1
- 'loadavg': double,
- 'gateway': mac
- }
- },
- ...
- }
- 'timestamp': isoformat
- }
-
-### flags (bool)
-
-- online
-- gateway
-
-## Old data format
-
-If you want to still use the old [ffmap-d3](https://github.com/ffnord/ffmap-d3)
-front end, you can use the file `ffmap-d3.jq` to convert the new output to the
-old one:
-
-```
-jq -n -f ffmap-d3.jq \
- --argfile nodes nodedb/nodes.json \
- --argfile graph nodedb/graph.json \
- > nodedb/ffmap-d3.json
-```
-
-Then point your ffmap-d3 instance to the `ffmap-d3.json` file.
-
-# Removing owner information
-
-If you'd like to redact information about the node owner from `nodes.json`,
-you may use a filter like [jq]. In this case, specify an output directory
-different from your webserver directory, e.g.:
-
- ./backend.py -d /ffmap-data
-
-Don't write to files generated in there. ffmap-backend uses them as its
-database.
-
-After running ffmap-backend, copy `graph.json` to your webserver. Then,
-filter `nodes.json` using `jq` like this:
-
- jq '.nodes = (.nodes | with_entries(del(.value.nodeinfo.owner)))' \
- < /ffmap-data/nodes.json > /var/www/data/nodes.json
-
-This will remove owner information from nodes.json before copying the data
-to your webserver.
-
-[jq]: https://stedolan.github.io/jq/
+For the script's regular execution add the following to the crontab:
+
+*/5 * * * * python3 -mffmap.run --input-alfred --input-badadv --output-d3json /home/ffmap/www/nodes.json
+
diff --git a/alfred_merge.py b/alfred_merge.py
deleted file mode 100755
index ee1143f..0000000
--- a/alfred_merge.py
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env python3
-import subprocess
-import json
-
-from collections import MutableMapping
-
-def rec_merge(d1, d2):
- '''
- Update two dicts of dicts recursively,
- if either mapping has leaves that are non-dicts,
- the second's leaf overwrites the first's.
- '''
- for k, v in d1.items(): # in Python 2, use .iteritems()!
- if k in d2:
- # this next check is the only difference!
- if all(isinstance(e, MutableMapping) for e in (v, d2[k])):
- d2[k] = rec_merge(v, d2[k])
- # we could further check types and merge as appropriate here.
- d3 = d1.copy()
- d3.update(d2)
- return d3
-
-
-class alfred_merge:
- def __init__(self,request_data_type_1 = 158, request_data_type_2 = 159):
- self.request_data_type_1 = request_data_type_1
- self.request_data_type_2 = request_data_type_2
-
- def aliases(self):
- output = subprocess.check_output(["/usr/local/bin/alfred-json","-z", "-r",str(self.request_data_type_1),"-f","json"])
- alfred_data_1 = json.loads(output.decode("utf-8"))
- output = subprocess.check_output(["/usr/local/bin/alfred-json","-z", "-r",str(self.request_data_type_2),"-f","json"])
- alfred_data_2 = json.loads(output.decode("utf-8"))
-
- return json.dumps(rec_merge(alfred_data_1, alfred_data_2))
-
-
-if __name__ == "__main__":
- ad = alfred_merge()
- al = ad.aliases()
- print(al)
-
diff --git a/aliases.json_sample b/aliases.json_sample
deleted file mode 100644
index db35900..0000000
--- a/aliases.json_sample
+++ /dev/null
@@ -1,36 +0,0 @@
-[
- {
- "node_id": "krtek",
- "hostname": "krtek",
- "location": {
- "longitude": 10.74,
- "latitude": 53.86
- },
- "network": {
- "mesh": {
- "bat0": {
- "interfaces": {
- "tunnel": [
- "00:25:86:e6:f1:bf"
- ]
- }
- }
- }
- }
- },
- {
- "node_id": "gw1",
- "hostname": "burgtor",
- "network": {
- "mesh": {
- "bat0": {
- "interfaces": {
- "tunnel": [
- "52:54:00:f3:62:d9"
- ]
- }
- }
- }
- }
- }
-]
diff --git a/backend.py b/backend.py
deleted file mode 100755
index 8b9d9a6..0000000
--- a/backend.py
+++ /dev/null
@@ -1,194 +0,0 @@
-#!/usr/bin/env python3
-"""
-backend.py - ffmap-backend runner
-https://github.com/ffnord/ffmap-backend
-"""
-import argparse
-import json
-import os
-import sys
-from datetime import datetime
-
-import networkx as nx
-from networkx.readwrite import json_graph
-
-from lib import graph, nodes
-from lib.alfred import Alfred
-from lib.batman import Batman
-from lib.rrddb import RRD
-from lib.nodelist import export_nodelist
-from lib.validate import validate_nodeinfos
-
-NODES_VERSION = 1
-GRAPH_VERSION = 1
-
-
-def main(params):
- os.makedirs(params['dest_dir'], exist_ok=True)
-
- nodes_fn = os.path.join(params['dest_dir'], 'nodes.json')
- tmp_nodes_fn = os.path.join(params['dest_dir'], 'nodes.json.tmp')
-
- graph_fn = os.path.join(params['dest_dir'], 'graph.json')
- tmp_graph_fn = os.path.join(params['dest_dir'], 'graph.json.tmp')
-
- nodelist_fn = os.path.join(params['dest_dir'], 'nodelist.json')
- tmp_nodelist_fn = os.path.join(params['dest_dir'], 'nodelist.json.tmp')
-
- now = datetime.utcnow().replace(microsecond=0)
-
- # parse mesh param and instantiate Alfred/Batman instances
- alfred_instances = []
- batman_instances = []
- for value in params['mesh']:
- # (1) only batman-adv if, no alfred sock
- if ':' not in value:
- if len(params['mesh']) > 1:
- raise ValueError(
- 'Multiple mesh interfaces require the use of '
- 'alfred socket paths.')
- alfred_instances.append(Alfred(unix_sockpath=None))
- batman_instances.append(Batman(mesh_interface=value))
- else:
- # (2) batman-adv if + alfred socket
- try:
- batif, alfredsock = value.split(':')
- alfred_instances.append(Alfred(unix_sockpath=alfredsock))
- batman_instances.append(Batman(mesh_interface=batif,
- alfred_sockpath=alfredsock))
- except ValueError:
- raise ValueError(
- 'Unparseable value "{0}" in --mesh parameter.'.
- format(value))
-
- # read nodedb state from node.json
- try:
- with open(nodes_fn, 'r') as nodedb_handle:
- nodedb = json.load(nodedb_handle)
- except IOError:
- nodedb = {'nodes': dict()}
-
- # flush nodedb if it uses the old format
- if 'links' in nodedb:
- nodedb = {'nodes': dict()}
-
- # set version we're going to output
- nodedb['version'] = NODES_VERSION
-
- # update timestamp and assume all nodes are offline
- nodedb['timestamp'] = now.isoformat()
- for node_id, node in nodedb['nodes'].items():
- node['flags']['online'] = False
-
- # integrate alfred nodeinfo
- for alfred in alfred_instances:
- nodeinfo = validate_nodeinfos(alfred.nodeinfo())
- nodes.import_nodeinfo(nodedb['nodes'], nodeinfo,
- now, assume_online=True)
-
- # integrate static aliases data
- for aliases in params['aliases']:
- with open(aliases, 'r') as f:
-# nodeinfo = validate_nodeinfos(json.load(f))
- nodes.import_nodeinfo(nodedb['nodes'], json.load(f),
- now, assume_online=False, statics=True)
-
- nodes.reset_statistics(nodedb['nodes'])
- for alfred in alfred_instances:
- nodes.import_statistics(nodedb['nodes'], alfred.statistics())
-
- # acquire gwl and visdata for each batman instance
- mesh_info = []
- for batman in batman_instances:
- vd = batman.vis_data()
- gwl = batman.gateway_list()
- mesh_info.append((vd, gwl))
-
- # update nodedb from batman-adv data
- for vd, gwl in mesh_info:
- nodes.import_mesh_ifs_vis_data(nodedb['nodes'], vd)
- nodes.import_vis_clientcount(nodedb['nodes'], vd)
- nodes.mark_vis_data_online(nodedb['nodes'], vd, now)
- nodes.mark_gateways(nodedb['nodes'], gwl)
-
- # clear the nodedb from nodes that have not been online in $prune days
- if params['prune']:
- nodes.prune_nodes(nodedb['nodes'], now, params['prune'])
-
- # build nxnetworks graph from nodedb and visdata
- batadv_graph = nx.DiGraph()
- for vd, gwl in mesh_info:
- graph.import_vis_data(batadv_graph, nodedb['nodes'], vd)
-
- # force mac addresses to be vpn-link only (like gateways for example)
- if params['vpn']:
- graph.mark_vpn(batadv_graph, frozenset(params['vpn']))
-
- def extract_tunnel(nodes):
- macs = set()
- for id, node in nodes.items():
- try:
- for mac in node["nodeinfo"]["network"]["mesh"]["bat0"]["interfaces"]["tunnel"]:
- macs.add(mac)
- for mac in node["nodeinfo"]["network"]["mesh"]["bat-ffhh"]["interfaces"]["tunnel"]:
- macs.add(mac)
- except KeyError:
- pass
-
- return macs
-
- graph.mark_vpn(batadv_graph, extract_tunnel(nodedb['nodes']))
-
- batadv_graph = graph.merge_nodes(batadv_graph)
- batadv_graph = graph.to_undirected(batadv_graph)
-
- # write processed data to dest dir
- with open(tmp_nodes_fn, 'w') as f:
- json.dump(nodedb, f)
-
- graph_out = {'batadv': json_graph.node_link_data(batadv_graph),
- 'version': GRAPH_VERSION}
-
- with open(tmp_graph_fn, 'w') as f:
- json.dump(graph_out, f)
-
- with open(tmp_nodelist_fn, 'w') as f:
- json.dump(export_nodelist(now, nodedb), f)
-
- os.rename(tmp_nodes_fn, nodes_fn)
- os.rename(tmp_graph_fn, graph_fn)
- os.rename(tmp_nodelist_fn, nodelist_fn)
-
- # optional rrd graphs (trigger with --rrd)
- if params['rrd']:
- script_directory = os.path.dirname(os.path.realpath(__file__))
- rrd = RRD(os.path.join(script_directory, 'nodedb'),
- os.path.join(params['dest_dir'], 'nodes'))
- rrd.update_database(nodedb['nodes'])
- rrd.update_images()
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser()
-
- parser.add_argument('-a', '--aliases',
- help='Read aliases from FILE',
- nargs='+', default=[], metavar='FILE')
- parser.add_argument('-m', '--mesh',
- default=['bat0'], nargs='+',
- help='Use given batman-adv mesh interface(s) (defaults'
- 'to bat0); specify alfred unix socket like '
- 'bat0:/run/alfred0.sock.')
- parser.add_argument('-d', '--dest-dir', action='store',
- help='Write output to destination directory',
- required=True)
- parser.add_argument('-V', '--vpn', nargs='+', metavar='MAC',
- help='Assume MAC addresses are part of vpn')
- parser.add_argument('-p', '--prune', metavar='DAYS', type=int,
- help='forget nodes offline for at least DAYS')
- parser.add_argument('--with-rrd', dest='rrd', action='store_true',
- default=False,
- help='enable the rendering of RRD graphs (cpu '
- 'intensive)')
-
- options = vars(parser.parse_args())
- main(options)
diff --git a/ffmap-d3.jq b/ffmap-d3.jq
deleted file mode 100644
index ebeece1..0000000
--- a/ffmap-d3.jq
+++ /dev/null
@@ -1,52 +0,0 @@
-{
- "meta": {
- "timestamp": $nodes.timestamp
- },
- "nodes": (
- $graph.batadv.nodes
- | map(
- if has("node_id") and .node_id
- then (
- $nodes.nodes[.node_id] as $node
- | {
- "id": .id,
- "uptime": $node.statistics.uptime,
- "flags": ($node.flags + {"client": false}),
- "name": $node.nodeinfo.hostname,
- "clientcount": (if $node.statistics.clients >= 0 then $node.statistics.clients else 0 end),
- "hardware": $node.nodeinfo.hardware.model,
- "firmware": $node.nodeinfo.software.firmware.release,
- "geo": (if $node.nodeinfo.location then [$node.nodeinfo.location.latitude, $node.nodeinfo.location.longitude] else null end),
- #"lastseen": $node.lastseen,
- "network": $node.nodeinfo.network
- }
- )
- else
- {
- "flags": {},
- "id": .id,
- "geo": null,
- "clientcount": 0
- }
- end
- )
- ),
- "links": (
- $graph.batadv.links
- | map(
- $graph.batadv.nodes[.source].node_id as $source_id
- | $graph.batadv.nodes[.target].node_id as $target_id
- | select(
- $source_id and $target_id and
- ($nodes.nodes | (has($source_id) and has($target_id)))
- )
- | {
- "target": .target,
- "source": .source,
- "quality": "\(.tq), \(.tq)",
- "id": ($source_id + "-" + $target_id),
- "type": (if .vpn then "vpn" else null end)
- }
- )
- )
-}
diff --git a/ffmap/__init__.py b/ffmap/__init__.py
new file mode 100644
index 0000000..9542acc
--- /dev/null
+++ b/ffmap/__init__.py
@@ -0,0 +1,42 @@
+import importlib
+
+from ffmap.nodedb import NodeDB
+
+def run(inputs, outputs):
+ """Fill the database with given inputs and give it to given outputs.
+
+ Arguments:
+ inputs -- list of Input instances (with a compatible get_data(nodedb) method)
+ outputs -- list of Output instances (with a compatible output(nodedb) method)
+ """
+ db = NodeDB()
+ for input_ in inputs:
+ input_.get_data(db)
+
+ for output in outputs:
+ output.output(db)
+
+def run_names(inputs, outputs):
+ """Fill the database with inputs and give it to outputs, each given
+ by names.
+
+ In contrast to run(inputs, outputs), this method expects only the
+ names of the modules to use, not instances thereof.
+ Arguments:
+ inputs -- list of dicts, each dict having the keys "name" with the
+ name of the input to use (directory name in inputs/), and
+ the key "options" with a dict of input-dependent options.
+ outputs -- list of dicts, see inputs.
+ """
+ input_instances = []
+ output_instances = []
+
+ for input_ in inputs:
+ module = importlib.import_module(".inputs." + input_["name"], "ffmap")
+ input_instances.append(module.Input(**input_["options"]))
+
+ for output in outputs:
+ module = importlib.import_module(".outputs." + output["name"], "ffmap")
+ output_instances.append(module.Output(**output["options"]))
+
+ run(input_instances, output_instances)
diff --git a/nodedb/.gitkeep b/ffmap/inputs/__init__.py
similarity index 100%
rename from nodedb/.gitkeep
rename to ffmap/inputs/__init__.py
diff --git a/ffmap/inputs/alfred.py b/ffmap/inputs/alfred.py
new file mode 100644
index 0000000..9df5f5e
--- /dev/null
+++ b/ffmap/inputs/alfred.py
@@ -0,0 +1,29 @@
+import subprocess
+import json
+
+class Input:
+ def __init__(self,request_data_type = 158):
+ self.request_data_type = request_data_type
+
+ @staticmethod
+ def _call_alfred(request_data_type):
+ return json.loads(subprocess.check_output([
+ "alfred-json",
+ "-z",
+ "-r", str(request_data_type),
+ "-f", "json",
+ ]).decode("utf-8"))
+
+ def get_data(self, nodedb):
+ """Add data from alfred to the supplied nodedb"""
+ nodeinfo = self._call_alfred(self.request_data_type)
+ statistics = self._call_alfred(self.request_data_type+1)
+
+ # merge statistics into nodeinfo to be compatible with earlier versions
+ for mac, node in statistics.items():
+ if mac in nodeinfo:
+ nodeinfo[mac]['statistics'] = statistics[mac]
+
+ for mac, node in nodeinfo.items():
+ aliases = [mac] + node.get('network', {}).get('mesh_interfaces', [])
+ nodedb.add_or_update(aliases, node)
diff --git a/ffmap/inputs/batadv.py b/ffmap/inputs/batadv.py
new file mode 100644
index 0000000..3a5abc9
--- /dev/null
+++ b/ffmap/inputs/batadv.py
@@ -0,0 +1,100 @@
+import subprocess
+import json
+
+class Input:
+ """Fill the NodeDB with links from batadv-vis.
+
+ The links are added as lists containing the neighboring nodes, not
+ only their identifiers! Mind this when exporting the database, as
+ it probably leads to recursion.
+ """
+ def __init__(self, mesh_interface="bat0"):
+ self.mesh_interface = mesh_interface
+
+ @staticmethod
+ def _is_similar_mac(a, b):
+ """Determine if two MAC addresses are similar."""
+ if a == b:
+ return True
+
+ # Split the address into bytes
+ try:
+ mac_a = list(int(i, 16) for i in a.split(":"))
+ mac_b = list(int(i, 16) for i in b.split(":"))
+ except ValueError:
+ return False
+
+ # Second and third byte musn't differ
+ if mac_a[1] != mac_b[1] or mac_a[2] != mac_b[2]:
+ return False
+
+ # First byte must only differ in bits 2 and 3
+ if mac_a[0] | 6 != mac_b[0] | 6:
+ return False
+
+ # Count differing bytes after the third
+ c = [x for x in zip(mac_a[3:], mac_b[3:]) if x[0] != x[1]]
+
+ # No more than two additional bytes must differ
+ if len(c) > 2:
+ return False
+
+ # If no more bytes differ, they are very similar
+ if len(c) == 0:
+ return True
+
+ # If the sum of absolute differences isn't greater than 2, they
+ # are pretty similar
+ delta = sum(abs(i[0] - i[1]) for i in c)
+ return delta < 2
+
+ def get_data(self, nodedb):
+ """Add data from batadv-vis to the supplied nodedb"""
+ output = subprocess.check_output([
+ "batadv-vis",
+ "-i", str(self.mesh_interface),
+ "-f", "jsondoc",
+ ])
+ data = json.loads(output.decode("utf-8"))
+
+ # First pass
+ for node in data["vis"]:
+ # Determine possible other MAC addresses of this node by
+ # comparing all its client's MAC addresses to its primary
+ # MAC address. If they are similar, it probably is another
+ # address of the node itself! If it isn't, it is a real
+ # client.
+ node['aliases'] = [node["primary"]]
+ if 'secondary' in node:
+ node['aliases'].extend(node['secondary'])
+ real_clients = []
+ for mac in node["clients"]:
+ if self._is_similar_mac(mac, node["primary"]):
+ node['aliases'].append(mac)
+ else:
+ real_clients.append(mac)
+ node['clients'] = real_clients
+
+ # Add nodes and aliases without any information at first.
+ # This way, we can later link the objects themselves.
+ nodedb.add_or_update(node['aliases'])
+
+ # Second pass
+ for node in data["vis"]:
+ # We only need the primary address now, all aliases are
+ # already present in the database. Furthermore, we can be
+ # sure that all neighbors are in the database as well. If
+ # a neighbor isn't added already, we simply ignore it.
+ nodedb.add_or_update(
+ [node["primary"]],
+ {
+ "clients": node["clients"],
+ "neighbors": [
+ {
+ "metric": neighbor['metric'],
+ "neighbor": nodedb[neighbor['neighbor']],
+ } for neighbor in node["neighbors"]
+ if neighbor['neighbor'] in nodedb
+ ]
+ }
+ )
diff --git a/ffmap/inputs/wiki.py b/ffmap/inputs/wiki.py
new file mode 100755
index 0000000..ab36ad5
--- /dev/null
+++ b/ffmap/inputs/wiki.py
@@ -0,0 +1,71 @@
+import json
+import argparse
+from itertools import zip_longest
+from urllib.request import urlopen
+from bs4 import BeautifulSoup
+
+class Input:
+ def __init__(self, url="http://luebeck.freifunk.net/wiki/Knoten"):
+ self.url = url
+
+ def fetch_wikitable(self):
+ f = urlopen(self.url)
+ soup = BeautifulSoup(f)
+ table = soup.find("table")
+ rows = table.find_all("tr")
+ headers = []
+ data = []
+
+ def maybe_strip(x):
+ if isinstance(x.string, str):
+ return x.string.strip()
+ else:
+ return ""
+
+ for row in rows:
+ tds = list([maybe_strip(x) for x in row.find_all("td")])
+ ths = list([maybe_strip(x) for x in row.find_all("th")])
+
+ if any(tds):
+ data.append(tds)
+
+ if any(ths):
+ headers = ths
+
+ return [dict(zip(headers, d)) for d in data]
+
+ def get_data(self, nodedb):
+ nodes = self.fetch_wikitable()
+
+ for node in nodes:
+ if "MAC" not in node or not node["MAC"]:
+ # without MAC, we cannot merge this data with others, so
+ # we might as well ignore it
+ continue
+
+ newnode = {
+ "network": {
+ "mac": node.get("MAC").lower(),
+ },
+ "location": {
+ "latitude": float(node.get("GPS", " ").split(" ")[0]),
+ "longitude": float(node.get("GPS", " ").split(" ")[1]),
+ "description": node.get("Ort"),
+ } if " " in node.get("GPS", "") else None,
+ "hostname": node.get("Knotenname"),
+ "hardware": {
+ "model": node["Router"],
+ } if node.get("Router") else None,
+ "software": {
+ "firmware": {
+ "base": "LFF",
+ "release": node.get("LFF Version"),
+ },
+ },
+ "owner": {
+ "contact": node["Betreiber"],
+ } if node.get("Betreiber") else None,
+ }
+ # remove keys with None as value
+ newnode = {k: v for k,v in newnode.items() if v is not None}
+ nodedb.add_or_update([newnode["network"]["mac"]], newnode)
diff --git a/ffmap/node.py b/ffmap/node.py
new file mode 100644
index 0000000..b89dd19
--- /dev/null
+++ b/ffmap/node.py
@@ -0,0 +1,91 @@
+from collections import defaultdict
+
+class NoneDict:
+ """Act like None but return a NoneDict for every item request.
+
+ This is similar to the behaviour of collections.defaultdict in that
+ even previously inexistent keys can be accessed, but nothing is
+ stored permanently in this class.
+ """
+ def __repr__(self):
+ return 'NoneDict()'
+ def __bool__(self):
+ return False
+ def __getitem__(self, k):
+ return NoneDict()
+ def __json__(self):
+ return None
+ def __float__(self):
+ return float('NaN')
+ def __iter__(self):
+ # empty generator
+ return
+ yield
+ def __setitem__(self, key, value):
+ raise RuntimeError("NoneDict is readonly")
+
+class Node(defaultdict):
+ _id = None
+ def __init__(self, id_=None):
+ self._id = id_
+ super().__init__(NoneDict)
+
+ def __repr__(self):
+ return "Node(%s)" % self.id
+
+ @property
+ def id(self):
+ return self._id
+
+ def __hash__(self):
+ """Generate hash from the node's id.
+
+ WARNING: Obviously this hash doesn't cover all of the node's
+ data, but we need nodes to be hashable in order to eliminate
+ duplicates in the NodeDB.
+
+ At least the id cannot change after initialization...
+ """
+ return hash(self.id)
+
+ def deep_update(self, other):
+ """Update the dictionary like dict.update() but recursively."""
+ def dmerge(a, b):
+ for k, v in b.items():
+ if isinstance(v, dict) and isinstance(a.get(k), dict):
+ dmerge(a[k], v)
+ else:
+ a[k] = v
+ dmerge(self, other)
+
+ @property
+ def vpn_neighbors(self):
+ try:
+ vpn_neighbors = []
+ for neighbor in self['neighbors']:
+ if neighbor['neighbor']['vpn']:
+ vpn_neighbors.append(neighbor)
+ return vpn_neighbors
+ except TypeError:
+ return []
+
+ def export(self):
+ """Generate a serializable dict of the node.
+
+ In particular, this replaces any references to other nodes by
+ their id to prevent circular references.
+ """
+ ret = dict(self)
+ if "neighbors" in self:
+ ret["neighbors"] = []
+ for neighbor in self["neighbors"]:
+ new_neighbor = {}
+ for key, val in neighbor.items():
+ if isinstance(val, Node):
+ new_neighbor[key] = val.id
+ else:
+ new_neighbor[key] = val
+ ret["neighbors"].append(new_neighbor)
+ if "id" not in ret:
+ ret["id"] = self.id
+ return ret
diff --git a/ffmap/nodedb.py b/ffmap/nodedb.py
new file mode 100644
index 0000000..344ed29
--- /dev/null
+++ b/ffmap/nodedb.py
@@ -0,0 +1,60 @@
+from .node import Node
+
+class AmbiguityError(Exception):
+ """Indicate the ambiguity of identifiers.
+
+ This exception is raised if there is more than one match for a set
+ of identifiers.
+
+ Attributes:
+ identifiers -- set of ambiguous identifiers
+ """
+
+ identifiers = []
+
+ def __init__(self, identifiers):
+ self.identifiers = identifiers
+
+ def __str__(self):
+ return "Ambiguous identifiers: %s" % ", ".join(self.identifiers)
+
+class NodeDB(dict):
+ def add_or_update(self, ids, other=None):
+ """Add or update a node in the database.
+
+ Searches for an already existing node and updates it, or adds a new
+ one if no existing one is found. Raises an AmbiguityException if
+ more than one different nodes are found matching the criteria.
+
+ Arguments:
+ ids -- list of possible identifiers (probably MAC addresses) of the
+ node
+ other -- dict of values to update in an existing node or add to
+ the new one. Defaults to None, in which case no values
+ are added or updated, only the aliases of the
+ (possibly freshly created) node are updated.
+ """
+
+ # Find existing node, if any
+ node = None
+ node_id = None
+ for id_ in ids:
+ if id_ == node_id:
+ continue
+ if id_ in self:
+ if node is not None and node is not self[id_]:
+ raise AmbiguityError([node_id, id_])
+ node = self[id_]
+ node_id = id_
+
+ # If no node was found, create a new one
+ if node is None:
+ node = Node(ids[0])
+
+ # Update the node with the given properties using its own update method.
+ if other is not None:
+ node.deep_update(other)
+
+ # Add new aliases if any
+ for id_ in ids:
+ self[id_] = node
diff --git a/ffmap/outputs/__init__.py b/ffmap/outputs/__init__.py
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/ffmap/outputs/__init__.py
@@ -0,0 +1 @@
+
diff --git a/ffmap/outputs/d3json.py b/ffmap/outputs/d3json.py
new file mode 100644
index 0000000..31f03a6
--- /dev/null
+++ b/ffmap/outputs/d3json.py
@@ -0,0 +1,91 @@
+import json
+from datetime import datetime
+
+__all__ = ["Exporter"]
+
+class CustomJSONEncoder(json.JSONEncoder):
+ """
+ JSON encoder that uses an object's __json__() method to convert it
+ to something JSON-compatible.
+ """
+ def default(self, obj):
+ try:
+ return obj.__json__()
+ except AttributeError:
+ pass
+ return super().default(obj)
+
+class Output:
+ def __init__(self, filepath="nodes.json"):
+ self.filepath = filepath
+
+ @staticmethod
+ def generate(nodedb):
+ indexes = {}
+ nodes = []
+ count = 0
+ for node in set(nodedb.values()):
+ node_export = node.export()
+ node_export["flags"] = {
+ "gateway": "vpn" in node and node["vpn"],
+ "client": False,
+ "online": True
+ }
+ nodes.append(node_export)
+ indexes[node.id] = count
+ count += 1
+
+ links = {}
+ for node in set(nodedb.values()):
+ for neighbor in node.get("neighbors", []):
+ key = (neighbor["neighbor"].id, node.id)
+ rkey = tuple(reversed(key))
+ if rkey in links:
+ links[rkey]["quality"] += ","+neighbor["metric"]
+ else:
+ links[key] = {
+ "source": indexes[node.id],
+ "target": indexes[neighbor["neighbor"].id],
+ "quality": neighbor["metric"],
+ "type": "vpn" if neighbor["neighbor"]["vpn"] or node["vpn"] else None,
+ "id": "-".join((node.id, neighbor["neighbor"].id)),
+ }
+ clientcount = 0
+ for client in node.get("clients", []):
+ nodes.append({
+ "id": "%s-%s" % (node.id, clientcount),
+ "flags": {
+ "client": True,
+ "online": True,
+ "gateway": False
+ }
+ })
+ indexes[client] = count
+
+ links[(node.id, client)] = {
+ "source": indexes[node.id],
+ "target": indexes[client],
+ "quality": "TT",
+ "type": "client",
+ "id": "%s-%i" % (node.id, clientcount),
+ }
+ count += 1
+ clientcount += 1
+
+ return {
+ "nodes": nodes,
+ "links": list(links.values()),
+ "meta": {
+ "timestamp": datetime.utcnow()
+ .replace(microsecond=0)
+ .isoformat()
+ }
+ }
+
+ def output(self, nodedb):
+ with open(self.filepath, "w") as nodes_json:
+ json.dump(
+ self.generate(nodedb),
+ nodes_json,
+ cls=CustomJSONEncoder
+ )
diff --git a/ffmap/outputs/rrd.py b/ffmap/outputs/rrd.py
new file mode 100644
index 0000000..ce450c3
--- /dev/null
+++ b/ffmap/outputs/rrd.py
@@ -0,0 +1,30 @@
+import os
+from ffmap.rrd.rrds import NodeRRD, GlobalRRD
+
+class Output:
+ def __init__(self, directory="nodedb"):
+ self.directory = directory
+ try:
+ os.mkdir(self.directory)
+ except OSError:
+ pass
+
+ def output(self, nodedb):
+ nodes = set(nodedb.values())
+ clients = 0
+ nodecount = 0
+ for node in nodes:
+ clients += len(node.get("clients", []))
+ nodecount += 1
+ NodeRRD(
+ os.path.join(
+ self.directory,
+ str(node.id).replace(':', '') + '.rrd'
+ ),
+ node
+ ).update()
+
+ GlobalRRD(os.path.join(self.directory, "nodes.rrd")).update(
+ nodecount,
+ clients
+ )
diff --git a/lib/RRD.py b/ffmap/rrd/__init__.py
similarity index 83%
rename from lib/RRD.py
rename to ffmap/rrd/__init__.py
index 4e925a7..9bb87a0 100644
--- a/lib/RRD.py
+++ b/ffmap/rrd/__init__.py
@@ -1,20 +1,19 @@
import subprocess
import re
+import io
import os
+from tempfile import TemporaryFile
from operator import xor, eq
from functools import reduce
from itertools import starmap
import math
-
class RRDIncompatibleException(Exception):
"""
Is raised when an RRD doesn't have the desired definition and cannot be
upgraded to it.
"""
pass
-
-
class RRDOutdatedException(Exception):
"""
Is raised when an RRD doesn't have the desired definition, but can be
@@ -26,8 +25,7 @@ if not hasattr(__builtins__, "FileNotFoundError"):
class FileNotFoundError(Exception):
pass
-
-class RRD(object):
+class RRD:
"""
An RRD is a Round Robin Database, a database which forgets old data and
aggregates multiple records into new ones.
@@ -51,7 +49,7 @@ class RRD(object):
def _exec_rrdtool(self, cmd, *args, **kwargs):
pargs = ["rrdtool", cmd, self.filename]
- for k, v in kwargs.items():
+ for k,v in kwargs.items():
pargs.extend(["--" + k, str(v)])
pargs.extend(args)
subprocess.check_output(pargs)
@@ -59,7 +57,7 @@ class RRD(object):
def __init__(self, filename):
self.filename = filename
- def ensure_sanity(self, ds_list, rra_list, **kwargs):
+ def ensureSanity(self, ds_list, rra_list, **kwargs):
"""
Create or upgrade the RRD file if necessary to contain all DS in
ds_list. If it needs to be created, the RRAs in rra_list and any kwargs
@@ -67,13 +65,13 @@ class RRD(object):
database are NOT modified!
"""
try:
- self.check_sanity(ds_list)
+ self.checkSanity(ds_list)
except FileNotFoundError:
self.create(ds_list, rra_list, **kwargs)
except RRDOutdatedException:
self.upgrade(ds_list)
- def check_sanity(self, ds_list=()):
+ def checkSanity(self, ds_list=()):
"""
Check if the RRD file exists and contains (at least) the DS listed in
ds_list.
@@ -83,11 +81,8 @@ class RRD(object):
info = self.info()
if set(ds_list) - set(info['ds'].values()) != set():
for ds in ds_list:
- if ds.name in info['ds'] and\
- ds.type != info['ds'][ds.name].type:
- raise RRDIncompatibleException(
- "{} is {} but should be {}".format(
- ds.name, ds.type, info['ds'][ds.name].type))
+ if ds.name in info['ds'] and ds.type != info['ds'][ds.name].type:
+ raise RRDIncompatibleException("%s is %s but should be %s" % (ds.name, ds.type, info['ds'][ds.name].type))
else:
raise RRDOutdatedException()
@@ -110,10 +105,8 @@ class RRD(object):
if ds.name in info['ds']:
old_ds = info['ds'][ds.name]
if info['ds'][ds.name].type != ds.type:
- raise RuntimeError(
- "Cannot convert existing DS '{}'"
- "from type '{}' to '{}'".format(
- ds.name, old_ds.type, ds.type))
+ raise RuntimeError('Cannot convert existing DS "%s" from type "%s" to "%s"' %
+ (ds.name, old_ds.type, ds.type))
ds.index = old_ds.index
new_ds[ds.index] = ds
else:
@@ -123,11 +116,12 @@ class RRD(object):
dump = subprocess.Popen(
["rrdtool", "dump", self.filename],
- stdout=subprocess.PIPE)
-
+ stdout=subprocess.PIPE
+ )
restore = subprocess.Popen(
["rrdtool", "restore", "-", self.filename + ".new"],
- stdin=subprocess.PIPE)
+ stdin=subprocess.PIPE
+ )
echo = True
ds_definitions = True
for line in dump.stdout:
@@ -149,17 +143,19 @@ class RRD(object):
%s
%i
- """ % (ds.name,
- ds.type,
- ds.args[0],
- ds.args[1],
- ds.args[2],
- ds.last_ds,
- ds.value,
- ds.unknown_sec), "utf-8"))
+ """ % (
+ ds.name,
+ ds.type,
+ ds.args[0],
+ ds.args[1],
+ ds.args[2],
+ ds.last_ds,
+ ds.value,
+ ds.unknown_sec)
+ , "utf-8"))
if b'' in line:
- restore.stdin.write(added_ds_num * b"""
+ restore.stdin.write(added_ds_num*b"""
NaN
NaN
@@ -173,7 +169,7 @@ class RRD(object):
restore.stdin.write(
line.replace(
b'',
- (added_ds_num * b'NaN') + b''
+ (added_ds_num*b'NaN')+b''
)
)
@@ -241,8 +237,7 @@ class RRD(object):
for line in out.splitlines():
base = info
for match in self._info_regex.finditer(line):
- section, key, name, value = match.group(
- "section", "key", "name", "value")
+ section, key, name, value = match.group("section", "key", "name", "value")
if section and key:
try:
key = int(key)
@@ -263,8 +258,7 @@ class RRD(object):
base[name] = value
dss = {}
for name, ds in info['ds'].items():
- ds_obj = DS(name, ds['type'], ds['minimal_heartbeat'],
- ds['min'], ds['max'])
+ ds_obj = DS(name, ds['type'], ds['minimal_heartbeat'], ds['min'], ds['max'])
ds_obj.index = ds['index']
ds_obj.last_ds = ds['last_ds']
ds_obj.value = ds['value']
@@ -273,14 +267,12 @@ class RRD(object):
info['ds'] = dss
rras = []
for rra in info['rra'].values():
- rras.append(RRA(rra['cf'], rra['xff'],
- rra['pdp_per_row'], rra['rows']))
+ rras.append(RRA(rra['cf'], rra['xff'], rra['pdp_per_row'], rra['rows']))
info['rra'] = rras
self._cached_info = info
return info
-
-class DS(object):
+class DS:
"""
DS stands for Data Source and represents one line of data points in a Round
Robin Database (RRD).
@@ -292,7 +284,6 @@ class DS(object):
last_ds = 'U'
value = 0
unknown_sec = 0
-
def __init__(self, name, dst, *args):
self.name = name
self.type = dst
@@ -302,7 +293,7 @@ class DS(object):
return "DS:%s:%s:%s" % (
self.name,
self.type,
- ":".join(map(str, self._nan_to_u_args()))
+ ":".join(map(str, self._nan_to_U_args()))
)
def __repr__(self):
@@ -314,23 +305,22 @@ class DS(object):
)
def __eq__(self, other):
- return all(starmap(eq, zip(self.compare_keys(), other.compare_keys())))
+ return all(starmap(eq, zip(self._compare_keys(), other._compare_keys())))
def __hash__(self):
- return reduce(xor, map(hash, self.compare_keys()))
+ return reduce(xor, map(hash, self._compare_keys()))
- def _nan_to_u_args(self):
+ def _nan_to_U_args(self):
return tuple(
'U' if type(arg) is float and math.isnan(arg)
else arg
for arg in self.args
)
- def compare_keys(self):
- return self.name, self.type, self._nan_to_u_args()
+ def _compare_keys(self):
+ return (self.name, self.type, self._nan_to_U_args())
-
-class RRA(object):
+class RRA:
def __init__(self, cf, *args):
self.cf = cf
self.args = args
diff --git a/ffmap/rrd/rrds.py b/ffmap/rrd/rrds.py
new file mode 100644
index 0000000..2155d0c
--- /dev/null
+++ b/ffmap/rrd/rrds.py
@@ -0,0 +1,115 @@
+import os
+import subprocess
+from ffmap.node import Node
+from . import RRD, DS, RRA
+
+class NodeRRD(RRD):
+ ds_list = [
+ DS('upstate', 'GAUGE', 120, 0, 1),
+ DS('clients', 'GAUGE', 120, 0, float('NaN')),
+ DS('neighbors', 'GAUGE', 120, 0, float('NaN')),
+ DS('vpn_neighbors', 'GAUGE', 120, 0, float('NaN')),
+ DS('loadavg', 'GAUGE', 120, 0, float('NaN')),
+ DS('rx_bytes', 'DERIVE', 120, 0, float('NaN')),
+ DS('rx_packets', 'DERIVE', 120, 0, float('NaN')),
+ DS('tx_bytes', 'DERIVE', 120, 0, float('NaN')),
+ DS('tx_packets', 'DERIVE', 120, 0, float('NaN')),
+ DS('mgmt_rx_bytes', 'DERIVE', 120, 0, float('NaN')),
+ DS('mgmt_rx_packets', 'DERIVE', 120, 0, float('NaN')),
+ DS('mgmt_tx_bytes', 'DERIVE', 120, 0, float('NaN')),
+ DS('mgmt_tx_packets', 'DERIVE', 120, 0, float('NaN')),
+ DS('forward_bytes', 'DERIVE', 120, 0, float('NaN')),
+ DS('forward_packets', 'DERIVE', 120, 0, float('NaN')),
+ ]
+ rra_list = [
+ RRA('AVERAGE', 0.5, 1, 120), # 2 hours of 1 minute samples
+ RRA('AVERAGE', 0.5, 5, 1440), # 5 days of 5 minute samples
+ RRA('AVERAGE', 0.5, 60, 720), # 30 days of 1 hour samples
+ RRA('AVERAGE', 0.5, 720, 730), # 1 year of 12 hour samples
+ ]
+
+ def __init__(self, filename, node = None):
+ """
+ Create a new RRD for a given node.
+
+ If the RRD isn't supposed to be updated, the node can be omitted.
+ """
+ self.node = node
+ super().__init__(filename)
+ self.ensureSanity(self.ds_list, self.rra_list, step=60)
+
+ @property
+ def imagename(self):
+ return os.path.basename(self.filename).rsplit('.', 2)[0] + ".png"
+
+ def update(self):
+ values = {
+ 'upstate': 1,
+ 'clients': float(len(self.node.get('clients', []))),
+ 'neighbors': float(len(self.node.get('neighbors', []))),
+ 'vpn_neighbors': float(len(self.node.vpn_neighbors)),
+ 'loadavg': float(self.node['statistics']['loadavg']),
+ }
+ for item in ('rx', 'tx', 'mgmt_rx', 'mgmt_tx', 'forward'):
+ try:
+ values[item + '_bytes'] = int(self.node['statistics']['traffic'][item]['bytes'])
+ except TypeError:
+ pass
+ try:
+ values[item + '_packets'] = int(self.node['statistics']['traffic'][item]['packets'])
+ except TypeError:
+ pass
+ super().update(values)
+
+ def graph(self, directory, timeframe):
+ """
+ Create a graph in the given directory. The file will be named
+ basename.png if the RRD file is named basename.rrd
+ """
+ args = ['rrdtool','graph', os.path.join(directory, self.imagename),
+ '-s', '-' + timeframe ,
+ '-w', '800',
+ '-h', '400',
+ '-l', '0',
+ '-y', '1:1',
+ 'DEF:clients=' + self.filename + ':clients:AVERAGE',
+ 'VDEF:maxc=clients,MAXIMUM',
+ 'CDEF:c=0,clients,ADDNAN',
+ 'CDEF:d=clients,UN,maxc,UN,1,maxc,IF,*',
+ 'AREA:c#0F0:up\\l',
+ 'AREA:d#F00:down\\l',
+ 'LINE1:c#00F:clients connected\\l',
+ ]
+ subprocess.check_output(args)
+
+class GlobalRRD(RRD):
+ ds_list = [
+ # Number of nodes available
+ DS('nodes', 'GAUGE', 120, 0, float('NaN')),
+ # Number of client available
+ DS('clients', 'GAUGE', 120, 0, float('NaN')),
+ ]
+ rra_list = [
+ RRA('AVERAGE', 0.5, 1, 120), # 2 hours of 1 minute samples
+ RRA('AVERAGE', 0.5, 60, 744), # 31 days of 1 hour samples
+ RRA('AVERAGE', 0.5, 1440, 1780),# ~5 years of 1 day samples
+ ]
+
+ def __init__(self, filepath):
+ super().__init__(filepath)
+ self.ensureSanity(self.ds_list, self.rra_list, step=60)
+
+ def update(self, nodeCount, clientCount):
+ super().update({'nodes': nodeCount, 'clients': clientCount})
+
+ def graph(self, filename, timeframe):
+ args = ["rrdtool", 'graph', filename,
+ '-s', '-' + timeframe,
+ '-w', '800',
+ '-h' '400',
+ 'DEF:nodes=' + self.filename + ':nodes:AVERAGE',
+ 'LINE1:nodes#F00:nodes\\l',
+ 'DEF:clients=' + self.filename + ':clients:AVERAGE',
+ 'LINE2:clients#00F:clients',
+ ]
+ subprocess.check_output(args)
diff --git a/ffmap/run.py b/ffmap/run.py
new file mode 100644
index 0000000..a9e004f
--- /dev/null
+++ b/ffmap/run.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+import argparse
+import sys
+
+from ffmap import run_names
+
+class MyAction(argparse.Action):
+ def __call__(self, parser, namespace, values, option_string=None):
+ if self.dest.startswith(("input_", "output_")):
+ collection_name = self.dest.split("_")[0] + "s"
+ name = self.dest.split("_", 1)[1]
+ if not hasattr(namespace, collection_name):
+ setattr(namespace, collection_name, [])
+ collection = getattr(namespace, collection_name)
+ collection.append({
+ "name": name,
+ "options": {self.metavar.lower(): values}
+ if values is not None else {}
+ })
+ else:
+ raise Exception("Unexpected dest=" + self.dest)
+
+def parser_add_myarg(parser, name, metavar="OPT", help=None):
+ parser.add_argument("--" + name,
+ metavar=metavar,
+ type=str,
+ nargs='?',
+ const=None,
+ action=MyAction,
+ help=help)
+
+parser = argparse.ArgumentParser(
+ description="""Merge node data from multiple sources and generate
+ various output formats from this data""",
+)
+input_group = parser.add_argument_group("Inputs", description="""
+ Inputs are used in the order given on the command line, where later
+ inputs can overwrite attributes of earlier inputs if named equally,
+ but the first input encountering a node sets its id, which is
+ immutable afterwards.
+
+ The same input can be given multiple times, probably with different
+ options.
+""")
+output_group = parser.add_argument_group("Outputs")
+parser_add_myarg(input_group, 'input-alfred', metavar="REQUEST_DATA_TYPE",
+ help="read node details from A.L.F.R.E.D.")
+parser_add_myarg(input_group, 'input-wiki', metavar="URL",
+ help="read node details from a Wiki page")
+parser_add_myarg(input_group, 'input-batadv', metavar="MESH_INTERFACE",
+ help="add node's neighbors and clients from batadv-vis")
+parser_add_myarg(output_group, 'output-d3json', metavar="FILEPATH",
+ help="generate JSON file compatible with ffmap-d3")
+parser_add_myarg(output_group, 'output-rrd', metavar="DIRECTORY",
+ help="update RRDs with statistics, one global and one per node")
+
+args = parser.parse_args()
+
+if "inputs" not in args or not args.inputs:
+ parser.print_help(sys.stderr)
+ sys.stderr.write("\nERROR: No input has been defined!\n")
+ sys.exit(1)
+
+if "outputs" not in args or not args.outputs:
+ parser.print_help(sys.stderr)
+ sys.stderr.write("\nERROR: No output has been defined!\n")
+ sys.exit(1)
+
+run_names(inputs=args.inputs, outputs=args.outputs)
diff --git a/gateway.json b/gateway.json
deleted file mode 100644
index 4a72859..0000000
--- a/gateway.json
+++ /dev/null
@@ -1,29 +0,0 @@
-[
- {
- "node_id": "deadbfff0101",
- "hostname": "gw01"
- },
- {
- "node_id": "deadbeef0505",
- "hostname": "gw02.hamburg.freifunk.net",
- "network": {
- "mac": "de:ad:be:ef:05:05",
- "mesh": {
- "bat0": {
- "interfaces": {
- "tunnel": [
- "de:ad:be:ff:05:05",
- "de:ad:be:fc:05:05",
- "de:ad:bf:ff:05:05"
- ]
- }
- }
- }
- }
- },
- {
- "node_id": "00163efb9d8d",
- "hostname": "gw03"
- }
-]
-
diff --git a/generate_aliases.py b/generate_aliases.py
deleted file mode 100755
index 98fe407..0000000
--- a/generate_aliases.py
+++ /dev/null
@@ -1,110 +0,0 @@
-#!/usr/bin/env python2
-
-from __future__ import print_function
-
-import json
-import os
-import sys
-
-if len(sys.argv) != 2:
- print('usage: ' + sys.argv[0] + ' /path/to/peers')
- sys.exit(1)
-
-peersDir = sys.argv[1]
-
-def normalizeMac(mac):
- mac = mac.lower()
- normalized = ''
-
- n = 0
-
- for c in mac:
- if c != ':':
- if n > 0 and n % 2 == 0:
- normalized = normalized + ':'
- normalized = normalized + c
- n += 1
-
- return normalized
-
-def toAlias(peer):
- alias = {}
-
- if not (peer.has_key('name') and peer.has_key('mac')):
- return None
-
- name = peer['name']
- mac = peer['mac']
-
- alias['node_id'] = mac.replace(':', '')
- alias['hostname'] = name
-
- if peer.has_key('geo'):
- geo = peer['geo']
-
- location = {}
-
- if geo.has_key('lon'): location['longitude'] = geo['lon']
- if geo.has_key('lat'): location['latitude'] = geo['lat']
-
- alias['location'] = location
-
- #alias['network'] = {}
- #alias['network']['mesh_interfaces'] = [mac]
-
- return alias
-
-aliases = []
-
-for filename in os.listdir(peersDir):
- if len(filename) == 0 or filename[0] == '.':
- continue
-
- isGateway = False
-
- absFilename = peersDir + '/' + filename
- if os.path.isfile(absFilename):
- peerFile = open(absFilename, 'r')
- try:
- peerLines = peerFile.readlines()
- peer = {}
-
- for line in peerLines:
- parts = line.split()
-
- if len(parts) > 2:
- if parts[1] == 'Knotenname:':
- peer['name'] = parts[2]
-
- elif parts[0] == 'remote':
- isGateway = True
-
- elif parts[1] == 'MAC:':
- peer['mac'] = normalizeMac(parts[2])
-
- elif parts[1] == 'Koordinaten:' and len(parts) > 3:
- try:
- peer['geo'] = {'lat': float(parts[2]), 'lon': float(parts[3])}
-
- except ValueError:
- print('Error in %s: Invalid coordinates: %s' % (absFilename, parts[2:4]), file = sys.stderr)
-
- elif len(parts) == 2 and parts[0] == 'key':
- keyParts = parts[1].split('"')
- if len(keyParts) > 1:
- peer['vpn'] = keyParts[1].lower()
-
- if isGateway:
- continue
-
- alias = toAlias(peer)
- if alias:
- aliases.append(alias)
-
- except Exception as e:
- print('Error in %s, ignoring peer: %s' % (absFilename, e), file = sys.stderr)
-
- finally:
- peerFile.close()
-
-print(json.dumps(aliases))
diff --git a/generate_aliases_v2.py b/generate_aliases_v2.py
deleted file mode 100755
index 7a04c7c..0000000
--- a/generate_aliases_v2.py
+++ /dev/null
@@ -1,112 +0,0 @@
-#!/usr/bin/env python2
-
-from __future__ import print_function
-
-import json
-import os
-import sys
-
-if len(sys.argv) != 2:
- print('usage: ' + sys.argv[0] + ' /path/to/peers')
- sys.exit(1)
-
-peersDir = sys.argv[1]
-
-def normalizeMac(mac):
- mac = mac.lower()
- normalized = ''
-
- n = 0
-
- for c in mac:
- if c != ':':
- if n > 0 and n % 2 == 0:
- normalized = normalized + ':'
- normalized = normalized + c
- n += 1
-
- return normalized
-
-def toAlias(peer):
- alias = {}
-
- if not (peer.has_key('name') and peer.has_key('mac')):
- return None
-
- name = peer['name']
- mac = peer['mac']
-
- alias['node_id'] = mac.replace(':', '')
- alias['hostname'] = name
-
- if peer.has_key('geo'):
- geo = peer['geo']
-
- location = {}
-
- if geo.has_key('lon'): location['longitude'] = geo['lon']
- if geo.has_key('lat'): location['latitude'] = geo['lat']
-
- alias['location'] = location
-
- #alias['network'] = {}
- #alias['network']['mesh_interfaces'] = [mac]
-
- return {'nodeinfo':alias}
-
-aliases = {}
-
-for filename in os.listdir(peersDir):
- if len(filename) == 0 or filename[0] == '.':
- continue
-
- isGateway = False
-
- absFilename = peersDir + '/' + filename
- if os.path.isfile(absFilename):
- peerFile = open(absFilename, 'r')
- try:
- peerLines = peerFile.readlines()
- peer = {}
-
- for line in peerLines:
- parts = line.split()
-
- if len(parts) > 2:
- if parts[1] == 'Knotenname:':
- peer['name'] = parts[2]
-
- elif parts[0] == 'remote':
- isGateway = True
-
- elif parts[1] == 'MAC:':
- peer['mac'] = normalizeMac(parts[2])
-
- elif parts[1] == 'Koordinaten:' and len(parts) > 3:
- try:
- peer['geo'] = {'lat': float(parts[2]), 'lon': float(parts[3])}
-
- except ValueError:
- print('Error in %s: Invalid coordinates: %s' % (absFilename, parts[2:4]), file = sys.stderr)
-
- elif len(parts) == 2 and parts[0] == 'key':
- keyParts = parts[1].split('"')
- if len(keyParts) > 1:
- peer['vpn'] = keyParts[1].lower()
-
- if isGateway:
- continue
-
- alias = toAlias(peer)
- if alias:
- tmpid = alias['nodeinfo']['node_id']
-# alias['nodeinfo'].pop('node_id')
- aliases[tmpid] = alias
-
- except Exception as e:
- print('Error in %s, ignoring peer: %s' % (absFilename, e), file = sys.stderr)
-
- finally:
- peerFile.close()
-
-print(json.dumps(aliases))
diff --git a/lib/GlobalRRD.py b/lib/GlobalRRD.py
deleted file mode 100644
index 47235f2..0000000
--- a/lib/GlobalRRD.py
+++ /dev/null
@@ -1,40 +0,0 @@
-import os
-import subprocess
-
-from lib.RRD import DS, RRA, RRD
-
-
-class GlobalRRD(RRD):
- ds_list = [
- # Number of nodes available
- DS('nodes', 'GAUGE', 120, 0, float('NaN')),
- # Number of client available
- DS('clients', 'GAUGE', 120, 0, float('NaN')),
- ]
- rra_list = [
- # 2 hours of 1 minute samples
- RRA('AVERAGE', 0.5, 1, 120),
- # 31 days of 1 hour samples
- RRA('AVERAGE', 0.5, 60, 744),
- # ~5 years of 1 day samples
- RRA('AVERAGE', 0.5, 1440, 1780),
- ]
-
- def __init__(self, directory):
- super().__init__(os.path.join(directory, "nodes.rrd"))
- self.ensure_sanity(self.ds_list, self.rra_list, step=60)
-
- # TODO: fix this, python does not support function overloading
- def update(self, node_count, client_count):
- super().update({'nodes': node_count, 'clients': client_count})
-
- def graph(self, filename, timeframe):
- args = ["rrdtool", 'graph', filename,
- '-s', '-' + timeframe,
- '-w', '800',
- '-h' '400',
- 'DEF:nodes=' + self.filename + ':nodes:AVERAGE',
- 'LINE1:nodes#F00:nodes\\l',
- 'DEF:clients=' + self.filename + ':clients:AVERAGE',
- 'LINE2:clients#00F:clients']
- subprocess.check_output(args)
diff --git a/lib/NodeRRD.py b/lib/NodeRRD.py
deleted file mode 100644
index afabe6f..0000000
--- a/lib/NodeRRD.py
+++ /dev/null
@@ -1,61 +0,0 @@
-import os
-import subprocess
-
-from lib.RRD import DS, RRA, RRD
-
-
-class NodeRRD(RRD):
- ds_list = [
- DS('upstate', 'GAUGE', 120, 0, 1),
- DS('clients', 'GAUGE', 120, 0, float('NaN')),
- ]
- rra_list = [
- # 2 hours of 1 minute samples
- RRA('AVERAGE', 0.5, 1, 120),
- # 5 days of 5 minute samples
- RRA('AVERAGE', 0.5, 5, 1440),
- # 30 days of 1 hour samples
- RRA('AVERAGE', 0.5, 60, 720),
- # 1 year of 12 hour samples
- RRA('AVERAGE', 0.5, 720, 730),
- ]
-
- def __init__(self, filename, node=None):
- """
- Create a new RRD for a given node.
-
- If the RRD isn't supposed to be updated, the node can be omitted.
- """
- self.node = node
- super().__init__(filename)
- self.ensure_sanity(self.ds_list, self.rra_list, step=60)
-
- @property
- def imagename(self):
- return "{basename}.png".format(
- basename=os.path.basename(self.filename).rsplit('.', 2)[0])
-
- # TODO: fix this, python does not support function overloading
- def update(self):
- super().update({'upstate': int(self.node['flags']['online']),
- 'clients': self.node['statistics']['clients']})
-
- def graph(self, directory, timeframe):
- """
- Create a graph in the given directory. The file will be named
- basename.png if the RRD file is named basename.rrd
- """
- args = ['rrdtool', 'graph', os.path.join(directory, self.imagename),
- '-s', '-' + timeframe,
- '-w', '800',
- '-h', '400',
- '-l', '0',
- '-y', '1:1',
- 'DEF:clients=' + self.filename + ':clients:AVERAGE',
- 'VDEF:maxc=clients,MAXIMUM',
- 'CDEF:c=0,clients,ADDNAN',
- 'CDEF:d=clients,UN,maxc,UN,1,maxc,IF,*',
- 'AREA:c#0F0:up\\l',
- 'AREA:d#F00:down\\l',
- 'LINE1:c#00F:clients connected\\l']
- subprocess.check_output(args)
diff --git a/lib/__init__.py b/lib/__init__.py
deleted file mode 100644
index 64bd3f3..0000000
--- a/lib/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-__author__ = 'hexa'
diff --git a/lib/alfred.py b/lib/alfred.py
deleted file mode 100644
index 531eaea..0000000
--- a/lib/alfred.py
+++ /dev/null
@@ -1,33 +0,0 @@
-import subprocess
-import json
-import os
-
-
-class Alfred(object):
- """
- Bindings for the alfred-json utility
- """
- def __init__(self, unix_sockpath=None):
- self.unix_sock = unix_sockpath
- if unix_sockpath is not None and not os.path.exists(unix_sockpath):
- raise RuntimeError('alfred: invalid unix socket path given')
-
- def _fetch(self, data_type):
- cmd = ['/usr/local/bin/alfred-json',
- '-z',
- '-f', 'json',
- '-r', str(data_type)]
- if self.unix_sock:
- cmd.extend(['-s', self.unix_sock])
-
- output = subprocess.check_output(cmd)
- return json.loads(output.decode("utf-8")).values()
-
- def nodeinfo(self):
- return self._fetch(158)
-
- def statistics(self):
- return self._fetch(159)
-
- def vis(self):
- return self._fetch(160)
diff --git a/lib/batman.py b/lib/batman.py
deleted file mode 100644
index ee0198b..0000000
--- a/lib/batman.py
+++ /dev/null
@@ -1,98 +0,0 @@
-import subprocess
-import json
-import os
-import re
-
-
-class Batman(object):
- """
- Bindings for B.A.T.M.A.N. Advanced
- commandline interface "batctl"
- """
- def __init__(self, mesh_interface='bat0', alfred_sockpath=None):
- self.mesh_interface = mesh_interface
- self.alfred_sock = alfred_sockpath
-
- # ensure /usr/sbin and /usr/local/sbin are in PATH
- env = os.environ
- path = set(env['PATH'].split(':'))
- path.add('/usr/sbin/')
- path.add('/usr/local/sbin')
- env['PATH'] = ':'.join(path)
- self.environ = env
-
- # compile regular expressions only once on startup
- self.mac_addr_pattern = re.compile(r'(([a-z0-9]{2}:){5}[a-z0-9]{2})')
-
- def vis_data(self):
- return self.vis_data_batadv_vis()
-
- @staticmethod
- def vis_data_helper(lines):
- vd_tmp = []
- for line in lines:
- try:
- utf8_line = line.decode('utf-8')
- vd_tmp.append(json.loads(utf8_line))
- except UnicodeDecodeError:
- pass
- return vd_tmp
-
- def vis_data_batadv_vis(self):
- """
- Parse "batadv-vis -i -f json"
- into an array of dictionaries.
- """
- cmd = ['batadv-vis', '-i', self.mesh_interface, '-f', 'json']
- if self.alfred_sock:
- cmd.extend(['-u', self.alfred_sock])
- output = subprocess.check_output(cmd, env=self.environ)
- lines = output.splitlines()
- return self.vis_data_helper(lines)
-
- def gateway_list(self):
- """
- Parse "batctl -m gwl -n"
- into an array of dictionaries.
- """
- cmd = ['batctl', '-m', self.mesh_interface, 'gwl', '-n']
- if os.geteuid() > 0:
- cmd.insert(0, 'sudo')
- output = subprocess.check_output(cmd, env=self.environ)
- output_utf8 = output.decode('utf-8')
- rows = output_utf8.splitlines()
-
- gateways = []
-
- # local gateway
- header = rows.pop(0)
- mode, bandwidth = self.gateway_mode()
- if mode == 'server':
- local_gw_mac = self.mac_addr_pattern.search(header).group(0)
- gateways.append(local_gw_mac)
-
- # remote gateway(s)
- for row in rows:
- match = self.mac_addr_pattern.search(row)
- if match:
- gateways.append(match.group(1))
-
- return gateways
-
- def gateway_mode(self):
- """
- Parse "batctl -m gw"
- return: tuple mode, bandwidth, if mode != server then bandwidth is None
- """
- cmd = ['batctl', '-m', self.mesh_interface, 'gw']
- if os.geteuid() > 0:
- cmd.insert(0, 'sudo')
- output = subprocess.check_output(cmd, env=self.environ)
- chunks = output.decode("utf-8").split()
-
- return chunks[0], chunks[3] if 3 in chunks else None
-
-if __name__ == "__main__":
- bc = Batman()
- vd = bc.vis_data()
- gw = bc.gateway_list()
diff --git a/lib/graph.py b/lib/graph.py
deleted file mode 100644
index 2833e6e..0000000
--- a/lib/graph.py
+++ /dev/null
@@ -1,84 +0,0 @@
-from functools import reduce
-from itertools import chain
-
-import networkx as nx
-
-from lib.nodes import build_mac_table
-
-
-def import_vis_data(graph, nodes, vis_data):
- macs = build_mac_table(nodes)
- nodes_a = map(lambda d: 2 * [d['primary']],
- filter(lambda d: 'primary' in d, vis_data))
- nodes_b = map(lambda d: [d['secondary'], d['of']],
- filter(lambda d: 'secondary' in d, vis_data))
- graph.add_nodes_from(map(lambda a, b:
- (a, dict(primary=b, node_id=macs.get(b))),
- *zip(*chain(nodes_a, nodes_b))))
-
- edges = filter(lambda d: 'neighbor' in d, vis_data)
- graph.add_edges_from(map(lambda d: (d['router'], d['neighbor'],
- dict(tq=float(d['label']))), edges))
-
-
-def mark_vpn(graph, vpn_macs):
- components = map(frozenset, nx.weakly_connected_components(graph))
- components = filter(vpn_macs.intersection, components)
- nodes = reduce(lambda a, b: a | b, components, set())
- for node in nodes:
- for k, v in graph[node].items():
- v['vpn'] = True
-
-
-def to_multigraph(graph):
- def f(a):
- node = graph.node[a]
- return node['primary'] if node else a
-
- def map_node(node, data):
- return (data['primary'],
- dict(node_id=data['node_id'])) if data else (node, dict())
-
- digraph = nx.MultiDiGraph()
- digraph.add_nodes_from(map(map_node, *zip(*graph.nodes_iter(data=True))))
- digraph.add_edges_from(map(lambda a, b, data: (f(a), f(b), data),
- *zip(*graph.edges_iter(data=True))))
-
- return digraph
-
-
-def merge_nodes(graph):
- def merge_edges(data):
- tq = min(map(lambda d: d['tq'], data))
- vpn = all(map(lambda d: d.get('vpn', False), data))
- return dict(tq=tq, vpn=vpn)
-
- multigraph = to_multigraph(graph)
- digraph = nx.DiGraph()
- digraph.add_nodes_from(multigraph.nodes_iter(data=True))
- edges = chain.from_iterable([[(e, d, merge_edges(
- multigraph[e][d].values()))
- for d in multigraph[e]] for e in multigraph])
- digraph.add_edges_from(edges)
-
- return digraph
-
-
-def to_undirected(graph):
- multigraph = nx.MultiGraph()
- multigraph.add_nodes_from(graph.nodes_iter(data=True))
- multigraph.add_edges_from(graph.edges_iter(data=True))
-
- def merge_edges(data):
- tq = max(map(lambda d: d['tq'], data))
- vpn = all(map(lambda d: d.get('vpn', False), data))
- return dict(tq=tq, vpn=vpn, bidirect=len(data) == 2)
-
- graph = nx.Graph()
- graph.add_nodes_from(multigraph.nodes_iter(data=True))
- edges = chain.from_iterable([[(e, d, merge_edges(
- multigraph[e][d].values()))
- for d in multigraph[e]] for e in multigraph])
- graph.add_edges_from(edges)
-
- return graph
diff --git a/lib/nodelist.py b/lib/nodelist.py
deleted file mode 100644
index a931dcf..0000000
--- a/lib/nodelist.py
+++ /dev/null
@@ -1,27 +0,0 @@
-def export_nodelist(now, nodedb):
- nodelist = list()
-
- for node_id, node in nodedb["nodes"].items():
- node_out = dict()
- node_out["id"] = node_id
- node_out["name"] = node["nodeinfo"]["hostname"]
-
- if "location" in node["nodeinfo"]:
- node_out["position"] = {"lat": node["nodeinfo"]["location"]["latitude"],
- "long": node["nodeinfo"]["location"]["longitude"]}
-
- node_out["status"] = dict()
- node_out["status"]["online"] = node["flags"]["online"]
-
- if "firstseen" in node:
- node_out["status"]["firstcontact"] = node["firstseen"]
-
- if "lastseen" in node:
- node_out["status"]["lastcontact"] = node["lastseen"]
-
- if "clients" in node["statistics"]:
- node_out["status"]["clients"] = node["statistics"]["clients"]
-
- nodelist.append(node_out)
-
- return {"version": "1.0.1", "nodes": nodelist, "updated_at": now.isoformat()}
diff --git a/lib/nodes.py b/lib/nodes.py
deleted file mode 100644
index 86ce5b9..0000000
--- a/lib/nodes.py
+++ /dev/null
@@ -1,202 +0,0 @@
-from collections import Counter, defaultdict
-from datetime import datetime
-from functools import reduce
-
-
-def build_mac_table(nodes):
- macs = dict()
- for node_id, node in nodes.items():
- try:
- macs[node['network']['mac']] = node_id
- except KeyError:
- pass
- try:
- for mac in node['nodeinfo']['network']['mesh_interfaces']:
- macs[mac] = node_id
- except KeyError:
- pass
-
- try:
- for mac in node['nodeinfo']['network']['mesh']['bat0']['interfaces']['wireless']:
- macs[mac] = node_id
- except KeyError:
- pass
-
- try:
- for mac in node['nodeinfo']['network']['mesh']['bat0']['interfaces']['tunnel']:
- macs[mac] = node_id
- except KeyError:
- pass
- try:
- for mac in node['nodeinfo']['network']['mesh']['bat-ffhh']['interfaces']['tunnel']:
- macs[mac] = node_id
- except KeyError:
- pass
-
- try:
- for mac in node['nodeinfo']['network']['mesh']['bat0']['interfaces']['other']:
- macs[mac] = node_id
- except KeyError:
- pass
-
- return macs
-
-
-def prune_nodes(nodes, now, days):
- prune = []
- for node_id, node in nodes.items():
- if 'lastseen' not in node:
- prune.append(node_id)
- continue
-
- lastseen = datetime.strptime(node['lastseen'], '%Y-%m-%dT%H:%M:%S')
- delta = (now - lastseen).days
-
- if delta >= days:
- prune.append(node_id)
-
- for node_id in prune:
- del nodes[node_id]
-
-
-def mark_online(node, now):
- node['lastseen'] = now.isoformat()
- node.setdefault('firstseen', now.isoformat())
- node['flags']['online'] = True
-
-
-def overrideFields(dest, src, fields):
- for field in fields:
- if field in src:
- dest[field] = src[field]
- else:
- dest.pop(field, None)
-
-
-def import_nodeinfo(nodes, nodeinfos, now, assume_online=False, statics=False):
- for nodeinfo in filter(lambda d: 'node_id' in d, nodeinfos):
- node = nodes.setdefault(nodeinfo['node_id'], {'flags': {'online': False, 'gateway': False}})
-
- if statics:
- node['nodeinfo'] = node.setdefault('nodeinfo', {})
- overrideFields(node['nodeinfo'], nodeinfo, ['hostname', 'location', 'node_id'])
- else:
- node['nodeinfo'] = nodeinfo
-
- if assume_online:
- mark_online(node, now)
-
-
-def reset_statistics(nodes):
- for node in nodes.values():
- node['statistics'] = {'clients': 0}
-
-
-def import_statistics(nodes, stats):
- def add(node, statistics, target, source, f=lambda d: d):
- try:
- node['statistics'][target] = f(reduce(dict.__getitem__,
- source,
- statistics))
- except (KeyError, TypeError, ZeroDivisionError):
- pass
-
- macs = build_mac_table(nodes)
- stats = filter(lambda d: 'node_id' in d, stats)
- stats = filter(lambda d: d['node_id'] in nodes, stats)
- for node, stats in map(lambda d: (nodes[d['node_id']], d), stats):
- add(node, stats, 'clients', ['clients', 'total'])
- add(node, stats, 'gateway', ['gateway'], lambda d: macs.get(d, d))
- add(node, stats, 'uptime', ['uptime'])
- add(node, stats, 'loadavg', ['loadavg'])
- add(node, stats, 'memory_usage', ['memory'],
- lambda d: 1 - d['free'] / d['total'])
- add(node, stats, 'rootfs_usage', ['rootfs_usage'])
- add(node, stats, 'traffic', ['traffic'])
-
-
-def import_mesh_ifs_vis_data(nodes, vis_data):
- macs = build_mac_table(nodes)
-
- mesh_ifs = defaultdict(lambda: set())
- for line in filter(lambda d: 'secondary' in d, vis_data):
- primary = line['of']
- mesh_ifs[primary].add(primary)
- mesh_ifs[primary].add(line['secondary'])
-
- def if_to_node(ifs):
- a = filter(lambda d: d in macs, ifs)
- a = map(lambda d: nodes[macs[d]], a)
- try:
- return next(a), ifs
- except StopIteration:
- return None
-
- mesh_nodes = filter(lambda d: d, map(if_to_node, mesh_ifs.values()))
-
- for v in mesh_nodes:
- node = v[0]
-
- ifs = set()
-
- try:
- ifs = ifs.union(set(node['nodeinfo']['network']['mesh_interfaces']))
- except KeyError:
- pass
-
- try:
- ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat0']['interfaces']['wireless']))
- except KeyError:
- pass
-
- try:
- ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat0']['interfaces']['tunnel']))
- except KeyError:
- pass
-
- try:
- ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat-ffhh']['interfaces']['tunnel']))
- except KeyError:
- pass
-
- try:
- ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat0']['interfaces']['other']))
- except KeyError:
- pass
-
- node['nodeinfo']['network']['mesh_interfaces'] = list(ifs | v[1])
-
-
-def import_vis_clientcount(nodes, vis_data):
- macs = build_mac_table(nodes)
- data = filter(lambda d: d.get('label', None) == 'TT', vis_data)
- data = filter(lambda d: d['router'] in macs, data)
- data = map(lambda d: macs[d['router']], data)
-
- for node_id, clientcount in Counter(data).items():
- nodes[node_id]['statistics'].setdefault('clients', clientcount)
-
-
-def mark_gateways(nodes, gateways):
- macs = build_mac_table(nodes)
- gateways = filter(lambda d: d in macs, gateways)
- for node in map(lambda d: nodes[macs[d]], gateways):
- node['flags']['gateway'] = True
-
-
-def mark_vis_data_online(nodes, vis_data, now):
- macs = build_mac_table(nodes)
-
- online = set()
- for line in vis_data:
- if 'primary' in line:
- online.add(line['primary'])
- elif 'secondary' in line:
- online.add(line['secondary'])
- elif 'gateway' in line:
- # This matches clients' MACs.
- # On pre-Gluon nodes the primary MAC will be one of it.
- online.add(line['gateway'])
-
- for mac in filter(lambda d: d in macs, online):
- mark_online(nodes[macs[mac]], now)
diff --git a/lib/rrddb.py b/lib/rrddb.py
deleted file mode 100644
index f1678f5..0000000
--- a/lib/rrddb.py
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/usr/bin/env python3
-import time
-import os
-
-from lib.GlobalRRD import GlobalRRD
-from lib.NodeRRD import NodeRRD
-
-
-class RRD(object):
- def __init__(self,
- database_directory,
- image_path,
- display_time_global="7d",
- display_time_node="1d"):
-
- self.dbPath = database_directory
- self.globalDb = GlobalRRD(self.dbPath)
- self.imagePath = image_path
- self.displayTimeGlobal = display_time_global
- self.displayTimeNode = display_time_node
-
- self.currentTimeInt = (int(time.time()) / 60) * 60
- self.currentTime = str(self.currentTimeInt)
-
- try:
- os.stat(self.imagePath)
- except OSError:
- os.mkdir(self.imagePath)
-
- def update_database(self, nodes):
- online_nodes = dict(filter(
- lambda d: d[1]['flags']['online'], nodes.items()))
- client_count = sum(map(
- lambda d: d['statistics']['clients'], online_nodes.values()))
-
- self.globalDb.update(len(online_nodes), client_count)
- for node_id, node in online_nodes.items():
- rrd = NodeRRD(os.path.join(self.dbPath, node_id + '.rrd'), node)
- rrd.update()
-
- def update_images(self):
- self.globalDb.graph(os.path.join(self.imagePath, "globalGraph.png"),
- self.displayTimeGlobal)
-
- nodedb_files = os.listdir(self.dbPath)
-
- for file_name in nodedb_files:
- if not os.path.isfile(os.path.join(self.dbPath, file_name)):
- continue
-
- node_name = os.path.basename(file_name).split('.')
- if node_name[1] == 'rrd' and not node_name[0] == "nodes":
- rrd = NodeRRD(os.path.join(self.dbPath, file_name))
- rrd.graph(self.imagePath, self.displayTimeNode)
diff --git a/lib/validate.py b/lib/validate.py
deleted file mode 100644
index eb6c11f..0000000
--- a/lib/validate.py
+++ /dev/null
@@ -1,19 +0,0 @@
-import json
-
-
-def validate_nodeinfos(nodeinfos):
- result = []
-
- for nodeinfo in nodeinfos:
- if validate_nodeinfo(nodeinfo):
- result.append(nodeinfo)
-
- return result
-
-
-def validate_nodeinfo(nodeinfo):
- if 'location' in nodeinfo:
- if 'latitude' not in nodeinfo['location'] or 'longitude' not in nodeinfo['location']:
- return False
-
- return True
diff --git a/mkmap.sh b/mkmap.sh
deleted file mode 100755
index 9423943..0000000
--- a/mkmap.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-FFMAPPATH='/opt/ffmap-backend/'
-PEERS="/etc/fastd/ffhh-mesh-vpn/peers"
-
-python2 $FFMAPPATH/generate_aliases.py $PEERS > $FFMAPPATH/aliases.json
-#python3 $FFMAPPATH/backend.py -d /var/www/meshviewer/ --aliases $FFMAPPATH/aliases.json $FFMAPPATH/gateway.json -m bat0:/var/run/alfred.sock -p 30 --vpn de:ad:be:ff:01:01 --vpn de:ad:be:ff:05:05 --vpn de:ad:be:ff:05:06 --vpn de:ad:be:ff:03:03 --vpn de:ad:be:ff:22:22 --vpn de:ad:be:ff:22:23 --vpn de:ad:be:ff:88:88 --vpn de:ad:be:ff:88:89 --vpn de:ad:bf:ff:88:88 --vpn de:ad:bf:ff:22:22 --vpn de:ad:bf:ff:03:03 --vpn de:ad:bf:ff:05:05 --vpn de:ad:bf:ff:01:01 --vpn de:ad:be:fc:03:03 --vpn 00:16:3e:53:75:0d --vpn de:ad:be:fc:05:05 --vpn de:ad:be:fc:01:01 --vpn de:ad:be:ef:03:03 --vpn de:ad:be:ef:01:01 --vpn de:ad:be:ef:05:05 --vpn 00:16:3e:fb:9d:8d --vpn 00:16:3e:fb:9d:9d
-python3 $FFMAPPATH/backend.py -d /var/www/meshviewer/ --aliases $FFMAPPATH/aliases.json $FFMAPPATH/gateway.json -m bat0:/var/run/alfred.sock -p 30 --vpn de:ad:be:ff:01:01 de:ad:be:ff:05:05 de:ad:be:ff:05:06 de:ad:be:ff:03:03 de:ad:be:ff:22:22 de:ad:be:ff:22:23 de:ad:be:ff:88:88 de:ad:be:ff:88:89 de:ad:bf:ff:88:88 de:ad:bf:ff:22:22 de:ad:bf:ff:03:03 de:ad:bf:ff:05:05 de:ad:bf:ff:01:01 de:ad:be:fc:03:03 00:16:3e:53:75:0d de:ad:be:fc:05:05 de:ad:be:fc:01:01 de:ad:be:ef:03:03 de:ad:be:ef:01:01 de:ad:be:ef:05:05 00:16:3e:fb:9d:8d 00:16:3e:fb:9d:9d
diff --git a/node_number.py b/node_number.py
deleted file mode 100755
index 20cd00e..0000000
--- a/node_number.py
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env python
-#Bibliotheken importieren
-import time
-import datetime
-import json
-import urllib2
-
-#Datei oeffnen
-Datei = urllib2.urlopen('https://map.hamburg.freifunk.net/nodes.json')
-Datei_Sued = urllib2.urlopen('https://map.hamburg.freifunk.net/hhsued/mv1/nodes.json')
-
-Text = Datei.read()
-Knotenzahl = Text.count('"online": true')
-Text = Datei_Sued.read()
-Knotenzahl = Knotenzahl + Text.count('"online":true')
-
-#Zeit holen
-thetime = datetime.datetime.now().isoformat()
-
-ffhh = None
-
-#Freifunk API-Datei einladen und JSON lesen
-with open('/var/www/meta/ffhh.json', 'r') as fp:
- ffhh = json.load(fp)
-
-#Attribute Zeitstempel und Knotenanzahl setzen
-ffhh['state']['lastchange'] = thetime
-ffhh['state']['nodes'] = Knotenzahl
-
-#Freifunk API-Datein mit geaenderten werten schreiben
-with open('/var/www/meta/ffhh.json', 'w') as fp:
- json.dump(ffhh, fp, indent=2, separators=(',', ': '))
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..4ee3d1f
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+
+from distutils.core import setup
+
+setup(name='FFmap',
+ version='0.1',
+ description='Freifunk map backend',
+ url='https://github.com/ffnord/ffmap-backend',
+ packages=['ffmap', 'ffmap.inputs', 'ffmap.outputs', 'ffmap.rrd'],
+ )