diff --git a/backend.py b/backend.py
index 685c606..b570ffa 100755
--- a/backend.py
+++ b/backend.py
@@ -15,7 +15,6 @@ from networkx.readwrite import json_graph
from lib import graph, nodes
from lib.alfred import Alfred
from lib.batman import Batman
-from lib.rrddb import RRD
from lib.nodelist import export_nodelist
NODES_VERSION = 1
@@ -133,13 +132,6 @@ def main(params):
with open(nodelist_fn, 'w') as f:
json.dump(export_nodelist(now, nodedb), f)
- # optional rrd graphs (trigger with --rrd)
- if params['rrd']:
- script_directory = os.path.dirname(os.path.realpath(__file__))
- rrd = RRD(os.path.join(script_directory, 'nodedb'),
- os.path.join(params['dest_dir'], 'nodes'))
- rrd.update_database(nodedb['nodes'])
- rrd.update_images()
if __name__ == '__main__':
@@ -160,10 +152,6 @@ if __name__ == '__main__':
help='Assume MAC addresses are part of vpn')
parser.add_argument('-p', '--prune', metavar='DAYS', type=int,
help='forget nodes offline for at least DAYS')
- parser.add_argument('--with-rrd', dest='rrd', action='store_true',
- default=False,
- help='enable the rendering of RRD graphs (cpu '
- 'intensive)')
options = vars(parser.parse_args())
main(options)
diff --git a/lib/GlobalRRD.py b/lib/GlobalRRD.py
deleted file mode 100644
index 47235f2..0000000
--- a/lib/GlobalRRD.py
+++ /dev/null
@@ -1,40 +0,0 @@
-import os
-import subprocess
-
-from lib.RRD import DS, RRA, RRD
-
-
-class GlobalRRD(RRD):
- ds_list = [
- # Number of nodes available
- DS('nodes', 'GAUGE', 120, 0, float('NaN')),
- # Number of client available
- DS('clients', 'GAUGE', 120, 0, float('NaN')),
- ]
- rra_list = [
- # 2 hours of 1 minute samples
- RRA('AVERAGE', 0.5, 1, 120),
- # 31 days of 1 hour samples
- RRA('AVERAGE', 0.5, 60, 744),
- # ~5 years of 1 day samples
- RRA('AVERAGE', 0.5, 1440, 1780),
- ]
-
- def __init__(self, directory):
- super().__init__(os.path.join(directory, "nodes.rrd"))
- self.ensure_sanity(self.ds_list, self.rra_list, step=60)
-
- # TODO: fix this, python does not support function overloading
- def update(self, node_count, client_count):
- super().update({'nodes': node_count, 'clients': client_count})
-
- def graph(self, filename, timeframe):
- args = ["rrdtool", 'graph', filename,
- '-s', '-' + timeframe,
- '-w', '800',
- '-h' '400',
- 'DEF:nodes=' + self.filename + ':nodes:AVERAGE',
- 'LINE1:nodes#F00:nodes\\l',
- 'DEF:clients=' + self.filename + ':clients:AVERAGE',
- 'LINE2:clients#00F:clients']
- subprocess.check_output(args)
diff --git a/lib/NodeRRD.py b/lib/NodeRRD.py
deleted file mode 100644
index afabe6f..0000000
--- a/lib/NodeRRD.py
+++ /dev/null
@@ -1,61 +0,0 @@
-import os
-import subprocess
-
-from lib.RRD import DS, RRA, RRD
-
-
-class NodeRRD(RRD):
- ds_list = [
- DS('upstate', 'GAUGE', 120, 0, 1),
- DS('clients', 'GAUGE', 120, 0, float('NaN')),
- ]
- rra_list = [
- # 2 hours of 1 minute samples
- RRA('AVERAGE', 0.5, 1, 120),
- # 5 days of 5 minute samples
- RRA('AVERAGE', 0.5, 5, 1440),
- # 30 days of 1 hour samples
- RRA('AVERAGE', 0.5, 60, 720),
- # 1 year of 12 hour samples
- RRA('AVERAGE', 0.5, 720, 730),
- ]
-
- def __init__(self, filename, node=None):
- """
- Create a new RRD for a given node.
-
- If the RRD isn't supposed to be updated, the node can be omitted.
- """
- self.node = node
- super().__init__(filename)
- self.ensure_sanity(self.ds_list, self.rra_list, step=60)
-
- @property
- def imagename(self):
- return "{basename}.png".format(
- basename=os.path.basename(self.filename).rsplit('.', 2)[0])
-
- # TODO: fix this, python does not support function overloading
- def update(self):
- super().update({'upstate': int(self.node['flags']['online']),
- 'clients': self.node['statistics']['clients']})
-
- def graph(self, directory, timeframe):
- """
- Create a graph in the given directory. The file will be named
- basename.png if the RRD file is named basename.rrd
- """
- args = ['rrdtool', 'graph', os.path.join(directory, self.imagename),
- '-s', '-' + timeframe,
- '-w', '800',
- '-h', '400',
- '-l', '0',
- '-y', '1:1',
- 'DEF:clients=' + self.filename + ':clients:AVERAGE',
- 'VDEF:maxc=clients,MAXIMUM',
- 'CDEF:c=0,clients,ADDNAN',
- 'CDEF:d=clients,UN,maxc,UN,1,maxc,IF,*',
- 'AREA:c#0F0:up\\l',
- 'AREA:d#F00:down\\l',
- 'LINE1:c#00F:clients connected\\l']
- subprocess.check_output(args)
diff --git a/lib/RRD.py b/lib/RRD.py
deleted file mode 100644
index 4e925a7..0000000
--- a/lib/RRD.py
+++ /dev/null
@@ -1,346 +0,0 @@
-import subprocess
-import re
-import os
-from operator import xor, eq
-from functools import reduce
-from itertools import starmap
-import math
-
-
-class RRDIncompatibleException(Exception):
- """
- Is raised when an RRD doesn't have the desired definition and cannot be
- upgraded to it.
- """
- pass
-
-
-class RRDOutdatedException(Exception):
- """
- Is raised when an RRD doesn't have the desired definition, but can be
- upgraded to it.
- """
- pass
-
-if not hasattr(__builtins__, "FileNotFoundError"):
- class FileNotFoundError(Exception):
- pass
-
-
-class RRD(object):
- """
- An RRD is a Round Robin Database, a database which forgets old data and
- aggregates multiple records into new ones.
-
- It contains multiple Data Sources (DS) which can be thought of as columns,
- and Round Robin Archives (RRA) which can be thought of as tables with the
- DS as columns and time-dependant rows.
- """
-
- # rra[2].cdp_prep[0].value = 1,8583033333e+03
- _info_regex = re.compile("""
- (?P<section>[a-z_]+)
- \[ (?P<key>[a-zA-Z0-9_]+) \]
- \.
- |
- (?P<name>[a-z_]+)
- \s*=\s*
- "? (?P<value>.*?) "?
- $""", re.X)
- _cached_info = None
-
- def _exec_rrdtool(self, cmd, *args, **kwargs):
- pargs = ["rrdtool", cmd, self.filename]
- for k, v in kwargs.items():
- pargs.extend(["--" + k, str(v)])
- pargs.extend(args)
- subprocess.check_output(pargs)
-
- def __init__(self, filename):
- self.filename = filename
-
- def ensure_sanity(self, ds_list, rra_list, **kwargs):
- """
- Create or upgrade the RRD file if necessary to contain all DS in
- ds_list. If it needs to be created, the RRAs in rra_list and any kwargs
- will be used for creation. Note that RRAs and options of an existing
- database are NOT modified!
- """
- try:
- self.check_sanity(ds_list)
- except FileNotFoundError:
- self.create(ds_list, rra_list, **kwargs)
- except RRDOutdatedException:
- self.upgrade(ds_list)
-
- def check_sanity(self, ds_list=()):
- """
- Check if the RRD file exists and contains (at least) the DS listed in
- ds_list.
- """
- if not os.path.exists(self.filename):
- raise FileNotFoundError(self.filename)
- info = self.info()
- if set(ds_list) - set(info['ds'].values()) != set():
- for ds in ds_list:
- if ds.name in info['ds'] and\
- ds.type != info['ds'][ds.name].type:
- raise RRDIncompatibleException(
- "{} is {} but should be {}".format(
- ds.name, ds.type, info['ds'][ds.name].type))
- else:
- raise RRDOutdatedException()
-
- def upgrade(self, dss):
- """
- Upgrade the DS definitions (!) of this RRD.
- (To update its values, use update())
-
- The list dss contains DSS objects to be updated or added. The
- parameters of a DS can be changed, but not its type. New DS are always
- added at the end in the order of their appearance in the list.
-
- This is done internally via an rrdtool dump -> rrdtool restore and
- modifying the dump on the fly.
- """
- info = self.info()
- new_ds = list(info['ds'].values())
- new_ds.sort(key=lambda ds: ds.index)
- for ds in dss:
- if ds.name in info['ds']:
- old_ds = info['ds'][ds.name]
- if info['ds'][ds.name].type != ds.type:
- raise RuntimeError(
- "Cannot convert existing DS '{}'"
- "from type '{}' to '{}'".format(
- ds.name, old_ds.type, ds.type))
- ds.index = old_ds.index
- new_ds[ds.index] = ds
- else:
- ds.index = len(new_ds)
- new_ds.append(ds)
- added_ds_num = len(new_ds) - len(info['ds'])
-
- dump = subprocess.Popen(
- ["rrdtool", "dump", self.filename],
- stdout=subprocess.PIPE)
-
- restore = subprocess.Popen(
- ["rrdtool", "restore", "-", self.filename + ".new"],
- stdin=subprocess.PIPE)
- echo = True
- ds_definitions = True
- for line in dump.stdout:
- if ds_definitions and b'<ds>' in line:
- echo = False
- if b'<!-- Round Robin Archives -->' in line:
- ds_definitions = False
- for ds in new_ds:
- restore.stdin.write(bytes("""
- <ds>
- <name> %s </name>
- <type> %s </type>
- <minimal_heartbeat>%i</minimal_heartbeat>
- <min>%s</min>
- <max>%s</max>
-
- <!-- PDP Status -->
- <last_ds>%s</last_ds>
- <value>%s</value>
- <unknown_sec> %i </unknown_sec>
- </ds>
- """ % (ds.name,
- ds.type,
- ds.args[0],
- ds.args[1],
- ds.args[2],
- ds.last_ds,
- ds.value,
- ds.unknown_sec), "utf-8"))
-
- if b'</cdp_prep>' in line:
- restore.stdin.write(added_ds_num * b"""
- <ds>
- <primary_value> NaN </primary_value>
- <secondary_value> NaN </secondary_value>
- <value> NaN </value>
- <unknown_datapoints> 0 </unknown_datapoints>
- </ds>
- """)
-
- # echoing of input line
- if echo:
- restore.stdin.write(
- line.replace(
- b'</row>',
- (added_ds_num * b'<v>NaN</v>') + b'</row>'
- )
- )
-
- if ds_definitions and b'</ds>' in line:
- echo = True
- dump.stdout.close()
- restore.stdin.close()
- dump.wait()
- restore.wait()
-
- os.rename(self.filename + ".new", self.filename)
- self._cached_info = None
-
- def create(self, ds_list, rra_list, **kwargs):
- """
- Create a new RRD file with the specified list of RRAs and DSs.
-
- Any kwargs are passed as --key=value to rrdtool create.
- """
- self._exec_rrdtool(
- "create",
- *map(str, rra_list + ds_list),
- **kwargs
- )
- self._cached_info = None
-
- def update(self, V):
- """
- Update the RRD with new values V.
-
- V can be either list or dict:
- * If it is a dict, its keys must be DS names in the RRD and it is
- ensured that the correct DS are updated with the correct values, by
- passing a "template" to rrdtool update (see man rrdupdate).
- * If it is a list, no template is generated and the order of the
- values in V must be the same as that of the DS in the RRD.
- """
- try:
- args = ['N:' + ':'.join(map(str, V.values()))]
- kwargs = {'template': ':'.join(V.keys())}
- except AttributeError:
- args = ['N:' + ':'.join(map(str, V))]
- kwargs = {}
- self._exec_rrdtool("update", *args, **kwargs)
- self._cached_info = None
-
- def info(self):
- """
- Return a dictionary with information about the RRD.
-
- See `man rrdinfo` for more details.
- """
- if self._cached_info:
- return self._cached_info
- env = os.environ.copy()
- env["LC_ALL"] = "C"
- proc = subprocess.Popen(
- ["rrdtool", "info", self.filename],
- stdout=subprocess.PIPE,
- env=env
- )
- out, err = proc.communicate()
- out = out.decode()
- info = {}
- for line in out.splitlines():
- base = info
- for match in self._info_regex.finditer(line):
- section, key, name, value = match.group(
- "section", "key", "name", "value")
- if section and key:
- try:
- key = int(key)
- except ValueError:
- pass
- if section not in base:
- base[section] = {}
- if key not in base[section]:
- base[section][key] = {}
- base = base[section][key]
- if name and value:
- try:
- base[name] = int(value)
- except ValueError:
- try:
- base[name] = float(value)
- except:
- base[name] = value
- dss = {}
- for name, ds in info['ds'].items():
- ds_obj = DS(name, ds['type'], ds['minimal_heartbeat'],
- ds['min'], ds['max'])
- ds_obj.index = ds['index']
- ds_obj.last_ds = ds['last_ds']
- ds_obj.value = ds['value']
- ds_obj.unknown_sec = ds['unknown_sec']
- dss[name] = ds_obj
- info['ds'] = dss
- rras = []
- for rra in info['rra'].values():
- rras.append(RRA(rra['cf'], rra['xff'],
- rra['pdp_per_row'], rra['rows']))
- info['rra'] = rras
- self._cached_info = info
- return info
-
-
-class DS(object):
- """
- DS stands for Data Source and represents one line of data points in a Round
- Robin Database (RRD).
- """
- name = None
- type = None
- args = []
- index = -1
- last_ds = 'U'
- value = 0
- unknown_sec = 0
-
- def __init__(self, name, dst, *args):
- self.name = name
- self.type = dst
- self.args = args
-
- def __str__(self):
- return "DS:%s:%s:%s" % (
- self.name,
- self.type,
- ":".join(map(str, self._nan_to_u_args()))
- )
-
- def __repr__(self):
- return "%s(%r, %r, %s)" % (
- self.__class__.__name__,
- self.name,
- self.type,
- ", ".join(map(repr, self.args))
- )
-
- def __eq__(self, other):
- return all(starmap(eq, zip(self.compare_keys(), other.compare_keys())))
-
- def __hash__(self):
- return reduce(xor, map(hash, self.compare_keys()))
-
- def _nan_to_u_args(self):
- return tuple(
- 'U' if type(arg) is float and math.isnan(arg)
- else arg
- for arg in self.args
- )
-
- def compare_keys(self):
- return self.name, self.type, self._nan_to_u_args()
-
-
-class RRA(object):
- def __init__(self, cf, *args):
- self.cf = cf
- self.args = args
-
- def __str__(self):
- return "RRA:%s:%s" % (self.cf, ":".join(map(str, self.args)))
-
- def __repr__(self):
- return "%s(%r, %s)" % (
- self.__class__.__name__,
- self.cf,
- ", ".join(map(repr, self.args))
- )
diff --git a/lib/rrddb.py b/lib/rrddb.py
deleted file mode 100644
index f1678f5..0000000
--- a/lib/rrddb.py
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/usr/bin/env python3
-import time
-import os
-
-from lib.GlobalRRD import GlobalRRD
-from lib.NodeRRD import NodeRRD
-
-
-class RRD(object):
- def __init__(self,
- database_directory,
- image_path,
- display_time_global="7d",
- display_time_node="1d"):
-
- self.dbPath = database_directory
- self.globalDb = GlobalRRD(self.dbPath)
- self.imagePath = image_path
- self.displayTimeGlobal = display_time_global
- self.displayTimeNode = display_time_node
-
- self.currentTimeInt = (int(time.time()) / 60) * 60
- self.currentTime = str(self.currentTimeInt)
-
- try:
- os.stat(self.imagePath)
- except OSError:
- os.mkdir(self.imagePath)
-
- def update_database(self, nodes):
- online_nodes = dict(filter(
- lambda d: d[1]['flags']['online'], nodes.items()))
- client_count = sum(map(
- lambda d: d['statistics']['clients'], online_nodes.values()))
-
- self.globalDb.update(len(online_nodes), client_count)
- for node_id, node in online_nodes.items():
- rrd = NodeRRD(os.path.join(self.dbPath, node_id + '.rrd'), node)
- rrd.update()
-
- def update_images(self):
- self.globalDb.graph(os.path.join(self.imagePath, "globalGraph.png"),
- self.displayTimeGlobal)
-
- nodedb_files = os.listdir(self.dbPath)
-
- for file_name in nodedb_files:
- if not os.path.isfile(os.path.join(self.dbPath, file_name)):
- continue
-
- node_name = os.path.basename(file_name).split('.')
- if node_name[1] == 'rrd' and not node_name[0] == "nodes":
- rrd = NodeRRD(os.path.join(self.dbPath, file_name))
- rrd.graph(self.imagePath, self.displayTimeNode)