From 9a85fd0c971ed196ada8ece2d6df29037e2036b0 Mon Sep 17 00:00:00 2001 From: Daniel Ehlers Date: Mon, 18 Nov 2013 21:35:40 +0100 Subject: [PATCH 1/6] Fix utf-8 enforcing hack. The function getdefaultlocale takes one optional argument, so the replace should behave the same. --- bat2nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bat2nodes.py b/bat2nodes.py index ab4d5fd..917136a 100755 --- a/bat2nodes.py +++ b/bat2nodes.py @@ -9,7 +9,7 @@ from d3mapbuilder import D3MapBuilder # Force encoding to UTF-8 import locale # Ensures that subsequent open()s -locale.getpreferredencoding = lambda: 'UTF-8' # are UTF-8 encoded. +locale.getpreferredencoding = lambda _=None: 'UTF-8' # are UTF-8 encoded. import sys #sys.stdin = open('/dev/stdin', 'r') From 1b0e830e162c5e99b277ba0450d9fb278bdf870f Mon Sep 17 00:00:00 2001 From: Daniel Ehlers Date: Mon, 18 Nov 2013 10:58:12 +0100 Subject: [PATCH 2/6] Add destination directory argument and write content instead of piping. --- bat2nodes.py | 7 ++++++- mkmap.sh | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/bat2nodes.py b/bat2nodes.py index 917136a..a3d3fb1 100755 --- a/bat2nodes.py +++ b/bat2nodes.py @@ -28,6 +28,9 @@ parser.add_argument('-g', '--gateway', action='append', parser.add_argument('batmanjson', help='output of batman vd json') +parser.add_argument('-d', '--destination-directory', action='store', + help='destination directory for generated files',required=True) + args = parser.parse_args() options = vars(args) @@ -44,4 +47,6 @@ if options['gateway']: m = D3MapBuilder(db) -print(m.build()) +nodes_json = open(options['destination_directory'] + '/nodes.json.new','w') +nodes_json.write(m.build()) +nodes_json.close() diff --git a/mkmap.sh b/mkmap.sh index 13ec9ec..064d1b9 100755 --- a/mkmap.sh +++ b/mkmap.sh @@ -20,7 +20,7 @@ fi "$(dirname "$0")"/ffhlwiki.py http://freifunk.metameute.de/wiki/Knoten > "$(dirname "$0")"/aliases_hl.json "$(dirname "$0")"/ffhlwiki.py http://freifunk.metameute.de/wiki/Moelln:Knoten > "$(dirname "$0")"/aliases_moelln.json -batctl vd json -n | "$(dirname "$0")"/bat2nodes.py -a "$(dirname "$0")"/aliases.json -a aliases_hl.json -a aliases_moelln.json $GWS - > $DEST/nodes.json.new +batctl vd json -n | "$(dirname "$0")"/bat2nodes.py -a "$(dirname "$0")"/aliases.json -a aliases_hl.json -a aliases_moelln.json -d $DEST $GWS - mv $DEST/nodes.json.new $DEST/nodes.json From 62413d81f37e65fd13426d5ea5011ff03ad37654 Mon Sep 17 00:00:00 2001 From: Daniel Ehlers Date: Mon, 18 Nov 2013 10:59:49 +0100 Subject: [PATCH 3/6] Generate node stats with rrd. We have an rrd for every active node. On every run we update stats about client count and node state from nodes.json to the rrd databases. After updating the rrds we are able to generate a stat graph (image) for every node. The rrds are stored in subdirectory "nodedb" of the scripts location and the generated images in subdirectory "nodes" of the destination directory. Both directorys must exist bevor starting the script. --- bat2nodes.py | 8 +++ rrd.py | 135 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100755 rrd.py diff --git a/bat2nodes.py b/bat2nodes.py index a3d3fb1..081eacf 100755 --- a/bat2nodes.py +++ b/bat2nodes.py @@ -3,7 +3,9 @@ import json import fileinput import argparse +import os +from rrd import rrd from nodedb import NodeDB from d3mapbuilder import D3MapBuilder @@ -45,6 +47,12 @@ if options['aliases']: if options['gateway']: db.mark_gateways(options['gateway']) +scriptdir = os.path.dirname(os.path.realpath(__file__)) + +rrd = rrd(scriptdir + "/nodedb/",options['destination_directory'] + "/nodes") +rrd.update_database(db) +rrd.update_images() + m = D3MapBuilder(db) nodes_json = open(options['destination_directory'] + '/nodes.json.new','w') diff --git a/rrd.py b/rrd.py new file mode 100755 index 0000000..dd4d7f3 --- /dev/null +++ b/rrd.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +import subprocess +import time +import os + +class rrd: + def __init__( self + , databaseDirectory + , imagePath + , displayTimeGlobal = "7d" + , displayTimeNode = "1d" + ): + self.dbPath = databaseDirectory + self.globalDbFile = databaseDirectory + "/nodes.rrd" + self.imagePath = imagePath + self.displayTimeGlobal = displayTimeGlobal + self.displayTimeNode = displayTimeNode + + self.currentTimeInt = (int(time.time())/60)*60 + self.currentTime = str(self.currentTimeInt) + + def checkAndCreateIfNeededGlobalDatabase(self): + """ Creates the global database file iff it did not exist. + """ + if not os.path.exists(self.globalDbFile): + # Create Database with rrdtool + args = ["rrdtool",'create', self.globalDbFile + ,'--start', str(round(self.currentTimeInt - 60)) + ,'--step' , '60' + # Number of nodes available + ,'DS:nodes:GAUGE:120:0:U' + ,'RRA:LAST:0:1:44640' + ,'RRA:LAST:0:60:744' + ,'RRA:LAST:0:1440:1780' + # Number of client available + ,'DS:clients:GAUGE:120:0:U' + ,'RRA:LAST:0:1:44640' + ,'RRA:LAST:0:60:744' + ,'RRA:LAST:0:1440:1780' + ] + subprocess.call(args) + + def updateGlobalDatabase(self,nodeCount,clientCount): + """ Adds a new (#Nodes,#Clients) entry to the global database. + """ + # Update Global RRDatabase + args = ["rrdtool",'updatev', self.globalDbFile + # #Nodes #Clients + , self.currentTime + ":"+str(nodeCount)+":"+str(clientCount) + ] + subprocess.check_output(args) + + def createGlobalGraph(self): + nodeGraph = self.imagePath + "/" + "globalGraph.png" + args = ["rrdtool", 'graph', nodeGraph, '-s', '-' + self.displayTimeGlobal, '-w', '800', '-h' '400' + ,'DEF:nodes=' + self.globalDbFile + ':nodes:LAST', 'LINE1:nodes#F00:nodes\\l' + ,'DEF:clients=' + self.globalDbFile + ':clients:LAST','LINE2:clients#00F:clients' + ] + subprocess.check_output(args) + + + def nodeMACToRRDFile(self,nodeMAC): + return self.dbPath + "/" + str(nodeMAC).replace(":","") + ".rrd" + + def nodeMACToPNGFile(self,nodeMAC): + return self.imagePath + "/" + str(nodeMAC).replace(":","") + ".png" + + def checkAndCreateIfNeededNodeDatabase(self,nodePrimaryMAC): + # TODO check for bad nodeNames + nodeFile = self.nodeMACToRRDFile(nodePrimaryMAC); + if not os.path.exists(nodeFile): + # TODO Skalen anpassen + args = ["rrdtool",'create',nodeFile + ,'--start',str(round(self.currentTimeInt - 60)) + ,'--step' , '60' + ,'DS:upstate:GAUGE:120:0:1' + ,'RRA:LAST:0:1:44640' + # Number of client available + ,'DS:clients:GAUGE:120:0:U' + ,'RRA:LAST:0:1:44640' + ] + subprocess.check_output(args) + + # Call only if node is up + def updateNodeDatabase(self,nodePrimaryMAC,clientCount): + nodeFile = self.nodeMACToRRDFile(nodePrimaryMAC) + # Update Global RRDatabase + args = ["rrdtool",'updatev', nodeFile + # #Upstate #Clients + , self.currentTime + ":"+str(1)+":"+str(clientCount) + ] + subprocess.check_output(args) + + def createNodeGraph(self,nodePrimaryMAC,displayTimeNode): + nodeGraph = self.nodeMACToPNGFile(nodePrimaryMAC + '_upstate') + nodeFile = self.nodeMACToRRDFile(nodePrimaryMAC) + args = ['rrdtool', 'graph', nodeGraph, '-s', '-' + self.displayTimeNode , '-w', '800', '-h' '400' + ,'DEF:upstate=' + nodeFile + ':upstate:LAST', 'LINE1:upstate#F00:upstate\\l' + ] + subprocess.check_output(args) + + def update_database(self,db): + nodes = {} + clientCount = 0 + for node in db.get_nodes(): + if node.flags['online']: + if not node.flags['client']: + nodes[node.id] = node + node.clients = 0; + else: + clientCount += 1 + for link in db.get_links(): + if link.source in nodes and not link.target in nodes: + nodes[link.source].clients += 1 + elif link.target in nodes and not link.source in nodes: + nodes[link.source].clients += 1 + + self.checkAndCreateIfNeededGlobalDatabase() + self.updateGlobalDatabase(len(nodes),clientCount) + for mac in nodes: + self.checkAndCreateIfNeededNodeDatabase(mac) + self.updateNodeDatabase(mac,nodes[mac].clients) + + def update_images(self): + """ Creates a image for every rrd file in the database directory. + """ + + self.createGlobalGraph() + + nodeDbFiles = os.listdir(self.dbPath) + + for fileName in nodeDbFiles: + nodeName = os.path.basename(fileName).split('.') + if nodeName[1] == 'rrd' and not nodeName[0] == "nodes": + self.createNodeGraph(nodeName[0],self.displayTimeNode) From b87802867163d677b7c43fba502ad8a985590fde Mon Sep 17 00:00:00 2001 From: Daniel Ehlers Date: Mon, 18 Nov 2013 22:39:44 +0100 Subject: [PATCH 4/6] Fix link handling. Accidental the LinkConnector object was used for checking the nodes, this patch fixes this behaviour. --- rrd.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/rrd.py b/rrd.py index dd4d7f3..ac3c4e7 100755 --- a/rrd.py +++ b/rrd.py @@ -110,10 +110,12 @@ class rrd: else: clientCount += 1 for link in db.get_links(): - if link.source in nodes and not link.target in nodes: - nodes[link.source].clients += 1 - elif link.target in nodes and not link.source in nodes: - nodes[link.source].clients += 1 + source = link.source.interface + target = link.target.interface + if source in nodes and not target in nodes: + nodes[source].clients += 1 + elif target in nodes and not source in nodes: + nodes[target].clients += 1 self.checkAndCreateIfNeededGlobalDatabase() self.updateGlobalDatabase(len(nodes),clientCount) From 94dba3399c08943297ff831dfb3b743f274e2728 Mon Sep 17 00:00:00 2001 From: Wilfried Klaebe Date: Mon, 18 Nov 2013 10:42:32 +0100 Subject: [PATCH 5/6] Beautify node stats. Also display both rrdb fields in one graph. --- rrd.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/rrd.py b/rrd.py index ac3c4e7..dae7b4f 100755 --- a/rrd.py +++ b/rrd.py @@ -92,11 +92,17 @@ class rrd: subprocess.check_output(args) def createNodeGraph(self,nodePrimaryMAC,displayTimeNode): - nodeGraph = self.nodeMACToPNGFile(nodePrimaryMAC + '_upstate') + nodeGraph = self.nodeMACToPNGFile(nodePrimaryMAC) nodeFile = self.nodeMACToRRDFile(nodePrimaryMAC) - args = ['rrdtool', 'graph', nodeGraph, '-s', '-' + self.displayTimeNode , '-w', '800', '-h' '400' - ,'DEF:upstate=' + nodeFile + ':upstate:LAST', 'LINE1:upstate#F00:upstate\\l' - ] + args = ['rrdtool','graph', nodeGraph, '-s', '-' + self.displayTimeNode , '-w', '800', '-h', '400', '-l', '0', '-y', '1:1', + 'DEF:clients=' + nodeFile + ':clients:LAST', + 'VDEF:maxc=clients,MAXIMUM', + 'CDEF:c=0,clients,ADDNAN', + 'CDEF:d=clients,UN,maxc,UN,1,maxc,IF,*', + 'AREA:c#0F0:up\\l', + 'AREA:d#F00:down\\l', + 'LINE1:c#00F:clients connected\\l', + ] subprocess.check_output(args) def update_database(self,db): From ca3c436813b0ee894922bf4c8bc2b66c166a2c8d Mon Sep 17 00:00:00 2001 From: Daniel Ehlers Date: Mon, 18 Nov 2013 23:41:27 +0100 Subject: [PATCH 6/6] Reduce clientCounter for every node. We have an ghost client for every node, we don't want them in the global client measurement. --- rrd.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rrd.py b/rrd.py index dae7b4f..ddc25b5 100755 --- a/rrd.py +++ b/rrd.py @@ -113,6 +113,7 @@ class rrd: if not node.flags['client']: nodes[node.id] = node node.clients = 0; + clientCount -= 1 else: clientCount += 1 for link in db.get_links():