From f617f3b2be54849f173a136fa96ed680a3fb5667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20W=C3=BCllhorst?= Date: Tue, 19 Jul 2016 23:28:01 +0200 Subject: [PATCH] Added info module for generating info files about nodes, links, graphs and domains. Currently there is offline node info available only. Furthermore there is a attribute to append filters to info module --- NodeHierarchy.py | 12 +++++--- info/Info.py | 18 +++++++++++ info/InfoMeta.py | 75 +++++++++++++++++++++++++++++++++++++++++++++ info/OfflineInfo.py | 68 ++++++++++++++++++++++++++++++++++++++++ info/__init__.py | 0 5 files changed, 169 insertions(+), 4 deletions(-) create mode 100644 info/Info.py create mode 100644 info/InfoMeta.py create mode 100644 info/OfflineInfo.py create mode 100644 info/__init__.py diff --git a/NodeHierarchy.py b/NodeHierarchy.py index 9d1c433..27b1331 100755 --- a/NodeHierarchy.py +++ b/NodeHierarchy.py @@ -8,6 +8,7 @@ from cloud.GlobalGraph import GlobalGraph from parser.ShapesParser import ShapesParser from cloud.Domaene import Domaene from generator.NginxConfGen import NginxConfGen +from info.Info import Info class NodeHierarchy(object): def __init__(self): @@ -21,6 +22,7 @@ class NodeHierarchy(object): self.domains = self.__createDomains__() self.nginxConf = NginxConfGen(self.domains, self.__args__) self.nginxConf.writeNginxConfigFile() + self.infos = Info(self.__args__.info, self.__args__.info_out_path, self.__args__.info_out_type, self.__args__.info_filters, self.nodes, self.globalGraph, self.domains) def __parseShapes__(self): shapesJson = {} @@ -69,10 +71,12 @@ class NodeHierarchy(object): parser.add_argument('-j', '--json-path', required=False, default='https://service.freifunk-muensterland.de/maps/data/', help='Path of nodes.json and graph.json (can be local folder or remote URL).') parser.add_argument('-s', '--shapes-path', required=False, default='https://freifunk-muensterland.de/md-fw-dl/shapes/', help='Path of shapefiles (can be local folder or remote URL).') parser.add_argument('-t', '--targets', nargs='+', required=True, help='List of targets which should be proceeded. Example: -t citya cityb ...') - parser.add_argument('-o', '--out-file', required=False, help='Filename where the generated Output should stored.', default='./webserver-configuration') + parser.add_argument('-o', '--out-file', default='./webserver-configuration', required=False, help='Filename where the generated Output should stored.') parser.add_argument('-v', '--debug', required=False, action='store_true', help='Enable debugging output.') - parser.add_argument('-f', '--filters', nargs='*', required=False, choices=('exclude_clouds_with_lan_links', 'no_lan'), help='Filter out nodes and local clouds based on filter rules') - + parser.add_argument('-f', '--filters', nargs='*', required=False, choices=('exclude_clouds_with_lan_links', 'no_lan'), help='Filter out nodes and local clouds based on filter rules.') + parser.add_argument('-i', '--info', nargs='*', required=False, choices=('get_offline_nodes','offline'), help='Get infos about the graph, links and nodes.') + parser.add_argument('-if', '--info-filters', nargs='*', required=False, help='Filter info results. Currently supported: min_age:TIME_RANGE, max_age:TIME_RANGE. Examples: -if min_age:1d max_age:2w') + parser.add_argument('-iop', '--info-out-path', required=False, default='./', help='Folder where info files should be written. Default: ./') + parser.add_argument('-iot', '--info-out-type', nargs='+', required=False, default='csv', choices=('json', 'csv'), help='Defines the format of info output. Default: csv') return parser.parse_args() - NodeHierarchy() diff --git a/info/Info.py b/info/Info.py new file mode 100644 index 0000000..3aa23c6 --- /dev/null +++ b/info/Info.py @@ -0,0 +1,18 @@ +from info.OfflineInfo import OfflineInfo +class Info(object): + def __init__(self, infoTypes, infoOutFolder, infoOutType, infoFilters, nodes, globalGraph, domains): + self.__infoTypes__ = infoTypes + self.__infoOutFolder__ = infoOutFolder + self.__infoOutType__ = infoOutType + self.__infoFilters__ = infoFilters + self.__nodes__ = nodes + self.__globalGraph__ = globalGraph + self.__domains__ = domains + if self.__infoTypes__ != None: + if 'get_offline_nodes' in self.__infoTypes__: + self.__offlineNodes__ = OfflineInfo(self.__infoFilters__, self.__nodes__, self.__domains__) + print(self.__infoOutType__) + if 'csv' in self.__infoOutType__: + self.__offlineNodes__.writeCSVtoFile(self.__infoOutFolder__+'/offline_nodes.csv') + if 'json' in self.__infoOutType__: + self.__offlineNodes__.writeJsonToFile(self.__infoOutFolder__+'/offline_nodes.json') \ No newline at end of file diff --git a/info/InfoMeta.py b/info/InfoMeta.py new file mode 100644 index 0000000..7cb13f8 --- /dev/null +++ b/info/InfoMeta.py @@ -0,0 +1,75 @@ +import json +class InfoMeta(object): + def __init__(self): + self.resultNodes = None + self.resultGraph = None + + + def __generateNodesJson__(self): + if self.resultNodes == None: + return [] + result = [] + for node in self.resultNodes: + result.append(node.__jsonObject__) + return result + + def __generateNodesCSV__(self): + if self.resultNodes == None: + return '' + + result = '"hostname","site","nodeid","ipv6addresses","status","lastseen","firstseen","autoupdater","branch","firmware","hardware"\n' + for node in self.resultNodes: + nodeData = node.__jsonObject__ + nodeinfo = nodeData['nodeinfo'] + result += '"'+nodeinfo['hostname']+'",' + + try: + result +='"'+nodeinfo['system']['site_code']+'",' + except: + result += '"none",' + result += '"'+nodeinfo['node_id']+'","' + + #add array of public IPv6 Addresses + addresses = node.__getPublicAddresses__() + for i, address in enumerate(addresses): + if i == len(addresses)-1: + result += address + else: + result += address + ',' + result += '",' + + if nodeData['flags']['online'] == True: + result += '"online",' + else: + result += '"offline",' + + result += '"'+nodeData['lastseen']+'","'+nodeData['firstseen']+'",' + try: + if nodeinfo['software']['autoupdater']['enabled'] == True: + result += '"enabled",' + else: + result += '"disabled",' + except: + result += '"none",' + + try: + result += '"'+nodeinfo['software']['autoupdater']['branch']+'",' + except: + result += '"none",' + result += '"'+nodeinfo['software']['firmware']['release']+'",' + + try: + result += '"'+nodeinfo['hardware']['model']+'"' + except: + result += '"none"' + + result += '\n' + return result + + def writeCSVtoFile(self, filename): + with open(filename, 'w') as out: + out.write(self.__generateNodesCSV__()) + + def writeJsonToFile(self, filename): + with open(filename, 'w') as out: + out.write(json.dumps(self.__generateNodesJson__(), sort_keys=True, indent=4, ensure_ascii=False)) \ No newline at end of file diff --git a/info/OfflineInfo.py b/info/OfflineInfo.py new file mode 100644 index 0000000..2664461 --- /dev/null +++ b/info/OfflineInfo.py @@ -0,0 +1,68 @@ +from info.InfoMeta import InfoMeta +from shapely.geometry import Point +import datetime, re +from datetime import timedelta +class OfflineInfo(InfoMeta): + def __init__(self, filters, nodes, domains): + super().__init__() + self.__filters__ = filters + self.__nodes__ = nodes + self.__domains__ = domains + self.__minAge__, self.__maxAge__ = self.__parseFilters__() + self.resultNodes = self.__filterNodes__() + + def __filterNodes__(self): + + offlineNodes = [] + + for k,v in self.__nodes__.items(): + if v.isOnline == False: + if v.geo != None: + for dk, dv in self.__domains__.items(): + if dv.isPointInDomaene(Point((v.geo['lon'], v.geo['lat']))) == True: + nodeLastSeen = datetime.datetime.strptime(v.__jsonObject__['lastseen'],'%Y-%m-%dT%H:%M:%S') + if self.__minAge__ != None: + if self.__minAge__ < nodeLastSeen: + continue + if self.__maxAge__ != None: + if self.__maxAge__ > nodeLastSeen: + continue + offlineNodes.append(v) + + return offlineNodes + + + def __parseFilters__(self): + + if self.__filters__ == None: + return None, None + + regX = re.compile("([0-9]+)([a-zA-Z]+)") + minAge = None + maxAge = None + + for filter in self.__filters__: + attr = filter.split(':') + if len(attr) == 2: + if attr[0] == 'min_age' or attr[0] == 'max_age': + d = regX.match(attr[1]) + if d != None: + val = int(d.group(1)) + unit = d.group(2) + date = datetime.datetime.now() + if unit == 'd' or unit == 'day' or unit == 'days': + date = date - timedelta(days=val) + elif unit == 'w' or unit == 'week' or unit == 'weeks': + date = date - timedelta(days=val*7) + elif unit == 'm' or unit == 'month' or unit == 'months': + date = date - timedelta(days=val*30) + elif unit == 'y' or unit == 'year' or unit == 'years': + date = date - timedelta(days=val*365) + else: + date = None + + if attr[0] == 'min_age': + minAge = date + elif attr[0] == 'max_age': + maxAge = date + return minAge, maxAge \ No newline at end of file diff --git a/info/__init__.py b/info/__init__.py new file mode 100644 index 0000000..e69de29