Added info module for generating info files about nodes, links, graphs and domains. Currently there is offline node info available only. Furthermore there is a attribute to append filters to info module
This commit is contained in:
parent
779087bb99
commit
f617f3b2be
|
@ -8,6 +8,7 @@ from cloud.GlobalGraph import GlobalGraph
|
||||||
from parser.ShapesParser import ShapesParser
|
from parser.ShapesParser import ShapesParser
|
||||||
from cloud.Domaene import Domaene
|
from cloud.Domaene import Domaene
|
||||||
from generator.NginxConfGen import NginxConfGen
|
from generator.NginxConfGen import NginxConfGen
|
||||||
|
from info.Info import Info
|
||||||
|
|
||||||
class NodeHierarchy(object):
|
class NodeHierarchy(object):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
|
@ -21,6 +22,7 @@ class NodeHierarchy(object):
|
||||||
self.domains = self.__createDomains__()
|
self.domains = self.__createDomains__()
|
||||||
self.nginxConf = NginxConfGen(self.domains, self.__args__)
|
self.nginxConf = NginxConfGen(self.domains, self.__args__)
|
||||||
self.nginxConf.writeNginxConfigFile()
|
self.nginxConf.writeNginxConfigFile()
|
||||||
|
self.infos = Info(self.__args__.info, self.__args__.info_out_path, self.__args__.info_out_type, self.__args__.info_filters, self.nodes, self.globalGraph, self.domains)
|
||||||
|
|
||||||
def __parseShapes__(self):
|
def __parseShapes__(self):
|
||||||
shapesJson = {}
|
shapesJson = {}
|
||||||
|
@ -69,10 +71,12 @@ class NodeHierarchy(object):
|
||||||
parser.add_argument('-j', '--json-path', required=False, default='https://service.freifunk-muensterland.de/maps/data/', help='Path of nodes.json and graph.json (can be local folder or remote URL).')
|
parser.add_argument('-j', '--json-path', required=False, default='https://service.freifunk-muensterland.de/maps/data/', help='Path of nodes.json and graph.json (can be local folder or remote URL).')
|
||||||
parser.add_argument('-s', '--shapes-path', required=False, default='https://freifunk-muensterland.de/md-fw-dl/shapes/', help='Path of shapefiles (can be local folder or remote URL).')
|
parser.add_argument('-s', '--shapes-path', required=False, default='https://freifunk-muensterland.de/md-fw-dl/shapes/', help='Path of shapefiles (can be local folder or remote URL).')
|
||||||
parser.add_argument('-t', '--targets', nargs='+', required=True, help='List of targets which should be proceeded. Example: -t citya cityb ...')
|
parser.add_argument('-t', '--targets', nargs='+', required=True, help='List of targets which should be proceeded. Example: -t citya cityb ...')
|
||||||
parser.add_argument('-o', '--out-file', required=False, help='Filename where the generated Output should stored.', default='./webserver-configuration')
|
parser.add_argument('-o', '--out-file', default='./webserver-configuration', required=False, help='Filename where the generated Output should stored.')
|
||||||
parser.add_argument('-v', '--debug', required=False, action='store_true', help='Enable debugging output.')
|
parser.add_argument('-v', '--debug', required=False, action='store_true', help='Enable debugging output.')
|
||||||
parser.add_argument('-f', '--filters', nargs='*', required=False, choices=('exclude_clouds_with_lan_links', 'no_lan'), help='Filter out nodes and local clouds based on filter rules')
|
parser.add_argument('-f', '--filters', nargs='*', required=False, choices=('exclude_clouds_with_lan_links', 'no_lan'), help='Filter out nodes and local clouds based on filter rules.')
|
||||||
|
parser.add_argument('-i', '--info', nargs='*', required=False, choices=('get_offline_nodes','offline'), help='Get infos about the graph, links and nodes.')
|
||||||
|
parser.add_argument('-if', '--info-filters', nargs='*', required=False, help='Filter info results. Currently supported: min_age:TIME_RANGE, max_age:TIME_RANGE. Examples: -if min_age:1d max_age:2w')
|
||||||
|
parser.add_argument('-iop', '--info-out-path', required=False, default='./', help='Folder where info files should be written. Default: ./')
|
||||||
|
parser.add_argument('-iot', '--info-out-type', nargs='+', required=False, default='csv', choices=('json', 'csv'), help='Defines the format of info output. Default: csv')
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
NodeHierarchy()
|
NodeHierarchy()
|
||||||
|
|
18
info/Info.py
Normal file
18
info/Info.py
Normal file
|
@ -0,0 +1,18 @@
|
||||||
|
from info.OfflineInfo import OfflineInfo
|
||||||
|
class Info(object):
|
||||||
|
def __init__(self, infoTypes, infoOutFolder, infoOutType, infoFilters, nodes, globalGraph, domains):
|
||||||
|
self.__infoTypes__ = infoTypes
|
||||||
|
self.__infoOutFolder__ = infoOutFolder
|
||||||
|
self.__infoOutType__ = infoOutType
|
||||||
|
self.__infoFilters__ = infoFilters
|
||||||
|
self.__nodes__ = nodes
|
||||||
|
self.__globalGraph__ = globalGraph
|
||||||
|
self.__domains__ = domains
|
||||||
|
if self.__infoTypes__ != None:
|
||||||
|
if 'get_offline_nodes' in self.__infoTypes__:
|
||||||
|
self.__offlineNodes__ = OfflineInfo(self.__infoFilters__, self.__nodes__, self.__domains__)
|
||||||
|
print(self.__infoOutType__)
|
||||||
|
if 'csv' in self.__infoOutType__:
|
||||||
|
self.__offlineNodes__.writeCSVtoFile(self.__infoOutFolder__+'/offline_nodes.csv')
|
||||||
|
if 'json' in self.__infoOutType__:
|
||||||
|
self.__offlineNodes__.writeJsonToFile(self.__infoOutFolder__+'/offline_nodes.json')
|
75
info/InfoMeta.py
Normal file
75
info/InfoMeta.py
Normal file
|
@ -0,0 +1,75 @@
|
||||||
|
import json
|
||||||
|
class InfoMeta(object):
|
||||||
|
def __init__(self):
|
||||||
|
self.resultNodes = None
|
||||||
|
self.resultGraph = None
|
||||||
|
|
||||||
|
|
||||||
|
def __generateNodesJson__(self):
|
||||||
|
if self.resultNodes == None:
|
||||||
|
return []
|
||||||
|
result = []
|
||||||
|
for node in self.resultNodes:
|
||||||
|
result.append(node.__jsonObject__)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def __generateNodesCSV__(self):
|
||||||
|
if self.resultNodes == None:
|
||||||
|
return ''
|
||||||
|
|
||||||
|
result = '"hostname","site","nodeid","ipv6addresses","status","lastseen","firstseen","autoupdater","branch","firmware","hardware"\n'
|
||||||
|
for node in self.resultNodes:
|
||||||
|
nodeData = node.__jsonObject__
|
||||||
|
nodeinfo = nodeData['nodeinfo']
|
||||||
|
result += '"'+nodeinfo['hostname']+'",'
|
||||||
|
|
||||||
|
try:
|
||||||
|
result +='"'+nodeinfo['system']['site_code']+'",'
|
||||||
|
except:
|
||||||
|
result += '"none",'
|
||||||
|
result += '"'+nodeinfo['node_id']+'","'
|
||||||
|
|
||||||
|
#add array of public IPv6 Addresses
|
||||||
|
addresses = node.__getPublicAddresses__()
|
||||||
|
for i, address in enumerate(addresses):
|
||||||
|
if i == len(addresses)-1:
|
||||||
|
result += address
|
||||||
|
else:
|
||||||
|
result += address + ','
|
||||||
|
result += '",'
|
||||||
|
|
||||||
|
if nodeData['flags']['online'] == True:
|
||||||
|
result += '"online",'
|
||||||
|
else:
|
||||||
|
result += '"offline",'
|
||||||
|
|
||||||
|
result += '"'+nodeData['lastseen']+'","'+nodeData['firstseen']+'",'
|
||||||
|
try:
|
||||||
|
if nodeinfo['software']['autoupdater']['enabled'] == True:
|
||||||
|
result += '"enabled",'
|
||||||
|
else:
|
||||||
|
result += '"disabled",'
|
||||||
|
except:
|
||||||
|
result += '"none",'
|
||||||
|
|
||||||
|
try:
|
||||||
|
result += '"'+nodeinfo['software']['autoupdater']['branch']+'",'
|
||||||
|
except:
|
||||||
|
result += '"none",'
|
||||||
|
result += '"'+nodeinfo['software']['firmware']['release']+'",'
|
||||||
|
|
||||||
|
try:
|
||||||
|
result += '"'+nodeinfo['hardware']['model']+'"'
|
||||||
|
except:
|
||||||
|
result += '"none"'
|
||||||
|
|
||||||
|
result += '\n'
|
||||||
|
return result
|
||||||
|
|
||||||
|
def writeCSVtoFile(self, filename):
|
||||||
|
with open(filename, 'w') as out:
|
||||||
|
out.write(self.__generateNodesCSV__())
|
||||||
|
|
||||||
|
def writeJsonToFile(self, filename):
|
||||||
|
with open(filename, 'w') as out:
|
||||||
|
out.write(json.dumps(self.__generateNodesJson__(), sort_keys=True, indent=4, ensure_ascii=False))
|
68
info/OfflineInfo.py
Normal file
68
info/OfflineInfo.py
Normal file
|
@ -0,0 +1,68 @@
|
||||||
|
from info.InfoMeta import InfoMeta
|
||||||
|
from shapely.geometry import Point
|
||||||
|
import datetime, re
|
||||||
|
from datetime import timedelta
|
||||||
|
class OfflineInfo(InfoMeta):
|
||||||
|
def __init__(self, filters, nodes, domains):
|
||||||
|
super().__init__()
|
||||||
|
self.__filters__ = filters
|
||||||
|
self.__nodes__ = nodes
|
||||||
|
self.__domains__ = domains
|
||||||
|
self.__minAge__, self.__maxAge__ = self.__parseFilters__()
|
||||||
|
self.resultNodes = self.__filterNodes__()
|
||||||
|
|
||||||
|
def __filterNodes__(self):
|
||||||
|
|
||||||
|
offlineNodes = []
|
||||||
|
|
||||||
|
for k,v in self.__nodes__.items():
|
||||||
|
if v.isOnline == False:
|
||||||
|
if v.geo != None:
|
||||||
|
for dk, dv in self.__domains__.items():
|
||||||
|
if dv.isPointInDomaene(Point((v.geo['lon'], v.geo['lat']))) == True:
|
||||||
|
nodeLastSeen = datetime.datetime.strptime(v.__jsonObject__['lastseen'],'%Y-%m-%dT%H:%M:%S')
|
||||||
|
if self.__minAge__ != None:
|
||||||
|
if self.__minAge__ < nodeLastSeen:
|
||||||
|
continue
|
||||||
|
if self.__maxAge__ != None:
|
||||||
|
if self.__maxAge__ > nodeLastSeen:
|
||||||
|
continue
|
||||||
|
offlineNodes.append(v)
|
||||||
|
|
||||||
|
return offlineNodes
|
||||||
|
|
||||||
|
|
||||||
|
def __parseFilters__(self):
|
||||||
|
|
||||||
|
if self.__filters__ == None:
|
||||||
|
return None, None
|
||||||
|
|
||||||
|
regX = re.compile("([0-9]+)([a-zA-Z]+)")
|
||||||
|
minAge = None
|
||||||
|
maxAge = None
|
||||||
|
|
||||||
|
for filter in self.__filters__:
|
||||||
|
attr = filter.split(':')
|
||||||
|
if len(attr) == 2:
|
||||||
|
if attr[0] == 'min_age' or attr[0] == 'max_age':
|
||||||
|
d = regX.match(attr[1])
|
||||||
|
if d != None:
|
||||||
|
val = int(d.group(1))
|
||||||
|
unit = d.group(2)
|
||||||
|
date = datetime.datetime.now()
|
||||||
|
if unit == 'd' or unit == 'day' or unit == 'days':
|
||||||
|
date = date - timedelta(days=val)
|
||||||
|
elif unit == 'w' or unit == 'week' or unit == 'weeks':
|
||||||
|
date = date - timedelta(days=val*7)
|
||||||
|
elif unit == 'm' or unit == 'month' or unit == 'months':
|
||||||
|
date = date - timedelta(days=val*30)
|
||||||
|
elif unit == 'y' or unit == 'year' or unit == 'years':
|
||||||
|
date = date - timedelta(days=val*365)
|
||||||
|
else:
|
||||||
|
date = None
|
||||||
|
|
||||||
|
if attr[0] == 'min_age':
|
||||||
|
minAge = date
|
||||||
|
elif attr[0] == 'max_age':
|
||||||
|
maxAge = date
|
||||||
|
return minAge, maxAge
|
0
info/__init__.py
Normal file
0
info/__init__.py
Normal file
Loading…
Reference in a new issue