From 8d4021f122505602b6f7e7938fecb662907c2818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20W=C3=BCllhorst?= Date: Sat, 19 Aug 2017 23:25:30 +0200 Subject: [PATCH] Several updates: - Implemented option to translate sitecode to domainname. - Added filter: Skip nodes if they are already in the target domain. - Several minor changes. --- NodeHierarchy.py | 16 +++++++++++++--- README.md | 16 ++++++++++------ cloud/NodeInit.py | 19 +++++++++++-------- generator/Filter.py | 10 ++++++++-- generator/NginxConfGen.py | 4 ++-- 5 files changed, 44 insertions(+), 21 deletions(-) diff --git a/NodeHierarchy.py b/NodeHierarchy.py index c914c81..aeb0721 100755 --- a/NodeHierarchy.py +++ b/NodeHierarchy.py @@ -45,10 +45,19 @@ class NodeHierarchy(object): for nodeID, nodeValue in self.__hopglass.nodes.items(): if nodeValue['nodeinfo']['node_id']: print('Create Node object #',len(nodes), '\r',end = '') - nodes[nodeID] = Node(nodeValue) + nodes[nodeID] = Node(self.__prepareNodeData__(nodeValue)) print('') return nodes - + + def __prepareNodeData__(self, nodeValue): + if self.__args__.site_to_target_prefix: + pref = self.__args__.site_to_target_prefix.split(',') + try: + nodeValue['nodeinfo']['system']['site_code'] = nodeValue['nodeinfo']['system']['site_code'].replace(pref[0],pref[1]) + except: + pass + return nodeValue + def __createLinkObjects__(self): links = [] for linkParID, linkPar in self.__hopglass.links.items(): @@ -67,9 +76,10 @@ class NodeHierarchy(object): parser.add_argument('-r', '--raw-json', required=False, default='https://karte.freifunk-muensterland.de/data/raw.json', help='Location of raw.json file (can be local folder or remote URL).') parser.add_argument('-s', '--shapes-path', required=False, default='https://freifunk-muensterland.de/md-fw-dl/shapes/', help='Path of shapefiles (can be local folder or remote URL).') parser.add_argument('-t', '--targets', nargs='+', required=True, help='List of targets which should be proceeded. Example: -t citya cityb ...') + parser.add_argument('-sttp', '--site-to-target-prefix', required=False, help='Used to match site and target also when prefixes are different. Example: -sttp "ffmsd,domaene"') parser.add_argument('-o', '--out-file', default='./webserver-configuration', required=False, help='Filename where the generated Output should stored.') parser.add_argument('-v', '--debug', required=False, action='store_true', help='Enable debugging output.') - parser.add_argument('-f', '--filters', nargs='*', required=False, choices=('exclude_clouds_with_lan_links', 'no_lan'), help='Filter out nodes and local clouds based on filter rules.') + parser.add_argument('-f', '--filters', nargs='*', required=False, choices=('exclude_clouds_with_lan_links', 'no_lan', 'domain_transitions_only'), help='Filter out nodes and local clouds based on filter rules.') parser.add_argument('-i', '--info', nargs='*', required=False, choices=('get_offline_nodes','offline'), help='Get infos about the graph, links and nodes.') parser.add_argument('-if', '--info-filters', nargs='*', required=False, help='Filter info results. Currently supported: min_age:TIME_RANGE, max_age:TIME_RANGE. Examples: -if min_age:1d max_age:2w') parser.add_argument('-iop', '--info-out-path', required=False, default='./', help='Folder where info files should be written. Default: ./') diff --git a/README.md b/README.md index 1f34579..bdcbf1d 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,9 @@ Die Hilfe liefert folgendes: ``` $ ./NodeHierarchy.py --help usage: NodeHierarchy.py [-h] [-r RAW_JSON] [-s SHAPES_PATH] -t TARGETS - [TARGETS ...] [-o OUT_FILE] [-v] - [-f [{exclude_clouds_with_lan_links,no_lan} [{exclude_clouds_with_lan_links,no_lan} ...]]] + [TARGETS ...] [-sttp SITE_TO_TARGET_PREFIX] + [-o OUT_FILE] [-v] + [-f [{exclude_clouds_with_lan_links,no_lan,domain_transitions_only} [{exclude_clouds_with_lan_links,no_lan,domain_transitions_only} ...]]] [-i [{get_offline_nodes,offline} [{get_offline_nodes,offline} ...]]] [-if [INFO_FILTERS [INFO_FILTERS ...]]] [-iop INFO_OUT_PATH] @@ -43,18 +44,20 @@ optional arguments: -h, --help show this help message and exit -r RAW_JSON, --raw-json RAW_JSON Location of raw.json file (can be local folder or - remote URL). Default: https://karte.freifunk- - muensterland.de/data/raw.json + remote URL). -s SHAPES_PATH, --shapes-path SHAPES_PATH Path of shapefiles (can be local folder or remote URL). -t TARGETS [TARGETS ...], --targets TARGETS [TARGETS ...] List of targets which should be proceeded. Example: -t citya cityb ... + -sttp SITE_TO_TARGET_PREFIX, --site-to-target-prefix SITE_TO_TARGET_PREFIX + Used to match site and target also when prefixes are + different. Example: -sttp "ffmsd,domaene" -o OUT_FILE, --out-file OUT_FILE Filename where the generated Output should stored. -v, --debug Enable debugging output. - -f [{exclude_clouds_with_lan_links,no_lan} [{exclude_clouds_with_lan_links,no_lan} ...]], --filters [{exclude_clouds_with_lan_links,no_lan} [{exclude_clouds_with_lan_links,no_lan} ...]] + -f [{exclude_clouds_with_lan_links,no_lan,domain_transitions_only} [{exclude_clouds_with_lan_links,no_lan,domain_transitions_only} ...]], --filters [{exclude_clouds_with_lan_links,no_lan,domain_transitions_only} [{exclude_clouds_with_lan_links,no_lan,domain_transitions_only} ...]] Filter out nodes and local clouds based on filter rules. -i [{get_offline_nodes,offline} [{get_offline_nodes,offline} ...]], --info [{get_offline_nodes,offline} [{get_offline_nodes,offline} ...]] @@ -90,7 +93,8 @@ Weitere Filterungen lassen sich über das ``--filters`` Attribut aktivieren. Folgende Filter sind derzeit implementiert (zukünftig folgen noch weitere): -- ``exclude_clouds_with_lan_links`` bzw. ``no_lan`` Filtert alle lokalen Wolken aus, in denen sich mindestens ein Mesh-on-LAN Link befindet +- ``exclude_clouds_with_lan_links`` bzw. ``no_lan`` filtert alle lokalen Wolken aus, in denen sich mindestens ein Mesh-on-LAN Link befindet +- ``domain_transitions_only`` filtert alle Knoten aus, die sich bereits in der richtigen Domäne befinden / die Firmware der richtigen Domäne besitzen ## Nginx Konfiguration diff --git a/cloud/NodeInit.py b/cloud/NodeInit.py index a7abf1e..131f10e 100644 --- a/cloud/NodeInit.py +++ b/cloud/NodeInit.py @@ -10,7 +10,7 @@ class NodeInit(object): self.autoupdaterBranch = self.__getBranch__() self.isOnline = self.__jsonObject__['nodeinfo']['isOnline'] self.publicIPv6Addresses = self.__getPublicAddresses__() - self.domID = self.__getSiteCode__() + self.domName = self.__getSiteCode__() def __getInterfaces__(self): try: @@ -31,19 +31,22 @@ class NodeInit(object): return None def __getGeo__(self): - geo = {} - if 'location' in self.__jsonObject__['nodeinfo'] and 'latitude' in self.__jsonObject__['nodeinfo']['location'] and 'longitude' in self.__jsonObject__['nodeinfo']['location']: - geo['lat'] = self.__jsonObject__['nodeinfo']['location']['latitude'] - geo['lon'] = self.__jsonObject__['nodeinfo']['location']['longitude'] - return geo - return None + try: + return { + 'lat' : self.__jsonObject__['nodeinfo']['location']['latitude'], + 'lon' : self.__jsonObject__['nodeinfo']['location']['longitude'] + } + except: + return None def __getPublicAddresses__(self): addresses = [] - if 'addresses' in self.__jsonObject__['nodeinfo']['network']: + try: for address in self.__jsonObject__['nodeinfo']['network']['addresses']: if not address.startswith('fe80'): addresses.append(address) + except: + pass return addresses def __getSiteCode__(self): diff --git a/generator/Filter.py b/generator/Filter.py index fad49c6..c141a9e 100644 --- a/generator/Filter.py +++ b/generator/Filter.py @@ -3,7 +3,7 @@ class Filter(object): self.__args__ = args self.__filters__ = self.__getFilters() - def filterLocalGraphs(self, localGraphs): + def filterLocalGraphs(self, domain, localGraphs): filteredGraphs = [] for localGraph in localGraphs: if localGraph.isAutoupdaterEnabledOnAllNodes() == False: @@ -21,8 +21,14 @@ class Filter(object): def __getFilters(self): return [] if self.__args__.filters == None else self.__args__.filters - def filterNodes(self, nodes): + def filterNodes(self, domain, nodes): filteredNodes = [] for node in nodes: + if 'domain_transitions_only' in self.__filters__: + try: + if domain.name == node.domName: + continue + except: + pass filteredNodes.append(node) return filteredNodes diff --git a/generator/NginxConfGen.py b/generator/NginxConfGen.py index 583ac4d..aedc895 100644 --- a/generator/NginxConfGen.py +++ b/generator/NginxConfGen.py @@ -17,9 +17,9 @@ class NginxConfGen(object): def __genDomain__(self, domain): nodes = {} - for localGraph in self.__filter__.filterLocalGraphs(domain.localGraphs): + for localGraph in self.__filter__.filterLocalGraphs(domain, domain.localGraphs): try: - for node in self.__filter__.filterNodes(localGraph.getNodesWithNoDependencies()): + for node in self.__filter__.filterNodes(domain, localGraph.getNodesWithNoDependencies()): nodes[node.nodeID] = { 'hostname' : node.hostname, 'ipv6_addresses' : node.publicIPv6Addresses