remove wikiimport from nodedb, move to ffhlwiki.py and call it from mkmap.sh

This commit is contained in:
Nils Schneider 2013-02-02 01:21:29 +01:00
parent 9b236c462b
commit 3ffbac8136
3 changed files with 97 additions and 82 deletions

93
ffhlwiki.py Executable file
View file

@ -0,0 +1,93 @@
#!/usr/bin/env python3
import json
import argparse
from itertools import zip_longest
from urllib.request import urlopen
from bs4 import BeautifulSoup
def import_wikigps(url):
def fetch_wikitable(url):
f = urlopen(url)
soup = BeautifulSoup(f)
table = soup.find_all("table")[0]
rows = table.find_all("tr")
headers = []
data = []
def maybe_strip(x):
if isinstance(x.string, str):
return x.string.strip()
else:
return ""
for row in rows:
tds = list([maybe_strip(x) for x in row.find_all("td")])
ths = list([maybe_strip(x) for x in row.find_all("th")])
if any(tds):
data.append(tds)
if any(ths):
headers = ths
nodes = []
for d in data:
nodes.append(dict(zip(headers, d)))
return nodes
nodes = fetch_wikitable(url)
aliases = []
for node in nodes:
try:
node['MAC'] = node['MAC'].split(',')
except KeyError:
pass
try:
node['GPS'] = node['GPS'].split(',')
except KeyError:
pass
try:
node['Knotenname'] = node['Knotenname'].split(',')
except KeyError:
pass
nodes = zip_longest(node['MAC'], node['GPS'], node['Knotenname'])
for data in nodes:
alias = {}
alias['mac'] = data[0].strip()
if data[1]:
alias['gps'] = data[1].strip()
if data[2]:
alias['name'] = data[2].strip()
aliases.append(alias)
return aliases
parser = argparse.ArgumentParser()
parser.add_argument('url', help='wiki URL')
args = parser.parse_args()
options = vars(args)
aliases = import_wikigps(options['url'])
print(json.dumps(aliases))

View file

@ -17,7 +17,10 @@ if [ `cat /sys/class/net/bat0/mesh/gw_mode` = server ]; then
GWS="$GWS -g $SELF" GWS="$GWS -g $SELF"
fi fi
batctl vd json | "$(dirname "$0")"/bat2nodes.py -a "$(dirname "$0")"/aliases.json $GWS - > $DEST/nodes.json.new "$(dirname "$0")"/ffhlwiki.py http://freifunk.metameute.de/wiki/Knoten > "$(dirname "$0")"/aliases_hl.json
"$(dirname "$0")"/ffhlwiki.py http://freifunk.metameute.de/wiki/Moelln:Knoten > "$(dirname "$0")"/aliases_moelln.json
batctl vd json | "$(dirname "$0")"/bat2nodes.py -a "$(dirname "$0")"/aliases.json -a aliases_hl.json -a aliases_moelln.json $GWS - > $DEST/nodes.json.new
mv $DEST/nodes.json.new $DEST/nodes.json mv $DEST/nodes.json.new $DEST/nodes.json

View file

@ -3,10 +3,6 @@ from functools import reduce
from collections import defaultdict from collections import defaultdict
from node import Node, Interface from node import Node, Interface
from link import Link, LinkConnector from link import Link, LinkConnector
from itertools import zip_longest
from bs4 import BeautifulSoup
from urllib.request import urlopen
class NodeDB: class NodeDB:
def __init__(self): def __init__(self):
@ -213,83 +209,6 @@ class NodeDB:
link.type = "vpn" link.type = "vpn"
def import_wikigps(self, url):
def fetch_wikitable(url):
f = urlopen(url)
soup = BeautifulSoup(f)
table = soup.find_all("table")[0]
rows = table.find_all("tr")
headers = []
data = []
def maybe_strip(x):
if isinstance(x.string, str):
return x.string.strip()
else:
return ""
for row in rows:
tds = list([maybe_strip(x) for x in row.find_all("td")])
ths = list([maybe_strip(x) for x in row.find_all("th")])
if any(tds):
data.append(tds)
if any(ths):
headers = ths
nodes = []
for d in data:
nodes.append(dict(zip(headers, d)))
return nodes
nodes = fetch_wikitable(url)
for node in nodes:
try:
node['MAC'] = node['MAC'].split(',')
except KeyError:
pass
try:
node['GPS'] = node['GPS'].split(',')
except KeyError:
pass
try:
node['Knotenname'] = node['Knotenname'].split(',')
except KeyError:
pass
nodes = zip_longest(node['MAC'], node['GPS'], node['Knotenname'])
for data in nodes:
mac = data[0].strip()
if not mac:
continue
try:
node = self.maybe_node_by_fuzzy_mac(mac)
except KeyError:
node = Node()
self._nodes.append(node)
node.add_mac(mac)
if data[1]:
node.gps = data[1].strip()
if data[2]:
node.name = data[2].strip()
# compares two MACs and decides whether they are # compares two MACs and decides whether they are
# similar and could be from the same node # similar and could be from the same node
def is_similar(a, b): def is_similar(a, b):