add code to extract GPS data from new wiki
This commit is contained in:
parent
08187766b1
commit
5d203b0f07
57
nodedb.py
57
nodedb.py
|
@ -2,6 +2,9 @@ import json
|
||||||
from node import Node
|
from node import Node
|
||||||
from link import Link
|
from link import Link
|
||||||
|
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
import urllib
|
||||||
|
|
||||||
class NodeDB:
|
class NodeDB:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self._nodes = []
|
self._nodes = []
|
||||||
|
@ -120,3 +123,57 @@ class NodeDB:
|
||||||
link.strength = strength
|
link.strength = strength
|
||||||
return link
|
return link
|
||||||
|
|
||||||
|
def import_wikigps(self, url):
|
||||||
|
def fetch_wikitable(url):
|
||||||
|
f = urllib.urlopen(url)
|
||||||
|
|
||||||
|
soup = BeautifulSoup(f)
|
||||||
|
|
||||||
|
table = soup.find_all("table")[0]
|
||||||
|
|
||||||
|
rows = table.find_all("tr")
|
||||||
|
|
||||||
|
headers = []
|
||||||
|
|
||||||
|
data = []
|
||||||
|
|
||||||
|
def maybe_strip(x):
|
||||||
|
if isinstance(x.string, basestring):
|
||||||
|
return x.string.strip()
|
||||||
|
else:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
for row in rows:
|
||||||
|
tds = list([maybe_strip(x) for x in row.find_all("td")])
|
||||||
|
ths = list([maybe_strip(x) for x in row.find_all("th")])
|
||||||
|
|
||||||
|
if any(tds):
|
||||||
|
data.append(tds)
|
||||||
|
|
||||||
|
if any(ths):
|
||||||
|
headers = ths
|
||||||
|
|
||||||
|
nodes = []
|
||||||
|
|
||||||
|
for d in data:
|
||||||
|
nodes.append(dict(zip(headers, d)))
|
||||||
|
|
||||||
|
return nodes
|
||||||
|
|
||||||
|
nodes = fetch_wikitable(url)
|
||||||
|
|
||||||
|
for node in nodes:
|
||||||
|
if not ('MAC' in node and 'GPS' in node):
|
||||||
|
continue
|
||||||
|
|
||||||
|
macs = [s for s in [s.strip() for s in node['MAC'].split(',')] if s]
|
||||||
|
gps = [s for s in [s.strip() for s in node['GPS'].split(',')] if s]
|
||||||
|
|
||||||
|
for pair in zip(macs, gps):
|
||||||
|
try:
|
||||||
|
node = self.maybe_node_by_mac(pair[0])
|
||||||
|
except:
|
||||||
|
node = Node()
|
||||||
|
self._nodes.append(node)
|
||||||
|
|
||||||
|
node.gps = pair[1]
|
||||||
|
|
Loading…
Reference in a new issue