PageRenderTime 33ms CodeModel.GetById 24ms RepoModel.GetById 1ms app.codeStats 0ms

/geopy/geocoders/wiki_semantic.py

https://bitbucket.org/shishirk/geopy
Python | 108 lines | 85 code | 20 blank | 3 comment | 20 complexity | 27eced587d69a0e2cdea8f322e69f233 MD5 | raw file
    

  1. import xml.dom.minidom
    2. 

  2. from geopy.geocoders.base import Geocoder
    3. 

  3. from geopy.point import Point
    4. 

  4. from geopy.location import Location
    5. 

  5. from geopy import util
    6. 

  6. 

  7. try:
    8. 

  8.     from BeautifulSoup import BeautifulSoup
    9. 

  9. except ImportError:
    10. 

  10.     util.logger.warn("BeautifulSoup was not found. " \
    11. 

  11.           "The SemanticMediaWiki geocoder will not work.")
    12. 

  12. 

  13. try:
    14. 

  14.     set
    15. 

  15. except NameError:
    16. 

  16.     from sets import Set as set
    17. 

  17. 

  18. class SemanticMediaWiki(Geocoder):
    19. 

  19.     def __init__(self, format_url, attributes=None, relations=None,
    20. 

  20.                  prefer_semantic=False, transform_string=None):
    21. 

  21.         self.format_url = format_url
    22. 

  22.         self.attributes = attributes
    23. 

  23.         self.relations = relations
    24. 

  24.         self.prefer_semantic = prefer_semantic
    25. 

  25.         self.transform_string = transform_string
    26. 

  26.     
    27. 

  27.     def get_url(self, string):
    28. 

  28.         return self.format_url % self.transform_string(string)
    29. 

  29. 

  30.     def parse_rdf_link(self, page, mime_type='application/rdf+xml'):
    31. 

  31.         """Parse the URL of the RDF link from the <head> of ``page``."""
    32. 

  32.         soup = BeautifulSoup(page)
    33. 

  33.         link = soup.head.find('link', rel='alternate', type=mime_type)
    34. 

  34.         return link and link['href'] or None
    35. 

  35.     
    36. 

  36.     def parse_rdf_things(self, data):
    37. 

  37.         dom = xml.dom.minidom.parseString(data)
    38. 

  38.         thing_map = {}
    39. 

  39.         things = dom.getElementsByTagName('smw:Thing')
    40. 

  40.         things.reverse()
    41. 

  41.         for thing in things:
    42. 

  42.             name = thing.attributes['rdf:about'].value
    43. 

  43.             articles = thing.getElementsByTagName('smw:hasArticle')
    44. 

  44.             things[name] = articles[0].attributes['rdf:resource'].value
    45. 

  45.         
    46. 

  46.         return (things, thing)
    47. 

  47.     
    48. 

  48.     def transform_semantic(self, string):
    49. 

  49.         """Normalize semantic attribute and relation names by replacing spaces
    50. 

  50.         with underscores and capitalizing the result."""
    51. 

  51.         return string.replace(' ', '_').capitalize()
    52. 

  52. 

  53.     def get_relations(self, thing, relations=None):
    54. 

  54.         if relations is None:
    55. 

  55.             relations = self.relations
    56. 

  56.         
    57. 

  57.         for relation in relations:
    58. 

  58.             relation = self.transform_semantic(relation)
    59. 

  59.             for node in thing.getElementsByTagName('relation:' + relation):
    60. 

  60.                 resource = node.attributes['rdf:resource'].value
    61. 

  61.                 yield (relation, resource)
    62. 

  62.     
    63. 

  63.     def get_attributes(self, thing, attributes=None):
    64. 

  64.         if attributes is None:
    65. 

  65.             attributes = self.attributes
    66. 

  66. 

  67.         for attribute in attributes:
    68. 

  68.             attribute = self.transform_semantic(attribute)
    69. 

  69.             for node in thing.getElementsByTagName('attribute:' + attribute):
    70. 

  70.                 value = node.firstChild.nodeValue.strip()
    71. 

  71.                 yield (attribute, value)
    72. 

  72.     
    73. 

  73.     def get_thing_label(self, thing):
    74. 

  74.         return util.get_first_text(thing, 'rdfs:label')
    75. 

  75.     
    76. 

  76.     def geocode_url(self, url, attempted=None):
    77. 

  77.         if attempted is None:
    78. 

  78.             attempted = set()
    79. 

  79. 

  80.         util.logger.debug("Fetching %s..." % url)
    81. 

  81.         page = urlopen(url)
    82. 

  82.         soup = BeautifulSoup(page)
    83. 

  83. 

  84.         rdf_url = self.parse_rdf_link(soup)
    85. 

  85.         util.logger.debug("Fetching %s..." % rdf_url)
    86. 

  86.         page = urlopen(rdf_url)
    87. 

  87. 

  88.         things, thing = self.parse_rdf(page)
    89. 

  89.         name = self.get_label(thing)
    90. 

  90. 

  91.         attributes = self.get_attributes(thing)
    92. 

  92.         for attribute, value in attributes:
    93. 

  93.             latitude, longitude = util.parse_geo(value)
    94. 

  94.             if None not in (latitude, longitude):
    95. 

  95.                 break
    96. 

  96. 

  97.         if None in (latitude, longitude):
    98. 

  98.             relations = self.get_relations(thing)
    99. 

  99.             for relation, resource in relations:
    100. 

  100.                 url = things.get(resource, resource)
    101. 

  101.                 if url in tried: # Avoid cyclic relationships.
    102. 

  102.                     continue
    103. 

  103.                 tried.add(url)
    104. 

  104.                 name, (latitude, longitude) = self.geocode_url(url, tried)
    105. 

  105.                 if None not in (name, latitude, longitude):
    106. 

  106.                     break
    107. 

  107. 

  108.         return (name, (latitude, longitude))
    109. 

  109.