make reverse geolocation lookup more stableand removed unused modules

scripts/mkGeolocationTable.py

@@ -39,12 +39,13 @@ def mkGeolocationTable(file=''):
         import xml.etree.ElementTree as ET
     from sys import stdin
     import warnings
-    from time import sleep
     from geopy.geocoders import Photon
-    import urllib2 as URL
+    from geopy.extra.rate_limiter import RateLimiter
+    from geopy.exc import GeocoderServiceError
     import json as JSON
 
     # constants
+    url = 'https://photon.komoot.io/reverse?lon={lng:.3f}&lat={lat:.3f}'
     namespaces = {'sc3': 'http://geofon.gfz-potsdam.de/ns/seiscomp3-schema/0.7',
         'qml': 'http://quakeml.org/xmlns/bed/1.2'}
 
@@ -66,12 +67,13 @@ def mkGeolocationTable(file=''):
     # parse event.xml
     DOM = ET.parse(stdin).getroot()
     geolocator = Photon()
+    reverse_geolocate = RateLimiter(geolocator.reverse, min_delay_seconds=1)
 
     # iterate over all events
     for event in DOM.iterfind('qml:eventParameters/qml:event', namespaces):
         publicID = event.attrib['publicID'].split('/')[2]
-        lat = event.find('./qml:origin/qml:latitude/qml:value', namespaces).text
+        lat = float(event.find('./qml:origin/qml:latitude/qml:value', namespaces).text)
-        lng = event.find('./qml:origin/qml:longitude/qml:value', namespaces).text
+        lng = float(event.find('./qml:origin/qml:longitude/qml:value', namespaces).text)
         evaluationMode = event.find('./qml:origin/qml:evaluationMode', namespaces).text
 
         if publicID in geolocationTable:
@@ -80,11 +82,12 @@ def mkGeolocationTable(file=''):
             warnings.warn('Skipping automatic event %s' %(publicID))
         else:
             try:
-                location = geolocator.reverse("{lat}, {lng}".format(lat=lat, lng=lng))
+                locations = reverse_geolocate("{lat:.3f}, {lng:.3f}".format(lat=lat, lng=lng),exactly_one=False,limit=5)
-            except:
+            except GeocoderServiceError:
                 warnings.warn('Reverse Geolocation failed. Skipping event.')
                 continue
             place = []
+            for location in locations:
                 try:
                     place = location.raw['properties']['city']
                 except KeyError:
@@ -97,43 +100,11 @@ def mkGeolocationTable(file=''):
                             try:
                                 place = location.raw['properties']['county']
                             except KeyError:
-                            warnings.warn('Could not extract city for event {0} at {1} N / {2} E (URL: {3})'.format(publicID, lat, lng, url))
+                                pass
+            if not place:
+                warnings.warn('Could not extract city for event {id} at {lat:.3f} N / {lng:.3f} E (Service: {url})'.format(id=publicID, lat=lat, lng=lng, url=url.format(lat=lat,lng=lng)))
             geolocationTable[publicID] = place
-            sleep(1.1)
 
-            '''
-            #warnings.warn('Processing event %s' %publicID)
-            # send and evaluate nominatim request
-            url = 'https://open.mapquestapi.com/nominatim/v1/reverse.php?key=oE512xGQxeH1n2cueacJ6xzRv7qFlCib&lat={0}&lon={1}&zoom=10&format=json'.format(lat, lng)
-            response = URL.urlopen(url)
-            if ( response.msg == 'OK' ):
-                data = JSON.loads(response.read())
-                city = []
-                try:
-                    try:
-                        city = data['address']['city']
-                    except:
-                        warnings.warn('Request {3} for event {0} at {1} N / {2} E did not provide city attribute\n\t(Response: {4})'.format(publicID, lat, lng, url, data))
-                        try:
-                            city = data['address']['town']
-                            warnings.warn('Using attribute town ({1}) for event {0}'.format(publicID, city))
-                        except:
-                            try:
-                                city = data['address']['county']
-                                warnings.warn('Using attribute county ({1}) for event {0}'.format(publicID, city))
-                            except:
-                                warnings.warn('Skipping event')
-                                continue
-                    countryCode = data['address']['country_code'].upper()
-                    if ( countryCode == 'DE' ):
-                        geolocationTable[publicID] = city.encode('utf-8')
-                    else:
-                        geolocationTable[publicID] = '{0} ({1})'.format(city.encode('utf-8'), countryCode)
-                except:
-                    warnings.warn('Could not extract city for event {0} at {1} N / {2} E (URL: {3})'.format(publicID, lat, lng, url))
-            else:
-                warnings.warn('Request {0} failed'.format(url))
-            '''
     # dump json
     print('var geolocationTable = {0};'.format(JSON.dumps(geolocationTable, sort_keys=True)))