Incorrect fieldname "dbname" and using JSON content between processors

addok_psql/__init__.py

@@ -1,5 +1,5 @@
 "Import from PostgreSQL into addok."
-VERSION = (0, 1, 0)
+VERSION = (0, 1, 1)
 
 __author__ = 'Yohan Boniface'
 __contact__ = "yohan.boniface@data.gouv.fr"

addok_psql/config.py

@@ -1,5 +1,5 @@
 PSQL = {
-    'dbname': 'nominatim'
+    'database': 'nominatim'
 }
 PSQL_PROCESSORS = (
     'addok_psql.processors.query',

addok_psql/plugin.py

@@ -12,7 +12,7 @@ def preprocess():
 
 def process(args):
     print('Import from Postgresql')
-    keys = ['dbname', 'user', 'host', 'port']
+    keys = ['dbname', 'user', 'host', 'port', 'password']
     for key in keys:
         value = getattr(args, key, None)
         if value:
@@ -32,6 +32,7 @@ def register_command(subparsers):
     parser.set_defaults(func=process)
     parser.add_argument('--host', help='PostgreSQL host')
     parser.add_argument('--user', help='PostgreSQL user')
-    parser.add_argument('--dbname', help='PostgreSQL name')
+    parser.add_argument('--dbname', help='PostgreSQL database name')
+    parser.add_argument('--password', help='PostgreSQL password user')
     parser.add_argument('--port', help='PostgreSQL port')
     parser.add_argument('--limit', help='Limit retrieved rows')

addok_psql/processors.py

@@ -2,9 +2,7 @@
 
 import psycopg2
 import psycopg2.extras
-
-import psycopg2
-import psycopg2.extras
+import json
 
 from addok.config import config
 from addok.helpers import yielder
@@ -27,16 +25,17 @@ def query(*args):
     print('Query executed with itersize', cur.itersize)
 
     for row in cur.__iter__():
-        yield dict(row)
+        yield json.dumps(dict(row))
     cur.close()
 
 
 @yielder
-def get_context(row):
+def get_context(raw):
+    row = json.loads(raw)
     if "context" not in row:
         row['context'] = []
     add_parent(row, row)
-    return row
+    return json.dumps(row)
 
 
 def add_parent(child, row):
@@ -63,7 +62,8 @@ def add_parent_data(parent, row):
 
 
 @yielder
-def get_housenumbers(row):
+def get_housenumbers(raw):
+    row = json.loads(raw)
     if row['class'] == 'highway':
         sql = """SELECT housenumber, ST_X(ST_Centroid(geometry)) as lon,
             ST_Y(ST_Centroid(geometry)) as lat
@@ -79,11 +79,12 @@ def get_housenumbers(row):
             hn['housenumber']: {'lat': hn['lat'], 'lon': hn['lon']}
             for hn in housenumbers
         }
-    return row
+    return json.dumps(row)
 
 
 @yielder
-def row_to_doc(row):
+def row_to_doc(raw):
+    row = json.loads(raw)
     doc = {
         "id": row["osm_type"] + str(row["osm_id"]),
         "lat": row['lat'],
@@ -116,4 +117,4 @@ def row_to_doc(row):
     row['source'] = 'OSM'
     # See https://wiki.osm.org/wiki/Nominatim/Development_overview#Country_to_street_level  # noqa
     doc['importance'] = (row.get('rank_search', 30) / 30) * 0.1
-    return doc
+    return json.dumps(row)