feat: import addresses which use addr:place instead of addr:street (#565

)

* added: missing street schema

* added: "addr: place" support

* added: "addr: place" support

* docs: stream

* refaktor: sugesstions from code review

Co-authored-by: pbieniek <[email protected]>

config/features.js

@@ -6,7 +6,8 @@
 
 // default tags imported
 const tags = [
-  'addr:housenumber+addr:street'
+  'addr:housenumber+addr:street',
+  'addr:housenumber+addr:place'  // @ref https://github.com/pelias/pelias/issues/787#issuecomment-477137803
 ];
 
 // tags corresponding to venues

schema/address_karlsruhe.js

@@ -22,4 +22,4 @@ var KARLSRUHE_SCHEMA = {
   // 'addr:country':       'country'
 };
 
-module.exports = KARLSRUHE_SCHEMA;
+module.exports = KARLSRUHE_SCHEMA;

stream/addresses_without_street.js

@@ -0,0 +1,55 @@
+
+/**
+  This stream is responsible for filling 'addr:street' if is empty or missing,
+  with 'addr:place' value. Note that both 'addr:place' and 'addr:housenumber'
+  must be present inside tags!
+
+  @ref https://github.com/pelias/openstreetmap/pull/565
+**/
+
+const _ = require('lodash');
+const through = require('through2');
+const peliasLogger = require('pelias-logger').get('openstreetmap');
+
+
+module.exports = function(){
+
+  var stream = through.obj( function( doc, enc, next ) {
+
+    try {
+
+      // skip records with no tags
+      var tags = doc.getMeta('tags');
+      if( !tags ){
+        return next( null, doc );
+      }
+
+      // housenumber is required
+      if (!_.has(tags, 'addr:housenumber')){
+        return next( null, doc );
+      }
+
+      const street = _.get(tags, 'addr:street', '').trim();
+      const place = _.get(tags, 'addr:place', '').trim();
+
+      // when street is unset but place is set, use place for the street name
+      if (_.isEmpty(street) && !_.isEmpty(place)) {
+        _.set(tags, 'addr:street', place);
+      }
+    }
+
+    catch( e ){
+      peliasLogger.error( 'addresses_without_street error' );
+      peliasLogger.error( e.stack );
+      peliasLogger.error( JSON.stringify( doc, null, 2 ) );
+    }
+
+    return next( null, doc );
+
+  });
+
+  // catch stream errors
+  stream.on( 'error', peliasLogger.error.bind( peliasLogger, __filename ) );
+
+  return stream;
+};

stream/importPipeline.js

@@ -10,6 +10,7 @@ streams.pbfParser = require('./multiple_pbfs').create;
 streams.docConstructor = require('./document_constructor');
 streams.blacklistStream = require('pelias-blacklist-stream');
 streams.tagMapper = require('./tag_mapper');
+streams.addressesWithoutStreet = require('./addresses_without_street');
 streams.adminLookup = require('pelias-wof-admin-lookup').create;
 streams.addressExtractor = require('./address_extractor');
 streams.categoryMapper = require('./category_mapper');
@@ -22,6 +23,7 @@ streams.elasticsearch = require('pelias-dbclient');
 streams.import = function(){
   streams.pbfParser()
     .pipe( streams.docConstructor() )
+    .pipe( streams.addressesWithoutStreet() )
     .pipe( streams.tagMapper() )
     .pipe( streams.addressExtractor() )
     .pipe( streams.blacklistStream() )

test/run.js

@@ -16,7 +16,8 @@ var tests = [
   require('./stream/importPipeline'),
   require('./stream/pbf'),
   require('./stream/stats'),
-  require('./stream/tag_mapper')
+  require('./stream/tag_mapper'),
+  require('./stream/addresses_without_street')
 ];
 
 tests.map(function(t) {

test/stream/addresses_without_street.js

@@ -0,0 +1,102 @@
+
+const through = require('through2');
+const mapper = require('../../stream/addresses_without_street');
+const Document = require('pelias-model').Document;
+
+module.exports.tests = {};
+
+// test exports
+module.exports.tests.interface = function(test, common) {
+  test('interface: factory', function(t) {
+    t.equal(typeof mapper, 'function', 'stream factory');
+    t.end();
+  });
+  test('interface: stream', function(t) {
+    var stream = mapper();
+    t.equal(typeof stream, 'object', 'valid stream');
+    t.equal(typeof stream._read, 'function', 'valid readable');
+    t.equal(typeof stream._write, 'function', 'valid writeable');
+    t.end();
+  });
+};
+
+// a tags with missing required extra tag (addr:housenumber)
+// should pass through the stream without being modified.
+// @ref https://github.com/pelias/openstreetmap/pull/565#issuecomment-1062874227
+module.exports.tests.passthrough = function(test, common) {
+  test('passthrough: missing addr:housenumber tag', function(t) {
+    var original = new Document('a','b', 1);
+    original.setMeta('tags', { 'addr:place': 'L14' });
+    var stream = mapper();
+    stream.pipe( through.obj( function( doc, enc, next ){
+      t.deepEqual(doc.getMeta('tags'), original.getMeta('tags'), 'tags not modified' );
+      t.end(); // test will fail if not called (or called twice).
+      next();
+    }));
+    stream.write(original);
+  });
+};
+
+// // ======================== addresses =========================
+
+module.exports.tests.karlsruhe_schema = function(test, common) {
+  test('maps - karlsruhe schema with filled street', function(t) {
+    var doc = new Document('a','b', 1);
+    doc.setMeta('tags', { 'addr:street': 'BBB' });
+    var stream = mapper();
+    stream.pipe( through.obj( function( doc, enc, next ){
+      var tags = doc.getMeta('tags');
+      t.equal(tags['addr:street'], 'BBB', 'correctly mapped');
+      t.end(); // test will fail if not called (or called twice).
+      next();
+    }));
+    stream.write(doc);
+  });
+  test('maps - karlsruhe schema with empty street and filled place', function(t) {
+    var doc = new Document('a','b', 1);
+    doc.setMeta('tags', { 'addr:place': 'L14', 'addr:street': '', 'addr:housenumber': '14' });
+    var stream = mapper();
+    stream.pipe( through.obj( function( doc, enc, next ){
+      var tags = doc.getMeta('tags');
+      t.equal(tags['addr:street'], 'L14', 'correctly mapped');
+      t.end(); // test will fail if not called (or called twice).
+      next();
+    }));
+    stream.write(doc);
+  });
+  test('maps - karlsruhe schema without street and with filled place', function(t) {
+    var doc = new Document('a','b', 1);
+    doc.setMeta('tags', { 'addr:place': 'L14', 'addr:housenumber': '14' });
+    var stream = mapper();
+    stream.pipe( through.obj( function( doc, enc, next ){
+      var tags = doc.getMeta('tags');
+      t.equal(tags['addr:street'], 'L14', 'correctly mapped');
+      t.end(); // test will fail if not called (or called twice).
+      next();
+    }));
+    stream.write(doc);
+  });
+  test('maps - karlsruhe schema with both street and place', function(t) {
+    var doc = new Document('a','b', 1);
+    doc.setMeta('tags', { 'addr:place': 'L14', 'addr:street': 'BBB', 'addr:housenumber': '14' });
+    var stream = mapper();
+    stream.pipe( through.obj( function( doc, enc, next ){
+      var tags = doc.getMeta('tags');
+      t.equal(tags['addr:street'], 'BBB', 'correctly mapped');
+      t.end(); // test will fail if not called (or called twice).
+      next();
+    }));
+    stream.write(doc);
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('address_without_street: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/stream/importPipeline.js

@@ -11,6 +11,7 @@ module.exports.tests.interface = function(test, common) {
     'pbfParser',
     'docConstructor',
     'blacklistStream',
+    'addressesWithoutStreet',
     'tagMapper',
     'adminLookup',
     'addressExtractor',