feat: Merge pull request #184 from pelias/use-oa-hash

Use OA hash field instead of

.jshintrc

@@ -1,4 +1,5 @@
 {
+  "esversion": 6,
   "node": true,
   "curly": true,
   "eqeqeq": true,

lib/streams/documentStream.js

@@ -1,23 +1,28 @@
-var through = require( 'through2' );
+'use strict';
 
-var peliasModel = require( 'pelias-model' );
+const through = require( 'through2' );
+
+const peliasModel = require( 'pelias-model' );
 
 /*
  * Create a stream of Documents from valid, cleaned CSV records
  */
 function createDocumentStream(id_prefix, stats) {
   /**
-   * Used to track the UID of individual records passing through the stream
-   * created by `createRecordStream()`.  See `peliasModel.Document.setId()` for
-   * information about UIDs.
+   * Used to track the UID of individual records passing through the stream if
+   * there is no HASH that can be used as a more unique identifier.  See
+   * `peliasModel.Document.setId()` for information about UIDs.
    */
-  var uid = 0;
+  let uid = 0;
 
   return through.obj(
     function write( record, enc, next ){
-      var model_id = id_prefix + ':' + uid++;
+      const id_number = record.HASH || uid;
+      const model_id = `${id_prefix}:${id_number}`;
+      uid++;
+
       try {
-        var addrDoc = new peliasModel.Document( 'openaddresses', 'address', model_id )
+        const addrDoc = new peliasModel.Document( 'openaddresses', 'address', model_id )
         .setName( 'default', (record.NUMBER + ' ' + record.STREET) )
         .setCentroid( { lon: record.LON, lat: record.LAT } );
 

test/streams/documentStream.js

@@ -1,21 +1,23 @@
-var tape = require( 'tape' );
-var event_stream = require( 'event-stream' );
+'use strict';
 
-var DocumentStream = require( '../../lib/streams/documentStream' );
+const tape = require( 'tape' );
+const event_stream = require( 'event-stream' );
+
+const DocumentStream = require( '../../lib/streams/documentStream' );
 
 function test_stream(input, testedStream, callback) {
-  var input_stream = event_stream.readArray(input);
-  var destination_stream = event_stream.writeArray(callback);
+  const input_stream = event_stream.readArray(input);
+  const destination_stream = event_stream.writeArray(callback);
 
   input_stream.pipe(testedStream).pipe(destination_stream);
 }
 
 tape( 'documentStream catches records with no street', function(test) {
-  var input = {
+  const input = {
     NUMBER: 5
   };
-  var stats = { badRecordCount: 0 };
-  var documentStream = DocumentStream.create('prefix', stats);
+  const stats = { badRecordCount: 0 };
+  const documentStream = DocumentStream.create('prefix', stats);
 
   test_stream([input], documentStream, function(err, actual) {
     test.equal(actual.length, 0, 'no documents should be pushed' );
@@ -25,15 +27,15 @@ tape( 'documentStream catches records with no street', function(test) {
 });
 
 tape( 'documentStream does not set zipcode if zipcode is emptystring', function(test) {
-  var input = {
+  const input = {
     NUMBER: '5',
     STREET: '101st Avenue',
     LAT: 5,
     LON: 6,
     POSTCODE: ''
   };
-  var stats = { badRecordCount: 0 };
-  var documentStream = DocumentStream.create('prefix', stats);
+  const stats = { badRecordCount: 0 };
+  const documentStream = DocumentStream.create('prefix', stats);
 
   test_stream([input], documentStream, function(err, actual) {
     test.equal(actual.length, 1, 'the document should be pushed' );
@@ -44,16 +46,16 @@ tape( 'documentStream does not set zipcode if zipcode is emptystring', function(
 });
 
 tape( 'documentStream creates id with filename-based prefix', function(test) {
-  var input = {
+  const input = {
     NUMBER: '5',
     STREET: '101st Avenue',
     LAT: 5,
     LON: 6,
     POSTCODE: ''
   };
 
-  var stats = { badRecordCount: 0 };
-  var documentStream = DocumentStream.create('prefix', stats);
+  const stats = { badRecordCount: 0 };
+  const documentStream = DocumentStream.create('prefix', stats);
 
   test_stream([input], documentStream, function(err, actual) {
     test.equal(actual.length, 1, 'the document should be pushed' );
@@ -62,3 +64,23 @@ tape( 'documentStream creates id with filename-based prefix', function(test) {
     test.end();
   });
 });
+
+tape('documentStream uses HASH value if present', function(test) {
+  const input = {
+    NUMBER: '5',
+    STREET: '101st Avenue',
+    LAT: 5,
+    LON: 6,
+    HASH: 'abcd'
+  };
+
+  const stats = { badRecordCount: 0 };
+  const documentStream = DocumentStream.create('prefix', stats);
+
+  test_stream([input], documentStream, function(err, actual) {
+    test.equal(actual.length, 1, 'the document should be pushed' );
+    test.equal(stats.badRecordCount, 0, 'bad record count unchanged');
+    test.equal(actual[0].getId(), 'prefix:abcd');
+    test.end();
+  });
+});