diff --git a/.jshintrc b/.jshintrc index f9df9b9d7..8a7f48c00 100644 --- a/.jshintrc +++ b/.jshintrc @@ -1,4 +1,5 @@ { + "esversion": 6, "node": true, "curly": true, "eqeqeq": true, diff --git a/lib/streams/documentStream.js b/lib/streams/documentStream.js index c33e95ac5..d3ad4d8d1 100644 --- a/lib/streams/documentStream.js +++ b/lib/streams/documentStream.js @@ -1,23 +1,28 @@ -var through = require( 'through2' ); +'use strict'; -var peliasModel = require( 'pelias-model' ); +const through = require( 'through2' ); + +const peliasModel = require( 'pelias-model' ); /* * Create a stream of Documents from valid, cleaned CSV records */ function createDocumentStream(id_prefix, stats) { /** - * Used to track the UID of individual records passing through the stream - * created by `createRecordStream()`. See `peliasModel.Document.setId()` for - * information about UIDs. + * Used to track the UID of individual records passing through the stream if + * there is no HASH that can be used as a more unique identifier. See + * `peliasModel.Document.setId()` for information about UIDs. */ - var uid = 0; + let uid = 0; return through.obj( function write( record, enc, next ){ - var model_id = id_prefix + ':' + uid++; + const id_number = record.HASH || uid; + const model_id = `${id_prefix}:${id_number}`; + uid++; + try { - var addrDoc = new peliasModel.Document( 'openaddresses', 'address', model_id ) + const addrDoc = new peliasModel.Document( 'openaddresses', 'address', model_id ) .setName( 'default', (record.NUMBER + ' ' + record.STREET) ) .setCentroid( { lon: record.LON, lat: record.LAT } ); diff --git a/test/streams/documentStream.js b/test/streams/documentStream.js index a18e9173f..ac6876f2a 100644 --- a/test/streams/documentStream.js +++ b/test/streams/documentStream.js @@ -1,21 +1,23 @@ -var tape = require( 'tape' ); -var event_stream = require( 'event-stream' ); +'use strict'; -var DocumentStream = require( '../../lib/streams/documentStream' ); +const tape = require( 'tape' ); +const event_stream = require( 'event-stream' ); + +const DocumentStream = require( '../../lib/streams/documentStream' ); function test_stream(input, testedStream, callback) { - var input_stream = event_stream.readArray(input); - var destination_stream = event_stream.writeArray(callback); + const input_stream = event_stream.readArray(input); + const destination_stream = event_stream.writeArray(callback); input_stream.pipe(testedStream).pipe(destination_stream); } tape( 'documentStream catches records with no street', function(test) { - var input = { + const input = { NUMBER: 5 }; - var stats = { badRecordCount: 0 }; - var documentStream = DocumentStream.create('prefix', stats); + const stats = { badRecordCount: 0 }; + const documentStream = DocumentStream.create('prefix', stats); test_stream([input], documentStream, function(err, actual) { test.equal(actual.length, 0, 'no documents should be pushed' ); @@ -25,15 +27,15 @@ tape( 'documentStream catches records with no street', function(test) { }); tape( 'documentStream does not set zipcode if zipcode is emptystring', function(test) { - var input = { + const input = { NUMBER: '5', STREET: '101st Avenue', LAT: 5, LON: 6, POSTCODE: '' }; - var stats = { badRecordCount: 0 }; - var documentStream = DocumentStream.create('prefix', stats); + const stats = { badRecordCount: 0 }; + const documentStream = DocumentStream.create('prefix', stats); test_stream([input], documentStream, function(err, actual) { test.equal(actual.length, 1, 'the document should be pushed' ); @@ -44,7 +46,7 @@ tape( 'documentStream does not set zipcode if zipcode is emptystring', function( }); tape( 'documentStream creates id with filename-based prefix', function(test) { - var input = { + const input = { NUMBER: '5', STREET: '101st Avenue', LAT: 5, @@ -52,8 +54,8 @@ tape( 'documentStream creates id with filename-based prefix', function(test) { POSTCODE: '' }; - var stats = { badRecordCount: 0 }; - var documentStream = DocumentStream.create('prefix', stats); + const stats = { badRecordCount: 0 }; + const documentStream = DocumentStream.create('prefix', stats); test_stream([input], documentStream, function(err, actual) { test.equal(actual.length, 1, 'the document should be pushed' ); @@ -62,3 +64,23 @@ tape( 'documentStream creates id with filename-based prefix', function(test) { test.end(); }); }); + +tape('documentStream uses HASH value if present', function(test) { + const input = { + NUMBER: '5', + STREET: '101st Avenue', + LAT: 5, + LON: 6, + HASH: 'abcd' + }; + + const stats = { badRecordCount: 0 }; + const documentStream = DocumentStream.create('prefix', stats); + + test_stream([input], documentStream, function(err, actual) { + test.equal(actual.length, 1, 'the document should be pushed' ); + test.equal(stats.badRecordCount, 0, 'bad record count unchanged'); + test.equal(actual[0].getId(), 'prefix:abcd'); + test.end(); + }); +});