Merge pull request #207 from pelias/cleanup-data-file-globbing

Cleanup data file globbing
pelias · Jan 4, 2017 · 1fe9205 · 1fe9205
2 parents 898c64e + 821e4cf
commit 1fe9205
Show file tree

Hide file tree

Showing 2 changed files with 82 additions and 5 deletions.
diff --git a/lib/parameters.js b/lib/parameters.js
@@ -2,6 +2,7 @@ var fs = require( 'fs' );
 var util = require( 'util' );
 var glob = require( 'glob' );
 var path = require( 'path' );
+var _ = require('lodash');
 
 var minimist = require( 'minimist' );
 
@@ -91,13 +92,17 @@ function interpretUserArgs( argv, config ){
 }
 
 function getFullFileList(peliasConfig, args) {
-  var configFiles = peliasConfig.imports.openaddresses? peliasConfig.imports.openaddresses.files : undefined;
-  if (configFiles !== undefined && configFiles.length > 0) {
-    return configFiles.map(function(file) {
+  // get the files to process
+  const files = _.get(peliasConfig.imports.openaddresses, 'files', []);
+
+  if (_.isEmpty(files)) {
+    // no specific files listed, so return all .csv files
+    return glob.sync( args.dirPath + '/**/*.csv' );
+  } else {
+    // otherwise return the requested files with full path
+    return files.map(function(file) {
       return path.join(args.dirPath, file);
     });
-  } else {
-    return glob.sync( args.dirPath + '/**/*.csv' );
   }
 }
 

diff --git a/test/parameters.js b/test/parameters.js
@@ -1,5 +1,6 @@
 var tape = require( 'tape' );
 var path = require( 'path' );
+var fs = require('fs');
 
 var temp = require( 'temp' ).track();
 
@@ -94,6 +95,77 @@ tape('interpretUserArgs returns normalized path from config', function(test) {
   });
 });
 
+tape('getFileList returns all .csv path names when config has empty files list', function(test) {
+  temp.mkdir('multipleFiles', function(err, temp_dir) {
+    // add some files to the data path to be globbed
+    fs.mkdirSync(path.join(temp_dir, 'dirA'));
+    fs.writeFileSync(path.join(temp_dir, 'dirA', 'fileA.csv'), '');
+
+    fs.mkdirSync(path.join(temp_dir, 'dirB'));
+    fs.writeFileSync(path.join(temp_dir, 'dirB', 'fileB.csv'), '');
+
+    fs.writeFileSync(path.join(temp_dir, 'fileC.csv'), '');
+
+    // should not be included since it's not a .csv file
+    fs.writeFileSync(path.join(temp_dir, 'fileD.txt'), '');
+
+    var peliasConfig = {
+      imports: {
+        openaddresses: {
+          files: []
+        }
+      }
+    };
+    var args = {
+      dirPath: temp_dir
+    };
+
+    var actual = parameters.getFileList(peliasConfig, args);
+
+    test.equal(actual.length, 3);
+    test.ok(actual.find((f) => f === path.join(temp_dir, 'dirA', 'fileA.csv')));
+    test.ok(actual.find((f) => f === path.join(temp_dir, 'dirB', 'fileB.csv')));
+    test.ok(actual.find((f) => f === path.join(temp_dir, 'fileC.csv')));
+    test.end();
+
+  });
+});
+
+tape('getFileList returns all .csv path names when config doesn\'t have files property', function(test) {
+  temp.mkdir('multipleFiles', function(err, temp_dir) {
+    // add some files to the data path to be globbed
+    fs.mkdirSync(path.join(temp_dir, 'dirA'));
+    fs.writeFileSync(path.join(temp_dir, 'dirA', 'fileA.csv'), '');
+
+    fs.mkdirSync(path.join(temp_dir, 'dirB'));
+    fs.writeFileSync(path.join(temp_dir, 'dirB', 'fileB.csv'), '');
+
+    fs.writeFileSync(path.join(temp_dir, 'fileC.csv'), '');
+
+    // should not be included since it's not a .csv file
+    fs.writeFileSync(path.join(temp_dir, 'fileD.txt'), '');
+
+    var peliasConfig = {
+      imports: {
+        openaddresses: {
+        }
+      }
+    };
+    var args = {
+      dirPath: temp_dir
+    };
+
+    var actual = parameters.getFileList(peliasConfig, args);
+
+    test.equal(actual.length, 3);
+    test.ok(actual.find((f) => f === path.join(temp_dir, 'dirA', 'fileA.csv')));
+    test.ok(actual.find((f) => f === path.join(temp_dir, 'dirB', 'fileB.csv')));
+    test.ok(actual.find((f) => f === path.join(temp_dir, 'fileC.csv')));
+    test.end();
+
+  });
+});
+
 tape('getFileList returns fully qualified path names when config has a files list', function(test) {
   temp.mkdir('multipleFiles', function(err, temporary_dir) {
     var peliasConfig = {