From 36337fdfa4ef89eb61506dedd4a0f57cb49e303a Mon Sep 17 00:00:00 2001
From: Joxit <joxit972@gmail.com>
Date: Mon, 9 Mar 2020 11:59:52 +0100
Subject: [PATCH] feat(sqlite): supports for multiple databases

---
 src/pip/index.js      |  6 +++---
 src/pip/readStream.js | 28 ++++++++++++++++++++++------
 2 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/src/pip/index.js b/src/pip/index.js
index 203b600..436e2f7 100644
--- a/src/pip/index.js
+++ b/src/pip/index.js
@@ -46,9 +46,9 @@ module.exports.create = function createPIPService(datapath, layers, localizedAdm
   layers = _.intersection(defaultLayers, _.isEmpty(layers) ? defaultLayers : layers);
 
   if (isSqlite === true) {
-    const filename = path.join(datapath, 'sqlite', 'whosonfirst-data-latest.db');
-    if (!fs.existsSync(filename)) {
-      return callback(`unable to locate sqlite file ${filename}`);
+    const folder = path.join(datapath, 'sqlite');
+    if (!fs.existsSync(folder)) {
+      return callback(`unable to locate sqlite folder`);
     }
   } else {
     // keep track of any missing metafiles for later reporting and error conditions
diff --git a/src/pip/readStream.js b/src/pip/readStream.js
index 5aa812a..19ca4c2 100644
--- a/src/pip/readStream.js
+++ b/src/pip/readStream.js
@@ -7,8 +7,12 @@ const simplifyGeometry = require('./components/simplifyGeometry');
 const filterOutCitylessNeighbourhoods = require('./components/filterOutCitylessNeighbourhoods');
 const filterOutHierarchylessNeighbourhoods = require('./components/filterOutHierarchylessNeighbourhoods');
 const filterOutPointRecords = require('./components/filterOutPointRecords');
+const combinedStream = require('combined-stream');
+const fs = require('fs');
 const SQLiteStream = whosonfirst.SQLiteStream;
 
+const SQLITE_REGEX = /whosonfirst-data-[a-z0-9-]+\.db$/;
+
 function readBundleRecords(datapath, layer) {
   return whosonfirst.metadataStream(datapath).create(layer)
     .pipe(whosonfirst.parseMetaFiles())
@@ -17,13 +21,25 @@ function readBundleRecords(datapath, layer) {
     .pipe(whosonfirst.loadJSON(datapath, false));
 }
 
+function getSqliteFilePaths(root) {
+  return fs.readdirSync(root)
+    .filter(d => SQLITE_REGEX.test(d))
+    .map(db => path.join(root, db));
+}
+
 function readSqliteRecords(datapath, layer) {
-  return new SQLiteStream(
-    path.join(datapath, 'sqlite', 'whosonfirst-data-latest.db'),
-    config.importPlace ?
-    SQLiteStream.findGeoJSONByPlacetypeAndWOFId(layer, config.importPlace) :
-    SQLiteStream.findGeoJSONByPlacetype(layer)
-  ).pipe(whosonfirst.toJSONStream());
+  const sqliteStream = combinedStream.create();
+  getSqliteFilePaths(path.join(datapath, 'sqlite')).forEach(dbPath => {
+    sqliteStream.append(next => {
+      next(new SQLiteStream(
+        dbPath,
+        config.importPlace ?
+        SQLiteStream.findGeoJSONByPlacetypeAndWOFId(layer, config.importPlace) :
+        SQLiteStream.findGeoJSONByPlacetype(layer)
+      ));
+    });
+  });
+  return sqliteStream.pipe(whosonfirst.toJSONStream());
 }
 
 /**