Skip to content
Permalink

Comparing changes

This is a direct comparison between two commits made in this repository or its related repositories. View the default comparison for this range or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: pelias/placeholder
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: c1eb8597f7a8d17f04269d99eb9ed145f3b96d1d
Choose a base ref
..
head repository: pelias/placeholder
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: 9495956a8f674bf5d1f550a08bb367e244964782
Choose a head ref
Showing with 32 additions and 26 deletions.
  1. +1 −1 .travis.yml
  2. +1 −1 Dockerfile
  3. +5 −1 cmd/download_extract.sh
  4. +14 −4 cmd/wof_extract_sqlite.js
  5. +0 −8 docker-compose.yml
  6. +7 −7 lib/DocStore.js
  7. +2 −2 lib/TokenIndex.js
  8. +1 −1 package.json
  9. +1 −1 test/lib/TokenIndex.js
2 changes: 1 addition & 1 deletion .travis.yml
View file
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@ matrix:
fast_finish: true
script: npm run travis
env:
- BUCKET=https://storage.googleapis.com/pelias-data.geocode.earth/placeholder
- BUCKET=https://data.geocode.earth/placeholder/2019-01-28
before_install:
- npm i -g npm
before_script:
2 changes: 1 addition & 1 deletion Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -3,7 +3,7 @@ FROM pelias/baseimage

# downloader apt dependencies
# note: this is done in one command in order to keep down the size of intermediate containers
RUN apt-get update && apt-get install -y build-essential python jq && rm -rf /var/lib/apt/lists/*
RUN apt-get update && apt-get install -y build-essential python jq lbzip2 parallel && rm -rf /var/lib/apt/lists/*

# change working dir
ENV WORKDIR /code/pelias/placeholder
6 changes: 5 additions & 1 deletion cmd/download_extract.sh
View file
Original file line number Diff line number Diff line change
@@ -11,7 +11,11 @@ PLACETYPES=( 'neighbourhood' 'macrohood' 'borough' 'locality' 'localadmin' 'coun
# download and extract fields from contents of tar
function extract {
curl -so "/tmp/wof-${1}-latest-bundle.tar.bz2" "https://whosonfirst.mapzen.com/bundles/wof-${1}-latest-bundle.tar.bz2"
tar --wildcards '*.geojson' -jx --to-command 'jq -cMf "${DIR}/jq.filter"' -f "/tmp/wof-${1}-latest-bundle.tar.bz2"
if hash lbzip2 2>/dev/null; then
tar --wildcards '*.geojson' -x --use-compress-program=lbzip2 --to-command 'jq -cMf "${DIR}/jq.filter"' -f "/tmp/wof-${1}-latest-bundle.tar.bz2"
else
tar --wildcards '*.geojson' -jx --to-command 'jq -cMf "${DIR}/jq.filter"' -f "/tmp/wof-${1}-latest-bundle.tar.bz2"
fi
rc=$?; if [[ $rc != 0 ]]; then
>&2 echo "/tmp/wof-${1}-latest-bundle.tar.bz2"
>&2 echo "command exited with status: $rc"
18 changes: 14 additions & 4 deletions cmd/wof_extract_sqlite.js
View file
Original file line number Diff line number Diff line change
@@ -4,6 +4,9 @@ const whosonfirst = require('pelias-whosonfirst');
const SQLiteStream = whosonfirst.SQLiteStream;
const through = require('through2');
const Placeholder = require('../Placeholder');
const combinedStream = require('combined-stream');

const SQLITE_REGEX = /whosonfirst-data-[a-z0-9-]+\.db$/;

const WOF_DIR = process.env.WOF_DIR || '/data/whosonfirst-data/sqlite';

@@ -39,10 +42,17 @@ const output = () => {
}
};

new SQLiteStream(
path.join(WOF_DIR, 'whosonfirst-data-latest.db'),
SQLiteStream.findGeoJSONByPlacetype(layers)
)
const sqliteStream = combinedStream.create();
fs.readdirSync(WOF_DIR)
.filter(file => SQLITE_REGEX.test(file))
.map(file => path.join(WOF_DIR, file))
.forEach(dbPath => {
sqliteStream.append(next => {
next(new SQLiteStream(dbPath, SQLiteStream.findGeoJSONByPlacetype(layers)));
});
});

sqliteStream
.pipe(whosonfirst.toJSONStream())
.pipe(through.obj((row, _, next) => {
Object.keys(row.properties)
8 changes: 0 additions & 8 deletions docker-compose.yml
View file

This file was deleted.

14 changes: 7 additions & 7 deletions lib/DocStore.js
View file
Original file line number Diff line number Diff line change
@@ -47,13 +47,13 @@ DocStore.prototype.checkSchema = function(){
{ cid: 1, name: 'json', type: 'TEXT', notnull: 0, dflt_value: null, pk: 0 }
]);
Database.assertSchema(this.db, 'rtree', [
{ cid: 0, name: 'id', type: '', notnull: 0, dflt_value: null, pk: 0 },
{ cid: 1, name: 'minX', type: '', notnull: 0, dflt_value: null, pk: 0 },
{ cid: 2, name: 'maxX', type: '', notnull: 0, dflt_value: null, pk: 0 },
{ cid: 3, name: 'minY', type: '', notnull: 0, dflt_value: null, pk: 0 },
{ cid: 4, name: 'maxY', type: '', notnull: 0, dflt_value: null, pk: 0 },
{ cid: 5, name: 'minZ', type: '', notnull: 0, dflt_value: null, pk: 0 },
{ cid: 6, name: 'maxZ', type: '', notnull: 0, dflt_value: null, pk: 0 }
{ cid: 0, name: 'id', type: 'INT', notnull: 0, dflt_value: null, pk: 0 },
{ cid: 1, name: 'minX', type: 'NUM', notnull: 0, dflt_value: null, pk: 0 },
{ cid: 2, name: 'maxX', type: 'NUM', notnull: 0, dflt_value: null, pk: 0 },
{ cid: 3, name: 'minY', type: 'NUM', notnull: 0, dflt_value: null, pk: 0 },
{ cid: 4, name: 'maxY', type: 'NUM', notnull: 0, dflt_value: null, pk: 0 },
{ cid: 5, name: 'minZ', type: 'NUM', notnull: 0, dflt_value: null, pk: 0 },
{ cid: 6, name: 'maxZ', type: 'NUM', notnull: 0, dflt_value: null, pk: 0 }
]);
};

4 changes: 2 additions & 2 deletions lib/TokenIndex.js
View file
Original file line number Diff line number Diff line change
@@ -51,8 +51,8 @@ TokenIndex.prototype.checkSchema = function(){
};

TokenIndex.prototype.populate = function(){
this.db.exec('INSERT INTO fulltext(rowid, token) SELECT rowid, REPLACE(token," ","_") FROM tokens;');
this.db.exec('INSERT INTO fulltext(fulltext) VALUES(\'optimize\');');
this.db.exec(`INSERT INTO fulltext(rowid, token) SELECT rowid, REPLACE(token,' ','_') FROM tokens;`);
this.db.exec(`INSERT INTO fulltext(fulltext) VALUES('optimize');`);
};

TokenIndex.prototype.setLineage = function( id, pids, cb ){
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
@@ -34,7 +34,7 @@
"homepage": "https://github.com/pelias/placeholder#readme",
"dependencies": {
"async": "^3.0.1",
"better-sqlite3": "^5.0.0",
"better-sqlite3": "^6.0.1",
"express": "^4.15.2",
"lodash": "^4.17.4",
"lower-case": "^2.0.0",
2 changes: 1 addition & 1 deletion test/lib/TokenIndex.js
View file
Original file line number Diff line number Diff line change
@@ -42,7 +42,7 @@ module.exports.reset = function(test, common) {
]);

// ensure fts table has been created with the correct options
sql = 'select * from sqlite_master where type="table" and name="fulltext"';
sql = `SELECT * FROM sqlite_master WHERE type='table' AND name='fulltext'`;
const expected =
'CREATE VIRTUAL TABLE fulltext USING fts5( token, ' + [
`tokenize="unicode61 remove_diacritics 0 tokenchars '_'"`,