Skip to content

Commit

Permalink
indentation fixing
Browse files Browse the repository at this point in the history
  • Loading branch information
loadit1 committed Feb 26, 2020
1 parent f7a0d30 commit 825b83c
Show file tree
Hide file tree
Showing 3 changed files with 38 additions and 39 deletions.
34 changes: 17 additions & 17 deletions utils/download_data_all.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,44 +2,44 @@ const child_process = require('child_process');
const async = require('async');
const fs = require('fs-extra');
const os = require('os');
const url = require('url');
const path = require('path');
const commandExistsSync = require('command-exists').sync;

const bundles = require('../src/bundleList');
const config = require( 'pelias-config' ).generate(require('../schema'));
const config = require('pelias-config').generate(require('../schema'));

const wofDataHost = config.get('imports.whosonfirst.dataHost') || 'https://dist.whosonfirst.org';

function download(callback) {
//ensure required directory structure exists
// ensure required directory structure exists
fs.ensureDirSync(path.join(config.imports.whosonfirst.datapath, 'meta'));

// download one bundle for every other CPU (tar and bzip2 can both max out one core)
// (the maximum is configurable, to keep things from getting too intense, and defaults to 4)
//lower this number to make the downloader more CPU friendly
//raise this number to (possibly) make it faster
// lower this number to make the downloader more CPU friendly
// raise this number to (possibly) make it faster
const maxSimultaneousDownloads = config.get('imports.whosonfirst.maxDownloads') || 4;
const cpuCount = os.cpus().length;
const simultaneousDownloads = Math.max(maxSimultaneousDownloads, Math.min(1, cpuCount / 2));

// generate a shell command that does the following:
// 1.) use curl to download the bundle, piping directly to tar (this avoids the
// need for intermediate storage of the archive file)
// 2.) extract the archive so that the data directory goes in the right place and
// the README file is ignored (it just would get overridden by subsequent bundles)
// 3.) move the meta file to the meta files directory
function generateCommand(bundle, directory) {
let extract;
//Check if we have lbzip2 installed
if (commandExistsSync('lbzip2')) {
extract = `tar -x --use-compress-program=lbzip2`;
} else {
extract = `tar -xj`;
};

const csvFilename = bundle.replace(/-\d{8}T\d{6}-/, '-latest-') // support timestamped downloads
.replace('.tar.bz2', '.csv');
let extract;
// check if we have lbzip2 installed
if (commandExistsSync('lbzip2')) {
extract = `tar -x --use-compress-program=lbzip2`;
} else {
extract = `tar -xj`;
}

const csvFilename = bundle
.replace(/-\d{8}T\d{6}-/, '-latest-') // support timestamped downloads
.replace('.tar.bz2', '.csv');

return `curl -s ${wofDataHost}/bundles/${bundle} | ${extract} --strip-components=1 --exclude=README.txt -C ` +
`${directory} && mv ${path.join(directory, csvFilename)} ${path.join(directory, 'meta')}`;
Expand Down Expand Up @@ -69,4 +69,4 @@ function download(callback) {
});
}

module.exports.download = download;
module.exports.download = download;
31 changes: 15 additions & 16 deletions utils/download_sqlite_all.js
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ function download(callback) {
const generateSQLites = () => {
const files = {};
const content = JSON.parse(downloadFileSync(`${wofDataHost}/sqlite/inventory.json`))
// Only latest compressed files
// Only latest compressed files
.filter(e => e.name_compressed.indexOf('latest') >= 0)
// Postalcodes only when importPostalcodes is ture and without --admin-only arg
.filter(e => e.name_compressed.indexOf('postalcode') < 0 ||
Expand All @@ -51,20 +51,19 @@ function download(callback) {
const generateCommand = (sqlite, directory) => {
let extract;
if (/\.db\.bz2$/.test(sqlite.name_compressed)) {
//Check if we have lbzip2 installed
if ( commandExistsSync('lbzip2') ) {
extract = `lbzip2`;
} else {
extract = `bunzip2`;
};
} else if(/\.db\.tar\.bz2$/.test(sqlite.name_compressed)) {
//Check if we have lbzip2 installed
if ( commandExistsSync('lbzip2') ) {
//Aim tar to use lbzip2
extract = `tar -xO --use-compress-program=lbzip2`;
} else {
extract = `tar -xjO`;
};
// Check if we have lbzip2 installed
if (commandExistsSync('lbzip2')) {
extract = 'lbzip';
} else {
extract = 'bunzip';
}
} else if (/\.db\.tar\.bz2$/.test(sqlite.name_compressed)) {
// Check if we have lbzip2 installed
if (commandExistsSync('lbzip2')) {
extract = 'tar -xO --use-compress-program=lbzip';
} else {
extract = 'tar -xj';
}
} else {
throw new Error('What is this extension ?!?');
}
Expand All @@ -90,4 +89,4 @@ function download(callback) {
async.parallelLimit(downloadFunctions, simultaneousDownloads, callback);
}

module.exports.download = download;
module.exports.download = download;
12 changes: 6 additions & 6 deletions utils/sqlite_download.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,13 @@ REMOTE_PATH="$3/sqlite/${DB_FILENAME}.bz2"
info() { echo -e "\e[33m[$1]\t\e[0m $2" >&2; }
err() { echo -e "\e[31m[$1]\t\e[0m \e[91m$2\e[0m" >&2; }

#Check if we have lbzip2 (https://lbzip2.org/) installed
# Check if we have lbzip2 (https://lbzip2.org/) installed
decompress_utility() {
if hash lbzip2 2>/dev/null; then
lbzip2 -d -f "${LOCAL_BZ2_PATH}" > "${LOCAL_DB_PATH}"
else
bunzip2 -f "${LOCAL_BZ2_PATH}" > "${LOCAL_DB_PATH}"
fi
if hash lbzip2 2>/dev/null; then
lbzip2 -d -f "${LOCAL_BZ2_PATH}" > "${LOCAL_DB_PATH}"
else
bunzip2 -f "${LOCAL_BZ2_PATH}" > "${LOCAL_DB_PATH}"
fi
}
extract_file() {
info 'whosonfirst-sqlite-decompress' "${LOCAL_BZ2_PATH}"
Expand Down

0 comments on commit 825b83c

Please sign in to comment.