From 05535014752903b80309ec09126b4a05edf4f280 Mon Sep 17 00:00:00 2001 From: Nicky Kimaina Date: Fri, 26 Apr 2019 10:40:46 +0300 Subject: [PATCH] Added build scripts --- build-hiv-summary.js | 52 ++++++++++++++++++++++ cleanup.js | 29 ++++++++++++ config.js | 9 ++++ index.js | 21 +++++++++ package-lock.json | 97 ++++++++++++++++++++++++++++++++++------- package.json | 6 ++- query-runner.js | 84 +++++++++++++++++++++++++++++++++++ schedule-hiv-summary.js | 34 +++++++++++++++ 8 files changed, 315 insertions(+), 17 deletions(-) create mode 100644 build-hiv-summary.js create mode 100644 cleanup.js create mode 100644 config.js create mode 100644 query-runner.js create mode 100644 schedule-hiv-summary.js diff --git a/build-hiv-summary.js b/build-hiv-summary.js new file mode 100644 index 0000000..1896907 --- /dev/null +++ b/build-hiv-summary.js @@ -0,0 +1,52 @@ + +let QueryRunner = require('./query-runner'); + +let queryRunner = new QueryRunner().getInstance(); + +class BuildHivSummary { + + constructor() { + + } + + runJob() { + return new Promise((resolve, reject) => { + let sql = `select count(*) as items_in_queue from etl.flat_hiv_summary_build_queue`; + + queryRunner.runQuery(sql) + .then((result) => { + let items = result.results[0].items_in_queue; + console.log('Items in queue:' + items); + + let batches = items < 50 ? 1 : items / 50; + batches = Math.ceil(batches); + console.log('batches: ' + batches); + + let queries = []; + + for(let i =0; i < batches; i++) { + let qry = `call generate_hiv_summary_v15_7("build",${i},50,20);`; + queries.push(queryRunner.runQuery(qry)); + } + + Promise.all(queries).then((results)=>{ + console.log('finished running all the queries', results); + resolve(results); + }).catch((err)=>{ + reject(err); + console.error('Error running all the queries', err); + }); + + //resolve(parallelism) + }) + .catch((err) => { + // handle error + reject(err); + }); + + }); + } + +} + +module.exports = BuildHivSummary; diff --git a/cleanup.js b/cleanup.js new file mode 100644 index 0000000..d82739c --- /dev/null +++ b/cleanup.js @@ -0,0 +1,29 @@ +// Object to capture process exits and call app specific cleanup function + +function noOp() {}; + +exports.Cleanup = function Cleanup(callback) { + + // attach user callback to the process event emitter + // if no callback, it will still exit gracefully on Ctrl-C + callback = callback || noOp; + process.on('cleanup',callback); + + // do app specific cleaning before exiting + process.on('exit', function () { + process.emit('cleanup'); + }); + + // catch ctrl+c event and exit normally + process.on('SIGINT', function () { + console.log('Ctrl-C...'); + process.exit(2); + }); + + //catch uncaught exceptions, trace, then exit normally + process.on('uncaughtException', function(e) { + console.log('Uncaught Exception...'); + console.log(e.stack); + process.exit(99); + }); +}; \ No newline at end of file diff --git a/config.js b/config.js new file mode 100644 index 0000000..5459c5d --- /dev/null +++ b/config.js @@ -0,0 +1,9 @@ +let config = { + host : 'localhost', + user : 'admin', + password: 'admin', + database: 'test' +}; + + +module.exports = config; \ No newline at end of file diff --git a/index.js b/index.js index e69de29..01f2422 100644 --- a/index.js +++ b/index.js @@ -0,0 +1,21 @@ +let BuildHivSummary = require('./build-hiv-summary'); +let ScheduleHivSummary = require('./schedule-hiv-summary'); +let Moment = require('moment'); +try { + let buildJob = new BuildHivSummary(); + let startedAt = Moment(); + console.log('Starting at:', startedAt.toLocaleString()) + buildJob.runJob().then(() => { + console.log('Finshed all jobs.'); + let endedAt = Moment(); + let diff = endedAt.diff(startedAt, 'seconds'); + console.log('Took ' + diff + ' seconds.'); + }).catch((err) => { + console.error('error running hiv summary job'); + throw err; + }); +} catch (error) { + console.error('Error running pipeline', error); +} + +process.stdin.resume(); diff --git a/package-lock.json b/package-lock.json index 4ee60fd..66934d2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,13 @@ { - "requires": true, + "name": "etl-sync-scripts", + "version": "0.1.0", "lockfileVersion": 1, + "requires": true, "dependencies": { "assertion-error": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", - "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==", - "dev": true + "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==" }, "balanced-match": { "version": "1.0.0", @@ -14,6 +15,11 @@ "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "dev": true }, + "bignumber.js": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-4.1.0.tgz", + "integrity": "sha512-eJzYkFYy9L4JzXsbymsFn3p54D+llV27oTQ+ziJG7WFRheJcNZilgVXMG0LoZtlQSKBsJdWtLFqOD0u+U0jZKA==" + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -34,7 +40,6 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/chai/-/chai-4.2.0.tgz", "integrity": "sha512-XQU3bhBukrOsQCuwZndwGcCVQHyZi53fQ6Ys1Fym7E4olpIqqZZhhoFJoaKVvV17lWQoXYwgWN2nF5crA8J2jw==", - "dev": true, "requires": { "assertion-error": "1.1.0", "check-error": "1.0.2", @@ -47,8 +52,7 @@ "check-error": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", - "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=", - "dev": true + "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=" }, "commander": { "version": "2.15.1", @@ -62,6 +66,11 @@ "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", "dev": true }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, "debug": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", @@ -75,7 +84,6 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz", "integrity": "sha512-+QeIQyN5ZuO+3Uk5DYh6/1eKO0m0YmJFGNmFHGACpf1ClL1nmlV/p4gNgbl2pJGxgXb4faqo6UE+M5ACEMyVcw==", - "dev": true, "requires": { "type-detect": "4.0.8" } @@ -101,8 +109,7 @@ "get-func-name": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", - "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=", - "dev": true + "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=" }, "glob": { "version": "7.1.2", @@ -149,8 +156,12 @@ "inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", - "dev": true + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, "minimatch": { "version": "3.0.4", @@ -195,12 +206,28 @@ "supports-color": "5.4.0" } }, + "moment": { + "version": "2.24.0", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", + "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==" + }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", "dev": true }, + "mysql": { + "version": "2.16.0", + "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.16.0.tgz", + "integrity": "sha512-dPbN2LHonQp7D5ja5DJXNbCLe/HRdu+f3v61aguzNRQIrmZLOeRoymBYyeThrR6ug+FqzDL95Gc9maqZUJS+Gw==", + "requires": { + "bignumber.js": "4.1.0", + "readable-stream": "2.3.6", + "safe-buffer": "5.1.2", + "sqlstring": "2.3.1" + } + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -219,8 +246,44 @@ "pathval": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz", - "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=", - "dev": true + "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=" + }, + "process-nextick-args": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "1.0.0", + "process-nextick-args": "2.0.0", + "safe-buffer": "5.1.2", + "string_decoder": "1.1.1", + "util-deprecate": "1.0.2" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "sqlstring": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz", + "integrity": "sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A=" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "5.1.2" + } }, "supports-color": { "version": "5.4.0", @@ -234,8 +297,12 @@ "type-detect": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", - "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", - "dev": true + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==" + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, "wrappy": { "version": "1.0.2", diff --git a/package.json b/package.json index 5ba030e..1a28492 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "etl-sync-scripts", - "version": "1.0.0", + "version": "0.1.0", "description": "", "main": "index.js", "directories": { @@ -9,7 +9,9 @@ "test": "test" }, "dependencies": { - "chai": "^4.2.0" + "chai": "^4.2.0", + "moment": "^2.24.0", + "mysql": "^2.16.0" }, "devDependencies": { "mocha": "^5.2.0" diff --git a/query-runner.js b/query-runner.js new file mode 100644 index 0000000..0a9390b --- /dev/null +++ b/query-runner.js @@ -0,0 +1,84 @@ +let mysql = require('mysql'); + +let config = require('./config.js'); + +let cleanup = require('./cleanup') + +class QueryRunner { + + constructor() { + this.pool = mysql.createPool(config); + let self = this; + cleanup.Cleanup(()=> { + this.releasePool(self.pool); + }); + } + + runQuery(sqlStatement) { + console.log('Executing: ' + sqlStatement); + let self = this; + return new Promise((resolve, reject) => { + self.pool.getConnection((err, connection) => { + if (err) { + reject(err); + } // not connected! + + // Use the connection + connection.query(sqlStatement, function (error, results, fields) { + // When done with the connection, release it. + connection.release(); + + // Handle error after the release. + if (error) { + console.error('Error running sql: ' + sqlStatement, error); + + // Error handling through repsonse object, to allow others to run to completion. + resolve({ + error: error + }); + } else { + let toReturn = { + results: results, + fields: fields + } + + resolve(toReturn); + + } + + + // Don't use the connection here, it has been returned to the pool. + }); + }); + }); + + } + + releasePool(pool) { + console.log('Closing pool'); + pool.end(function (err) { + if (err) { + console.error('Could not end all conections', err); + } else { + console.log('Ended mysql connection pool'); + } + }); + } +} + +class Singleton { + + constructor() { + if (!Singleton.instance) { + Singleton.instance = new QueryRunner(); + } + } + + getInstance() { + return Singleton.instance; + } + +} + +module.exports = Singleton; + diff --git a/schedule-hiv-summary.js b/schedule-hiv-summary.js new file mode 100644 index 0000000..1f28164 --- /dev/null +++ b/schedule-hiv-summary.js @@ -0,0 +1,34 @@ +let QueryRunner = require('./query-runner'); + +let queryRunner = new QueryRunner().getInstance(); + +class ScheduleHivSummary { + + constructor() { + + } + + runJob() { + return new Promise((resolve, reject) => { + let sql = `CALL schedule_hiv_summary(Now())`; + + queryRunner.runQuery(sql) + .then((result) => { + if(result.results) { + console.log('scheduled: ' + JSON.stringify(result.results)); + } else { + console.log('Error scheduling hiv summary'); + } + resolve(result); + }) + .catch((err) => { + // handle error + reject(err); + }); + + }); + } + +} + +module.exports = ScheduleHivSummary;