From 46644f2e3a654ef4662cc6f8260670cc65e13a62 Mon Sep 17 00:00:00 2001 From: Evans Dianga Date: Fri, 17 Jan 2025 11:14:45 +0300 Subject: [PATCH] feat(editor): add date range filtering for CSV downloads Refs Tangerine-Community/Tangerine#ticket number - Updated components to support selecting a date range (from year/month to year/month). - Modified services to handle new date range parameters. - Adjusted backend routes and scripts to process date range for CSV generation. - Added validation to ensure correct date range selection. Refs Tangerine-Community/Tangerine#3649 --- .../download-csv/download-csv.component.ts | 16 +++++--- .../new-csv-data-set.component.html | 39 ++++++++++++++++--- .../new-csv-data-set.component.ts | 25 +++++++++--- .../src/app/groups/services/groups.service.ts | 10 ++--- server/src/modules/csv/views.js | 5 +-- server/src/routes/group-csv.js | 31 ++++++++++----- .../src/scripts/generate-csv-data-set/bin.js | 21 +++++----- .../generate-csv-data-set.js | 23 +++++++---- .../generate-csv-data-sets.js | 2 +- server/src/scripts/generate-csv/batch.js | 10 +++-- server/src/scripts/generate-csv/bin.js | 13 ++++--- 11 files changed, 134 insertions(+), 61 deletions(-) diff --git a/editor/src/app/groups/download-csv/download-csv.component.ts b/editor/src/app/groups/download-csv/download-csv.component.ts index 707eee6b9b..e1472079a5 100644 --- a/editor/src/app/groups/download-csv/download-csv.component.ts +++ b/editor/src/app/groups/download-csv/download-csv.component.ts @@ -17,8 +17,10 @@ export class DownloadCsvComponent implements OnInit, OnDestroy { months = []; years = []; - selectedMonth = '*'; - selectedYear = '*'; + fromYear = '*'; + fromMonth = '*'; + toYear = '*'; + toMonth = '*'; processing = false; stateUrl; downloadUrl; @@ -63,13 +65,17 @@ export class DownloadCsvComponent implements OnInit, OnDestroy { } async process() { - if ((this.selectedMonth === '*' && this.selectedYear !== '*') || (this.selectedMonth !== '*' && this.selectedYear === '*')) { - alert('You must choose a month and a year.') + if ((this.fromMonth === '*' && this.fromYear !== '*') || (this.fromMonth !== '*' && this.fromYear === '*')) { + alert('You must choose a start month and year.') + return + } + if ((this.toMonth === '*' && this.toYear !== '*') || (this.toMonth !== '*' && this.toYear === '*')) { + alert('You must choose an end month and year.') return } this.processing = true try { - const result: any = await this.groupsService.downloadCSV(this.groupName, this.formId, this.selectedYear, this.selectedMonth, this.excludePII); + const result: any = await this.groupsService.downloadCSV(this.groupName, this.formId, this.fromYear, this.fromMonth, this.toYear, this.toMonth, this.excludePII); this.stateUrl = result.stateUrl; this.downloadUrl = result.downloadUrl; // TODO call download status immediately then after every few second, Probably use RXJS to ensure we only use the latest values diff --git a/editor/src/app/groups/new-csv-data-set/new-csv-data-set.component.html b/editor/src/app/groups/new-csv-data-set/new-csv-data-set.component.html index ad289b7300..4fd61b32d2 100644 --- a/editor/src/app/groups/new-csv-data-set/new-csv-data-set.component.html +++ b/editor/src/app/groups/new-csv-data-set/new-csv-data-set.component.html @@ -6,23 +6,50 @@
-
+
+

From:

{{ "Month" | translate }} All months - {{ + {{ month }} {{ "Year" | translate }} - + + All years + {{ + year + }} + + +
+
+

To:

+
+ + {{ "Month" | translate }} + + All months + {{ + month + }} + + + + {{ "Year" | translate }} + All years {{ year @@ -100,7 +127,7 @@
- +
diff --git a/editor/src/app/groups/new-csv-data-set/new-csv-data-set.component.ts b/editor/src/app/groups/new-csv-data-set/new-csv-data-set.component.ts index b22cff545c..3595bc437f 100644 --- a/editor/src/app/groups/new-csv-data-set/new-csv-data-set.component.ts +++ b/editor/src/app/groups/new-csv-data-set/new-csv-data-set.component.ts @@ -27,8 +27,10 @@ export class NewCsvDataSetComponent implements OnInit { months = [] years = [] description - selectedMonth = '*' - selectedYear = '*' + fromYear = '*' + fromMonth = '*' + toYear = '*' + toMonth = '*' selectedForms = [] allFormsSelected = false groupId = '' @@ -110,12 +112,16 @@ export class NewCsvDataSetComponent implements OnInit { const forms = this.selectedForms .map(formId => this.templateSelections[formId] ? `${formId}:${this.templateSelections[formId]}` : formId) .toString() - if ((this.selectedMonth === '*' && this.selectedYear !== '*') || (this.selectedMonth !== '*' && this.selectedYear === '*')) { - alert('You must choose a month and a year.') + if ((this.fromMonth === '*' && this.fromYear !== '*') || (this.fromMonth !== '*' && this.fromYear === '*')) { + alert('You must choose a start month and year.') + return + } + if ((this.toMonth === '*' && this.toMonth !== '*') || (this.toMonth !== '*' && this.toMonth === '*')) { + alert('You must choose an end month and year.') return } try { - const result: any = await this.groupsService.downloadCSVDataSet(this.groupId, forms, this.selectedYear, this.selectedMonth, this.description, this.excludePII); + const result: any = await this.groupsService.downloadCSVDataSet(this.groupId, forms, this.fromYear, this.fromMonth, this.toYear, this. toMonth, this.description, this.excludePII); this.stateUrl = result.stateUrl; this.router.navigate(['../', result.id], { relativeTo: this.route }) } catch (error) { @@ -140,4 +146,13 @@ export class NewCsvDataSetComponent implements OnInit { } } + canSubmit(){ + return ( + this.selectedForms.length < 1 || + this.toYear < this.fromYear || + (this.fromYear === "*" && this.toYear !== "*")|| + (`${this.fromYear}:${this.fromMonth}` > `${this.toYear}:${this.toMonth}`) + ); + } + } diff --git a/editor/src/app/groups/services/groups.service.ts b/editor/src/app/groups/services/groups.service.ts index efb25b5809..055640c9ed 100644 --- a/editor/src/app/groups/services/groups.service.ts +++ b/editor/src/app/groups/services/groups.service.ts @@ -106,20 +106,20 @@ export class GroupsService { } } - async downloadCSV(groupName: string, formId: string, selectedYear = '*', selectedMonth = '*', excludePII: boolean) { + async downloadCSV(groupName: string, formId: string, fromYear = '*', fromMonth = '*', toYear = '*', toMonth = '*', excludePII: boolean) { let sanitized = '' if (excludePII) { sanitized = '-sanitized' } try { - if (selectedMonth === '*' || selectedYear === '*') { + if (fromYear === '*' || fromMonth === '*') { const result = await this.httpClient .get(`/api/csv${sanitized}/${groupName}/${formId}`) .toPromise(); return result; } else { const result = await this.httpClient - .get(`/api/csv${sanitized}/${groupName}/${formId}/${selectedYear}/${selectedMonth}`) + .get(`/api/csv${sanitized}/${groupName}/${formId}/${fromYear}/${fromMonth}/${toYear}/${toMonth}`) .toPromise(); return result; } @@ -129,11 +129,11 @@ export class GroupsService { } } } - async downloadCSVDataSet(groupName: string, formIds: string, selectedYear = '*', selectedMonth = '*', description: string,excludePII: boolean) { + async downloadCSVDataSet(groupName: string, formIds: string, fromYear = '*', fromMonth = '*', toYear = '*', toMonth = '*', description: string,excludePII: boolean) { try { let sanitized = excludePII? '-sanitized' : '' const result = await this.httpClient - .post(`/api/create/csvDataSet${sanitized}/${groupName}`, {formIds, description,selectedMonth,selectedYear}) + .post(`/api/create/csvDataSet${sanitized}/${groupName}`, {formIds, description,fromYear, fromMonth, toYear, toMonth}) .toPromise(); console.log(result) return result; diff --git a/server/src/modules/csv/views.js b/server/src/modules/csv/views.js index 50762611fc..dc39cf53a1 100644 --- a/server/src/modules/csv/views.js +++ b/server/src/modules/csv/views.js @@ -5,10 +5,9 @@ module.exports = { resultsByGroupFormId: { map: function (doc) { if (doc.formId) { - const MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; const startUnixtime = new Date(doc.startUnixtime); - const key = doc.formId + '_' + startUnixtime.getFullYear() + '_' + MONTHS[startUnixtime.getMonth()]; - //The emmitted value is in the form "formId" i.e `formId` and also "formId_2018_May" i.e `formId_Year_Month` + const key = doc.formId + '_' + startUnixtime.getFullYear() + '_' + startUnixtime.getMonth();// getMonth returns 0-based index i.e 0,1,2,3,4,5,6,7,8,9,10,11 + //The emitted value is in the form "formId" i.e `formId` and also "formId_2018_0" for Jan i.e `formId_Year_Month` Remember Javascript uses 0-based indexing for months emit(doc.formId); emit(key); } diff --git a/server/src/routes/group-csv.js b/server/src/routes/group-csv.js index 1e090f2a26..50184d4308 100644 --- a/server/src/routes/group-csv.js +++ b/server/src/routes/group-csv.js @@ -58,8 +58,15 @@ const generateCSV = async (req, res) => { const sleepTimeBetweenBatches = 0 let cmd = `cd /tangerine/server/src/scripts/generate-csv/ && ./bin.js ${dbName} ${formId} "${outputPath}" ${batchSize} ${sleepTimeBetweenBatches}` - if (req.params.year && req.params.month) { - cmd += ` ${sanitize(req.params.year)} ${sanitize(req.params.month)}` + if (req.params.fromYear && req.params.fromMonth) { + cmd += ` ${sanitize(req.params.fromYear)} ${sanitize(req.params.fromMonth)}` + } else{ + cmd += ` '' '' ` + } + if(req.params.toYear && req.params.toMonth) { + cmd += ` ${sanitize(req.params.toYear)} ${sanitize(req.params.toMonth)}` + } else{ + cmd += ` '' '' ` } log.info(`generating csv start: ${cmd}`) exec(cmd).then(status => { @@ -77,7 +84,7 @@ const generateCSVDataSet = async (req, res) => { const groupId = sanitize(req.params.groupId) // A list of formIds will be too long for sanitize's limit of 256 bytes so we split, map with sanitize, and join. const formIds = req.body.formIds.split(',').map(formId => formId).join(',') - const {selectedYear, selectedMonth, description} = req.body + const {fromYear, fromMonth, toYear, toMonth, description} = req.body const http = await getUser1HttpInterface() const group = (await http.get(`/nest/group/read/${groupId}`)).data const groupLabel = group.label.replace(/[&\/\\#,+()$~%.'":*?<>{}]/g, '') @@ -86,10 +93,10 @@ const generateCSVDataSet = async (req, res) => { } const fileName = `${sanitize(groupLabel, options)}-${Date.now()}.zip`.replace(/[&\/\\#,+()$~%'":*?<>^{}_ ]+/g, '_') let outputPath = `/csv/${fileName.replace(/[&\/\\#,+()$~%'":*?<>^{}_ ]+/g, '_')}` - let cmd = `cd /tangerine/server/src/scripts/generate-csv-data-set/ && ./bin.js ${groupId} ${formIds} ${outputPath} ${selectedYear === '*' ? `'*'` : sanitize(selectedYear)} ${selectedMonth === '*' ? `'*'` : sanitize(selectedMonth)} ${req.originalUrl.includes('-sanitized') ? '--sanitized': ''}` - log.info(`generating csv start: ${cmd}`) + let cmd = `cd /tangerine/server/src/scripts/generate-csv-data-set/ && ./bin.js ${groupId} ${formIds} ${outputPath} ${fromYear === '*' ? `'*'` : sanitize(fromYear)} ${fromMonth === '*' ? `'*'` : sanitize(fromMonth)} ${toYear === '*' ? `'*'` : sanitize(toYear)} ${toMonth === '*' ? `'*'` : sanitize(toMonth)} ${req.originalUrl.includes('-sanitized') ? '--sanitized': ''}` + log.info(`generating csv start: ${cmd}\n`) exec(cmd).then(status => { - log.info(`generate csv done: ${JSON.stringify(status)} ${outputPath}`) + log.info(`generate csv done: ${JSON.stringify(status)} ${outputPath}\n`) }).catch(error => { log.error(error) }) @@ -102,8 +109,10 @@ const generateCSVDataSet = async (req, res) => { stateUrl, downloadUrl, description, - year: selectedYear, - month: selectedMonth, + fromYear, + fromMonth, + toYear, + toMonth, dateCreated: Date.now() }) res.send({ @@ -223,8 +232,10 @@ const getDataset = async (datasetId) => { stateExists, excludePii, description: result.description, - month: result.month, - year: result.year, + fromYear: result.fromYear, + fromMonth: result.fromMonth, + toYear: result.toYear, + toMonth: result.toMonth, downloadUrl: result.downloadUrl, fileName: result.fileName, dateCreated: result.dateCreated, diff --git a/server/src/scripts/generate-csv-data-set/bin.js b/server/src/scripts/generate-csv-data-set/bin.js index d645fae945..9960ff33f4 100755 --- a/server/src/scripts/generate-csv-data-set/bin.js +++ b/server/src/scripts/generate-csv-data-set/bin.js @@ -5,10 +5,11 @@ const generateCsvDataSet = require('./generate-csv-data-set.js') if (process.argv[2] === '--help') { console.log('Usage:') - console.log(' generate-csv-data-set [--exclude-pii] [--excludeArchivedForms] [--excludeUserProfileAndReports]') + console.log(' generate-csv-data-set [--exclude-pii] [--excludeArchivedForms] [--excludeUserProfileAndReports]') console.log('Examples:') - console.log(` generate-csv-data-set group-abdc form1,form2 ./output.csv 2018 Jan --exclude-pii`) - console.log(` generate-csv-data-set group-abdc form1,form2 ./output.csv * * --exclude-pii`) + console.log(` generate-csv-data-set group-abdc form1,form2 ./output.csv 2018 4 2019 1 --exclude-pii`) + console.log(`'Remember Javascript uses 0-based indexing for months) i.e January is 0, February is 1, December is 11 etc.'`) + console.log(` generate-csv-data-set group-abdc form1,form2 ./output.csv * * * * --exclude-pii`) process.exit() } @@ -16,17 +17,19 @@ const params = { dbName: process.argv[2], formIds: process.argv[3].split(','), outputPath: process.argv[4], - year: (process.argv[5]) ? process.argv[5] : null, - month: (process.argv[6]) ? process.argv[6] : null, - excludePii: process.argv[7] ? true : false, - excludeArchivedForms: process.argv[8] ? true : false, - excludeUserProfileAndReports: process.argv[9] ? true : false + fromYear: (process.argv[5]) ? process.argv[5] : null, + fromMonth: (process.argv[6]) ? process.argv[6] : null, + toYear: (process.argv[7]) ? process.argv[7] : null, + toMonth: (process.argv[8]) ? process.argv[8] : null, + excludePii: process.argv[9] ? true : false, + excludeArchivedForms: process.argv[10] ? true : false, + excludeUserProfileAndReports: process.argv[11] ? true : false } async function go(params) { try { log.debug("generateCsvDataSet bin.js") - await generateCsvDataSet(params.dbName, params.formIds, params.outputPath, params.year, params.month, params.excludePii, params.excludeArchivedForms, params.excludeUserProfileAndReports) + await generateCsvDataSet(params.dbName, params.formIds, params.outputPath, params.fromYear, params.fromMonth, params.toYear, params.toMonth, params.excludePii, params.excludeArchivedForms, params.excludeUserProfileAndReports) process.exit() } catch (error) { console.error(error) diff --git a/server/src/scripts/generate-csv-data-set/generate-csv-data-set.js b/server/src/scripts/generate-csv-data-set/generate-csv-data-set.js index 4aa14f16c9..e9c6cdff14 100755 --- a/server/src/scripts/generate-csv-data-set/generate-csv-data-set.js +++ b/server/src/scripts/generate-csv-data-set/generate-csv-data-set.js @@ -26,7 +26,7 @@ const writeState = async function (state) { } const sleep = (milliseconds) => new Promise((res) => setTimeout(() => res(true), milliseconds)) -function generateCsv(dbName, formId, outputPath, year = '*', month = '*', csvTemplateId) { +function generateCsv(dbName, formId, outputPath, fromYear = '*', fromMonth = '*', toYear="*", toMonth="*", csvTemplateId) { return new Promise(async function(resolve, reject) { let csvTemplate if (csvTemplateId) { @@ -36,9 +36,14 @@ function generateCsv(dbName, formId, outputPath, year = '*', month = '*', csvTem const batchSize = (process.env.T_CSV_BATCH_SIZE) ? process.env.T_CSV_BATCH_SIZE : 5 const sleepTimeBetweenBatches = 0 let cmd = `cd /tangerine/server/src/scripts/generate-csv/ && ./bin.js ${dbName} ${formId} "${outputPath}" ${batchSize} ${sleepTimeBetweenBatches}` - if (year !== '*' && month !== '*') { - cmd += ` ${sanitize(year)} ${sanitize(month)}` - } else { + if (fromYear !== '*' && fromMonth !== '*') { + cmd += ` ${sanitize(fromYear)} ${sanitize(fromMonth)}` + } else{ + cmd += ` '' '' ` + } if(toYear !== '*' && toMonth !== '*') { + cmd += ` ${sanitize(toYear)} ${sanitize(toMonth)}` + } + else { cmd += ` '' '' ` } cmd = `${cmd} ${csvTemplate ? `"${csvTemplate.headers.join(',')}"` : ''}` @@ -53,7 +58,7 @@ function generateCsv(dbName, formId, outputPath, year = '*', month = '*', csvTem }) } -async function generateCsvDataSet(groupId = '', formIds = [], outputPath = '', year = '*', month = '*', excludePii = false, excludeArchivedForms = false, excludeUserProfileAndReports = false) { +async function generateCsvDataSet(groupId = '', formIds = [], outputPath = '', fromYear = '*', fromMonth = '*', toYear="*", toMonth="*", excludePii = false, excludeArchivedForms = false, excludeUserProfileAndReports = false) { const http = await getUser1HttpInterface() const group = (await http.get(`/nest/group/read/${groupId}`)).data const groupLabel = group.label.replace(/ /g, '_') @@ -64,8 +69,10 @@ async function generateCsvDataSet(groupId = '', formIds = [], outputPath = '', y dbName: `${groupId}-reporting${excludePii ? '-sanitized' : ''}`, formIds, outputPath, - year, - month, + fromYear, + fromMonth, + toYear, + toMonth, excludePii, csvs: formIds.map(formId => { return { @@ -106,7 +113,7 @@ async function generateCsvDataSet(groupId = '', formIds = [], outputPath = '', y const csvOutputPath = `/csv/${fileName.replace(/['",]/g, "_")}` const csvStatePath = `${csvOutputPath.replace('.csv', '')}.state.json` log.debug("About to generateCsv in generate-csv-data-set.js") - generateCsv(state.dbName, formId, csvOutputPath, year, month, csv.csvTemplateId) + generateCsv(state.dbName, formId, csvOutputPath, fromYear, fromMonth, toYear, toMonth, csv.csvTemplateId) while (!await fs.pathExists(csvStatePath)) { await sleep(1*1000) } diff --git a/server/src/scripts/generate-csv-data-sets/generate-csv-data-sets.js b/server/src/scripts/generate-csv-data-sets/generate-csv-data-sets.js index b74129bead..40f98c6ac2 100755 --- a/server/src/scripts/generate-csv-data-sets/generate-csv-data-sets.js +++ b/server/src/scripts/generate-csv-data-sets/generate-csv-data-sets.js @@ -26,7 +26,7 @@ async function generateCsvDataSets(filename) { } for (let group of state.groups) { await writeState(state) - await generateCsvDataSet(group.id, group.formIds, group.outputPath, '*', '*', false, true, true) + await generateCsvDataSet(group.id, group.formIds, group.outputPath, '*', '*','*', '*',false, true, true) group.complete = true await writeState(state) } diff --git a/server/src/scripts/generate-csv/batch.js b/server/src/scripts/generate-csv/batch.js index aed71dccba..19dbf7bece 100755 --- a/server/src/scripts/generate-csv/batch.js +++ b/server/src/scripts/generate-csv/batch.js @@ -20,13 +20,15 @@ const params = { groupConfigurationDoc: process.argv[3] } -function getData(dbName, formId, skip, batchSize, year, month) { +function getData(dbName, formId, skip, batchSize, fromYear, fromMonth, toYear, toMonth) { console.log("Getting data in batch.js. dbName: " + dbName + " formId: " + formId) const limit = batchSize return new Promise((resolve, reject) => { try { - const key = (year && month) ? `${formId}_${year}_${month}` : formId - const target = `${dbDefaults.prefix}/${dbName}/_design/tangy-reporting/_view/resultsByGroupFormId?keys=["${key}"]&include_docs=true&skip=${skip}&limit=${limit}` + const startKey = (fromYear && fromMonth) ? `${formId}_${fromYear}_${fromMonth}` : formId + const endKey = (toYear && toMonth) ? `${formId}_${toYear}_${toMonth}` : formId + const target = `${dbDefaults.prefix}/${dbName}/_design/tangy-reporting/_view/resultsByGroupFormId?startkey="${startKey}"&endkey="${endKey}"&include_docs=true&skip=${skip}&limit=${limit}` + process.stderr.write(target) axios.get(target) .then(response => { resolve(response.data.rows.map(row => row.doc)) @@ -88,7 +90,7 @@ function handleCSVReplacementAndDisabledFields(value, csvReplacementCharacters) async function batch() { const state = JSON.parse(await readFile(params.statePath)) - const docs = await getData(state.dbName, state.formId, state.skip, state.batchSize, state.year, state.month) + const docs = await getData(state.dbName, state.formId, state.skip, state.batchSize, state.fromYear, state.fromMonth, state.toYear, state.toMonth) let outputDisabledFieldsToCSV = state.groupConfigurationDoc? state.groupConfigurationDoc["outputDisabledFieldsToCSV"] : false let csvReplacementCharacters = state.groupConfigurationDoc? state.groupConfigurationDoc["csvReplacementCharacters"] : false // let csvReplacement = csvReplacementCharacters? JSON.parse(csvReplacementCharacters) : false diff --git a/server/src/scripts/generate-csv/bin.js b/server/src/scripts/generate-csv/bin.js index 5f05bd0b9c..61cab03878 100755 --- a/server/src/scripts/generate-csv/bin.js +++ b/server/src/scripts/generate-csv/bin.js @@ -2,10 +2,11 @@ if (process.argv[2] === '--help') { console.log('Usage:') - console.log(' generate-csv [batchSize] [year] [month] [headers]` ') + console.log(' generate-csv [batchSize] [fromYear] [fromMonth] [toYear] [toMonth] [headers]` ') console.log('Example:') console.log(` generate-csv g2 class-12-lesson-observation-with-pupil-books ./output.csv`) - console.log(` generate-csv g2 class-12-lesson-observation-with-pupil-books ./output.csv 10 5 2018 Jan true 'first_name,last_name'`) + console.log(` generate-csv g2 class-12-lesson-observation-with-pupil-books ./output.csv 10 5 2018 0 2018 1 true 'first_name,last_name'`) + console.log('`Take note Javascript uses 0-based indexing for months) i.e January is 0, February is 1, December is 11 etc.)`') process.exit() } @@ -26,9 +27,11 @@ const params = { outputPath: process.argv[4], batchSize: (process.argv[5]) ? parseInt(process.argv[5]) : 5, sleepTimeBetweenBatches: (process.argv[6]) ? parseInt(process.argv[6]) : 0, - year: (process.argv[7]) ? process.argv[7] : null, - month: (process.argv[8]) ? process.argv[8] : null, - columnHeadersOverride: process.argv[9] ? process.argv[9].split(',') : [] + fromYear: (process.argv[7]) ? process.argv[7] : null, + fromMonth: (process.argv[8]) ? process.argv[8] : null, + toYear: (process.argv[9]) ? process.argv[9] : null, + toMonth: (process.argv[10]) ? process.argv[10] : null, + columnHeadersOverride: process.argv[11] ? process.argv[11].split(',') : [] } let state = Object.assign({}, params, {