feat(editor): add date range filtering for CSV downloads

editor/src/app/groups/download-csv/download-csv.component.ts

@@ -17,8 +17,10 @@ export class DownloadCsvComponent implements OnInit, OnDestroy {
 
   months = [];
   years = [];
-  selectedMonth = '*';
-  selectedYear = '*';
+  fromYear = '*';
+  fromMonth = '*';
+  toYear = '*';
+  toMonth = '*';
   processing = false;
   stateUrl;
   downloadUrl;
@@ -63,13 +65,17 @@ export class DownloadCsvComponent implements OnInit, OnDestroy {
   }
 
   async process() {
-    if ((this.selectedMonth === '*' && this.selectedYear !== '*') || (this.selectedMonth !== '*' && this.selectedYear === '*')) {
-      alert('You must choose a month and a year.')
+    if ((this.fromMonth === '*' && this.fromYear !== '*') || (this.fromMonth !== '*' && this.fromYear === '*')) {
+      alert('You must choose a start month and year.')
+      return
+    }
+    if ((this.toMonth === '*' && this.toYear !== '*') || (this.toMonth !== '*' && this.toYear === '*')) {
+      alert('You must choose an end month and year.')
       return
     }
     this.processing = true
     try {
-      const result: any = await this.groupsService.downloadCSV(this.groupName, this.formId, this.selectedYear, this.selectedMonth, this.excludePII);
+      const result: any = await this.groupsService.downloadCSV(this.groupName, this.formId, this.fromYear, this.fromMonth, this.toYear, this.toMonth, this.excludePII);
       this.stateUrl = result.stateUrl;
       this.downloadUrl = result.downloadUrl;
       // TODO call download status immediately then after every few second, Probably use RXJS to ensure we only use the latest values

editor/src/app/groups/new-csv-data-set/new-csv-data-set.component.html

@@ -6,23 +6,50 @@
   </div>
   <form class="form">
     <div class="container">
-      <div id="year-and-month">
+      <div id="from-year-and-month">
+        <h3>From:</h3>
         <mat-form-field appearance="fill" color="primary">
           <mat-label>{{ "Month" | translate }}</mat-label>
           <mat-select
-            name="selectedMonth"
+            name="fromMonth"
             class="month"
-            [(ngModel)]="selectedMonth"
+            [(ngModel)]="fromMonth"
           >
             <mat-option value="*" selected="selected">All months</mat-option>
-            <mat-option *ngFor="let month of months" value="{{ month }}">{{
+            <mat-option *ngFor="let month of months; index as i " value="{{ i }}">{{
               month
             }}</mat-option>
           </mat-select>
         </mat-form-field>
         <mat-form-field appearance="fill" color="primary">
           <mat-label>{{ "Year" | translate }}</mat-label>
-          <mat-select name="selectedYear" class="year" [(ngModel)]="selectedYear">
+          <mat-select name="fromYear" class="year" [(ngModel)]="fromYear">
+            <mat-option value="*" selected="selected">All years</mat-option>
+            <mat-option *ngFor="let year of years" value="{{ year }}">{{
+              year
+            }}</mat-option>
+          </mat-select>
+        </mat-form-field>
+      </div>
+      <br/>
+      <h3>To:</h3>
+      <div id="to-year-and-month">
+        <mat-form-field appearance="fill" color="primary">
+          <mat-label>{{ "Month" | translate }}</mat-label>
+          <mat-select
+            name="toMonth"
+            class="month"
+            [(ngModel)]="toMonth"
+          >
+            <mat-option value="*" selected="selected">All months</mat-option>
+            <mat-option *ngFor="let month of months; index as i" value="{{ i }}">{{
+              month
+            }}</mat-option>
+          </mat-select>
+        </mat-form-field>
+        <mat-form-field appearance="fill" color="primary">
+          <mat-label>{{ "Year" | translate }}</mat-label>
+          <mat-select name="toYear" class="year" [(ngModel)]="toYear">
             <mat-option value="*" selected="selected">All years</mat-option>
             <mat-option *ngFor="let year of years" value="{{ year }}">{{
               year
@@ -100,7 +127,7 @@
       </tr>
     </table>
     <div id="submit-container">
-      <button mat-raised-button color="warn" (click)="process()" [disabled]="selectedForms.length<1">{{'Submit Request'|translate}}</button>
+      <button mat-raised-button color="warn" (click)="process()" [disabled]="canSubmit()">{{'Submit Request'|translate}}</button>
     </div>
   </form>
 </div>

editor/src/app/groups/new-csv-data-set/new-csv-data-set.component.ts

@@ -27,8 +27,10 @@ export class NewCsvDataSetComponent implements OnInit {
   months = []
   years = []
   description
-  selectedMonth = '*'
-  selectedYear = '*'
+  fromYear = '*'
+  fromMonth = '*'
+  toYear = '*'
+  toMonth = '*'
   selectedForms = []
   allFormsSelected = false
   groupId = ''
@@ -110,12 +112,16 @@ export class NewCsvDataSetComponent implements OnInit {
     const forms = this.selectedForms
       .map(formId => this.templateSelections[formId] ? `${formId}:${this.templateSelections[formId]}` : formId)
       .toString()
-    if ((this.selectedMonth === '*' && this.selectedYear !== '*') || (this.selectedMonth !== '*' && this.selectedYear === '*')) {
-      alert('You must choose a month and a year.')
+    if ((this.fromMonth === '*' && this.fromYear !== '*') || (this.fromMonth !== '*' && this.fromYear === '*')) {
+      alert('You must choose a start month and year.')
+      return
+    }
+    if ((this.toMonth === '*' && this.toMonth !== '*') || (this.toMonth !== '*' && this.toMonth === '*')) {
+      alert('You must choose an end month and year.')
       return
     }
     try {
-      const result: any = await this.groupsService.downloadCSVDataSet(this.groupId, forms, this.selectedYear, this.selectedMonth, this.description, this.excludePII);
+      const result: any = await this.groupsService.downloadCSVDataSet(this.groupId, forms, this.fromYear, this.fromMonth, this.toYear, this. toMonth, this.description, this.excludePII);
       this.stateUrl = result.stateUrl;
       this.router.navigate(['../', result.id], { relativeTo: this.route })
     } catch (error) {
@@ -140,4 +146,13 @@ export class NewCsvDataSetComponent implements OnInit {
     }
   }
 
+  canSubmit(){
+    return (
+      this.selectedForms.length < 1 ||
+      this.toYear < this.fromYear ||
+      (this.fromYear === "*" && this.toYear !== "*")||
+      (`${this.fromYear}:${this.fromMonth}` > `${this.toYear}:${this.toMonth}`)
+    );
+  }
+
 }

editor/src/app/groups/services/groups.service.ts

@@ -106,20 +106,20 @@ export class GroupsService {
     }
   }
 
-  async downloadCSV(groupName: string, formId: string, selectedYear = '*', selectedMonth = '*', excludePII: boolean) {
+  async downloadCSV(groupName: string, formId: string, fromYear = '*', fromMonth = '*', toYear = '*', toMonth = '*', excludePII: boolean) {
     let sanitized = ''
     if (excludePII) {
       sanitized = '-sanitized'
     }
     try {
-      if (selectedMonth === '*' || selectedYear === '*') {
+      if (fromYear === '*' || fromMonth === '*') {
         const result = await this.httpClient
           .get(`/api/csv${sanitized}/${groupName}/${formId}`)
           .toPromise();
         return result;
       } else {
         const result = await this.httpClient
-          .get(`/api/csv${sanitized}/${groupName}/${formId}/${selectedYear}/${selectedMonth}`)
+          .get(`/api/csv${sanitized}/${groupName}/${formId}/${fromYear}/${fromMonth}/${toYear}/${toMonth}`)
           .toPromise();
         return result;
       }
@@ -129,11 +129,11 @@ export class GroupsService {
       }
     }
   }
-  async downloadCSVDataSet(groupName: string, formIds: string, selectedYear = '*', selectedMonth = '*', description: string,excludePII: boolean) {
+  async downloadCSVDataSet(groupName: string, formIds: string, fromYear = '*', fromMonth = '*', toYear = '*', toMonth = '*', description: string,excludePII: boolean) {
     try {
       let sanitized = excludePII? '-sanitized' : ''
       const result = await this.httpClient
-          .post(`/api/create/csvDataSet${sanitized}/${groupName}`, {formIds, description,selectedMonth,selectedYear})
+          .post(`/api/create/csvDataSet${sanitized}/${groupName}`, {formIds, description,fromYear, fromMonth, toYear, toMonth})
           .toPromise();
           console.log(result)
         return result;

server/src/modules/csv/views.js

@@ -5,10 +5,9 @@ module.exports = {
     resultsByGroupFormId: {
       map: function (doc) {
         if (doc.formId) {
-          const MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
           const startUnixtime = new Date(doc.startUnixtime);
-          const key = doc.formId + '_' + startUnixtime.getFullYear() + '_' + MONTHS[startUnixtime.getMonth()];
-          //The emmitted value is in the form "formId" i.e `formId` and also "formId_2018_May" i.e `formId_Year_Month`
+          const key = doc.formId + '_' + startUnixtime.getFullYear() + '_' + startUnixtime.getMonth();// getMonth returns 0-based index i.e 0,1,2,3,4,5,6,7,8,9,10,11
+          //The emitted value is in the form "formId" i.e `formId` and also "formId_2018_0" for Jan i.e `formId_Year_Month` Remember Javascript uses 0-based indexing for months
           emit(doc.formId);
           emit(key);
         }

server/src/routes/group-csv.js

@@ -58,8 +58,15 @@ const generateCSV = async (req, res) => {
 
   const sleepTimeBetweenBatches = 0
   let cmd = `cd /tangerine/server/src/scripts/generate-csv/ && ./bin.js ${dbName} ${formId} "${outputPath}" ${batchSize} ${sleepTimeBetweenBatches}`
-  if (req.params.year && req.params.month) {
-    cmd += ` ${sanitize(req.params.year)} ${sanitize(req.params.month)}`
+  if (req.params.fromYear && req.params.fromMonth) {
+    cmd += ` ${sanitize(req.params.fromYear)} ${sanitize(req.params.fromMonth)}`
+  } else{
+    cmd += ` '' '' `
+  }
+  if(req.params.toYear && req.params.toMonth) {
+    cmd += ` ${sanitize(req.params.toYear)} ${sanitize(req.params.toMonth)}`
+  } else{
+    cmd += ` '' '' `
   }
   log.info(`generating csv start: ${cmd}`)
   exec(cmd).then(status => {
@@ -77,7 +84,7 @@ const generateCSVDataSet = async (req, res) => {
   const groupId = sanitize(req.params.groupId)
   // A list of formIds will be too long for sanitize's limit of 256 bytes so we split, map with sanitize, and join.
   const formIds = req.body.formIds.split(',').map(formId => formId).join(',')
-  const {selectedYear, selectedMonth, description} = req.body
+  const {fromYear, fromMonth, toYear, toMonth, description} = req.body
   const http = await getUser1HttpInterface()
   const group = (await http.get(`/nest/group/read/${groupId}`)).data
   const groupLabel = group.label.replace(/[&\/\\#,+()$~%.'":*?<>{}]/g, '')
@@ -86,10 +93,10 @@ const generateCSVDataSet = async (req, res) => {
   }
   const fileName = `${sanitize(groupLabel, options)}-${Date.now()}.zip`.replace(/[&\/\\#,+()$~%'":*?<>^{}_ ]+/g, '_')
   let outputPath = `/csv/${fileName.replace(/[&\/\\#,+()$~%'":*?<>^{}_ ]+/g, '_')}`
-  let cmd = `cd /tangerine/server/src/scripts/generate-csv-data-set/ && ./bin.js ${groupId} ${formIds} ${outputPath} ${selectedYear === '*' ? `'*'` : sanitize(selectedYear)} ${selectedMonth === '*' ? `'*'` : sanitize(selectedMonth)} ${req.originalUrl.includes('-sanitized') ? '--sanitized': ''}`
-  log.info(`generating csv start: ${cmd}`)
+  let cmd = `cd /tangerine/server/src/scripts/generate-csv-data-set/ && ./bin.js ${groupId} ${formIds} ${outputPath} ${fromYear === '*' ? `'*'` : sanitize(fromYear)} ${fromMonth === '*' ? `'*'` : sanitize(fromMonth)} ${toYear === '*' ? `'*'` : sanitize(toYear)} ${toMonth === '*' ? `'*'` : sanitize(toMonth)} ${req.originalUrl.includes('-sanitized') ? '--sanitized': ''}`
+  log.info(`generating csv start: ${cmd}\n`)
   exec(cmd).then(status => {
-    log.info(`generate csv done: ${JSON.stringify(status)} ${outputPath}`)
+    log.info(`generate csv done: ${JSON.stringify(status)} ${outputPath}\n`)
   }).catch(error => {
     log.error(error)
   })
@@ -102,8 +109,10 @@ const generateCSVDataSet = async (req, res) => {
     stateUrl,
     downloadUrl,
     description,
-    year: selectedYear,
-    month: selectedMonth,
+    fromYear,
+    fromMonth,
+    toYear,
+    toMonth,
     dateCreated: Date.now()
   })
   res.send({
@@ -223,8 +232,10 @@ const getDataset = async (datasetId) => {
     stateExists,
     excludePii,
     description: result.description,
-    month: result.month,
-    year: result.year,
+    fromYear: result.fromYear,
+    fromMonth: result.fromMonth,
+    toYear: result.toYear,
+    toMonth: result.toMonth,
     downloadUrl: result.downloadUrl,
     fileName: result.fileName,
     dateCreated: result.dateCreated,

server/src/scripts/generate-csv-data-set/bin.js

@@ -5,28 +5,31 @@ const generateCsvDataSet = require('./generate-csv-data-set.js')
 
 if (process.argv[2] === '--help') {
   console.log('Usage:')
-  console.log('  generate-csv-data-set <groupId> <formId[,formId]> <outputPath> <year> <month> [--exclude-pii] [--excludeArchivedForms] [--excludeUserProfileAndReports]')
+  console.log('  generate-csv-data-set <groupId> <formId[,formId]> <outputPath> <fromYear> <froMonth> <toYear> <toMonth> [--exclude-pii] [--excludeArchivedForms] [--excludeUserProfileAndReports]')
   console.log('Examples:')
-  console.log(`  generate-csv-data-set group-abdc form1,form2 ./output.csv 2018 Jan --exclude-pii`)
-  console.log(`  generate-csv-data-set group-abdc form1,form2 ./output.csv * * --exclude-pii`)
+  console.log(`  generate-csv-data-set group-abdc form1,form2 ./output.csv 2018 4 2019 1 --exclude-pii`)
+  console.log(`'Remember Javascript uses 0-based indexing for months) i.e January is 0, February is 1, December is 11 etc.'`)
+  console.log(`  generate-csv-data-set group-abdc form1,form2 ./output.csv * * * * --exclude-pii`)
   process.exit()
 }
 
 const params = {
   dbName: process.argv[2],
   formIds: process.argv[3].split(','),
   outputPath: process.argv[4],
-  year: (process.argv[5]) ? process.argv[5] : null,
-  month: (process.argv[6]) ? process.argv[6] : null,
-  excludePii: process.argv[7] ? true : false,
-  excludeArchivedForms: process.argv[8] ? true : false,
-  excludeUserProfileAndReports: process.argv[9] ? true : false
+  fromYear: (process.argv[5]) ? process.argv[5] : null,
+  fromMonth: (process.argv[6]) ? process.argv[6] : null,
+  toYear: (process.argv[7]) ? process.argv[7] : null,
+  toMonth: (process.argv[8]) ? process.argv[8] : null,
+  excludePii: process.argv[9] ? true : false,
+  excludeArchivedForms: process.argv[10] ? true : false,
+  excludeUserProfileAndReports: process.argv[11] ? true : false
 }
 
 async function go(params) {
   try {
     log.debug("generateCsvDataSet bin.js")
-    await generateCsvDataSet(params.dbName, params.formIds, params.outputPath, params.year, params.month, params.excludePii, params.excludeArchivedForms, params.excludeUserProfileAndReports)
+    await generateCsvDataSet(params.dbName, params.formIds, params.outputPath, params.fromYear, params.fromMonth, params.toYear, params.toMonth, params.excludePii, params.excludeArchivedForms, params.excludeUserProfileAndReports)
     process.exit()
   } catch (error) {
     console.error(error)

server/src/scripts/generate-csv-data-set/generate-csv-data-set.js

@@ -26,7 +26,7 @@ const writeState = async function (state) {
 }
 const sleep = (milliseconds) => new Promise((res) => setTimeout(() => res(true), milliseconds))
 
-function generateCsv(dbName, formId, outputPath, year = '*', month = '*', csvTemplateId) {
+function generateCsv(dbName, formId, outputPath, fromYear = '*', fromMonth = '*', toYear="*", toMonth="*", csvTemplateId) {
   return new Promise(async function(resolve, reject) {
     let csvTemplate
     if (csvTemplateId) {
@@ -36,9 +36,14 @@ function generateCsv(dbName, formId, outputPath, year = '*', month = '*', csvTem
     const batchSize = (process.env.T_CSV_BATCH_SIZE) ? process.env.T_CSV_BATCH_SIZE : 5
     const sleepTimeBetweenBatches = 0
     let cmd = `cd /tangerine/server/src/scripts/generate-csv/ && ./bin.js ${dbName} ${formId} "${outputPath}" ${batchSize} ${sleepTimeBetweenBatches}`
-    if (year !== '*' && month !== '*') {
-      cmd += ` ${sanitize(year)} ${sanitize(month)}`
-    } else {
+    if (fromYear !== '*' && fromMonth !== '*') {
+      cmd += ` ${sanitize(fromYear)} ${sanitize(fromMonth)}`
+    } else{
+      cmd += ` '' '' `
+    } if(toYear !== '*' && toMonth !== '*') {
+      cmd += ` ${sanitize(toYear)} ${sanitize(toMonth)}`
+    }
+    else {
       cmd += ` '' '' `
     }
     cmd = `${cmd} ${csvTemplate ? `"${csvTemplate.headers.join(',')}"` : ''}`
@@ -53,7 +58,7 @@ function generateCsv(dbName, formId, outputPath, year = '*', month = '*', csvTem
   })
 }
 
-async function generateCsvDataSet(groupId = '', formIds = [], outputPath = '', year = '*', month = '*', excludePii = false, excludeArchivedForms = false, excludeUserProfileAndReports = false) {
+async function generateCsvDataSet(groupId = '', formIds = [], outputPath = '', fromYear = '*', fromMonth = '*', toYear="*", toMonth="*", excludePii = false, excludeArchivedForms = false, excludeUserProfileAndReports = false) {
   const http = await getUser1HttpInterface()
   const group = (await http.get(`/nest/group/read/${groupId}`)).data
   const groupLabel = group.label.replace(/ /g, '_')
@@ -64,8 +69,10 @@ async function generateCsvDataSet(groupId = '', formIds = [], outputPath = '', y
     dbName: `${groupId}-reporting${excludePii ? '-sanitized' : ''}`,
     formIds,
     outputPath,
-    year,
-    month,
+    fromYear,
+    fromMonth,
+    toYear,
+    toMonth,
     excludePii,
     csvs: formIds.map(formId => {
       return {
@@ -106,7 +113,7 @@ async function generateCsvDataSet(groupId = '', formIds = [], outputPath = '', y
       const csvOutputPath = `/csv/${fileName.replace(/['",]/g, "_")}`
       const csvStatePath = `${csvOutputPath.replace('.csv', '')}.state.json`
       log.debug("About to generateCsv in generate-csv-data-set.js")
-      generateCsv(state.dbName, formId, csvOutputPath, year, month, csv.csvTemplateId)
+      generateCsv(state.dbName, formId, csvOutputPath, fromYear, fromMonth, toYear, toMonth, csv.csvTemplateId)
       while (!await fs.pathExists(csvStatePath)) {
         await sleep(1*1000)
       }

server/src/scripts/generate-csv-data-sets/generate-csv-data-sets.js

@@ -26,7 +26,7 @@ async function generateCsvDataSets(filename) {
   }
   for (let group of state.groups) {
     await writeState(state)
-    await generateCsvDataSet(group.id, group.formIds, group.outputPath, '*', '*', false, true, true)
+    await generateCsvDataSet(group.id, group.formIds, group.outputPath, '*', '*','*', '*',false, true, true)
     group.complete = true
     await writeState(state)
   }

server/src/scripts/generate-csv/batch.js

@@ -20,13 +20,15 @@ const params = {
   groupConfigurationDoc: process.argv[3]
 }
 
-function getData(dbName, formId, skip, batchSize, year, month) {
+function getData(dbName, formId, skip, batchSize, fromYear, fromMonth, toYear, toMonth) {
   console.log("Getting data in batch.js. dbName: " + dbName + " formId: " + formId)
   const limit = batchSize
   return new Promise((resolve, reject) => {
     try {
-      const key = (year && month) ? `${formId}_${year}_${month}` : formId
-      const target = `${dbDefaults.prefix}/${dbName}/_design/tangy-reporting/_view/resultsByGroupFormId?keys=["${key}"]&include_docs=true&skip=${skip}&limit=${limit}`
+      const startKey = (fromYear && fromMonth) ? `${formId}_${fromYear}_${fromMonth}` : formId
+      const endKey = (toYear && toMonth) ? `${formId}_${toYear}_${toMonth}` : formId
+      const target = `${dbDefaults.prefix}/${dbName}/_design/tangy-reporting/_view/resultsByGroupFormId?startkey="${startKey}"&endkey="${endKey}"&include_docs=true&skip=${skip}&limit=${limit}`
+      process.stderr.write(target)
       axios.get(target)
         .then(response => {
           resolve(response.data.rows.map(row => row.doc))
@@ -88,7 +90,7 @@ function handleCSVReplacementAndDisabledFields(value, csvReplacementCharacters)
 
 async function batch() {
   const state = JSON.parse(await readFile(params.statePath))
-  const docs = await getData(state.dbName, state.formId, state.skip, state.batchSize, state.year, state.month)
+  const docs = await getData(state.dbName, state.formId, state.skip, state.batchSize, state.fromYear, state.fromMonth, state.toYear, state.toMonth)
   let outputDisabledFieldsToCSV = state.groupConfigurationDoc? state.groupConfigurationDoc["outputDisabledFieldsToCSV"] : false
   let csvReplacementCharacters = state.groupConfigurationDoc? state.groupConfigurationDoc["csvReplacementCharacters"] : false
   // let csvReplacement = csvReplacementCharacters? JSON.parse(csvReplacementCharacters) : false