Add support for features file (#113)

* feat(approbation): a dogs dinner

* feat(approbation): hack out a table for features while dreaming of a refactor

* basic wip feature uncovered list

* make uncoverd output greppable

* improve some error cases

* bump version, fix tests broken in comicly long function sig

* add features to readme

* 4.5, not 4.6

README.md

@@ -57,20 +57,22 @@ docker run -v "$(pwd):/run" ghcr.io/vegaprotocol/approbation:latest check-filena
 > Coverage statistics for acceptance criteria
     
 **Arguments**
-| **Parameter**   | **Type** | **Description**                      | **Example**          |
-|-----------------|----------|--------------------------------------|----------------------|
-| `--tests`         | glob     | tests to check for AC codes          | `tests/**/*.{py,feature}`    |
-| `--specs`         | glob     | specs to pull AC codes from          | `{specs/**/*.md}`    |
-| `--ignore`        | glob     | glob of files not to check for codes | `specs/0001-spec.md` |
-| `--categories`  | string  | JSON file that contains category mappings for specs          | `specs/categories.json`    |
-| `--show-branches` | boolean  | Show git branches for subfolders of the current folder | -  | 
-| `--show-mystery`  | boolean  | display criteria in tests that are not in any specs matched by `--specs`          | -    |
-| `--show-files`  | boolean  | display basic stats per file         | -    |
-| `--show-file-stats`  | boolean  | display detailed stats per file         | -    |
-| `--output-csv`  | boolean  | Outputs a CSV file to summarise the console output          | -    |
-| `--output-jenkins`  | boolean  | Outputs a text file to summarise the console output, to sendover to jenkins          | -    |
-| `--output`  | string  | A path to write the CSV or Jenkins output to | `./results`    |
-| `--verbose`  | boolean  | MORE output        | -    |
+| **Parameter**       | **Type** | **Description**                                                             | **Example**               |
+| ------------------- | -------- | --------------------------------------------------------------------------- | ------------------------- |
+| `--tests`           | glob     | tests to check for AC codes                                                 | `tests/**/*.{py,feature}` |
+| `--specs`           | glob     | specs to pull AC codes from                                                 | `{specs/**/*.md}`         |
+| `--ignore`          | glob     | glob of files not to check for codes                                        | `specs/0001-spec.md`      |
+| `--categories`      | string   | JSON file that contains category mappings for specs                         | `specs/categories.json`   |
+| `--features`        | string   | JSON file that contains features mappings for specs                         | `specs/features.json`     |
+| `--show-branches`   | boolean  | Show git branches for subfolders of the current folder                      | -                         |
+| `--show-mystery`    | boolean  | display criteria in tests that are not in any specs matched by `--specs`    | -                         |
+| `--show-files`      | boolean  | display basic stats per file                                                | -                         |
+| `--show-file-stats` | boolean  | display detailed stats per file                                             | -                         |
+| `--output-csv`      | boolean  | Outputs a CSV file to summarise the console output                          | -                         |
+| `--output-jenkins`  | boolean  | Outputs a text file to summarise the console output, to sendover to jenkins | -                         |
+| `--output`          | string   | A path to write the CSV or Jenkins output to                                | `./results`               |
+| `--verbose`         | boolean  | MORE output                                                                 | -                         |
+
 
 ### check-references example
 ```bash

bin/approbation.js

@@ -99,6 +99,7 @@ if (command === 'check-filenames') {
   const specsGlob = argv.specs
   const testsGlob = argv.tests
   const categories = argv.categories
+  const features = argv.features
   const ignoreGlob = argv.ignore
   const showMystery = argv['show-mystery'] === true
   const showCategoryStats = argv['category-stats'] === true
@@ -139,7 +140,7 @@ if (command === 'check-filenames') {
   }
 
   // TODO: Turn in to an object
-  res = checkReferences(specsGlob, testsGlob, categories, ignoreGlob, showMystery, isVerbose, showCategoryStats, showFiles, shouldOutputCSV, shouldOutputJenkins, shouldShowFileStats, outputPath)
+  res = checkReferences(specsGlob, testsGlob, categories, ignoreGlob, features, showMystery, isVerbose, showCategoryStats, showFiles, shouldOutputCSV, shouldOutputJenkins, shouldShowFileStats, outputPath)
 
   process.exit(res.exitCode)
 } else {
@@ -177,6 +178,7 @@ if (command === 'check-filenames') {
   showArg(`--specs="${pc.yellow('{specs/**/*.md}')}"`, 'glob of specs to pull AC codes from ')
   showArg(`--tests="${pc.yellow('tests/**/*.{py,feature}')}"`, 'glob of tests to match to the spec AC codes')
   showArg(`--categories="${pc.yellow('./specs/protocol/categories.json')}"`, 'Single JSON file that contains the categories for this test run')
+  showArg(`--features="${pc.yellow('./specs/protocol/features.json')}"`, 'Single JSON file that contains the features for this test run')
   showArg(`--ignore="${pc.yellow('{tests/**/*.{py,feature}')}"`, 'glob of files to ignore for both tests and specs')
   showArg('--show-mystery', 'If set, display criteria in tests that are not in any specs matched by --specs')
   showArg('--category-stats', 'Show more detail for referenced/unreferenced codes')

package.json

@@ -1,12 +1,12 @@
 {
   "name": "@vegaprotocol/approbation",
-  "version": "4.4.2",
+  "version": "4.5.0",
   "description": "Match Acceptance Criteria Codes with the tests that test them",
   "engine": ">= 18",
   "bin": "./bin/approbation.js",
   "scripts": {
-    "test:ci": "tape test/*.js",
-    "test": "tape test/*.js | tap-arc"
+    "test:ci": "tape 'test/**/*.test.js'",
+    "test": "tape 'test/**/*.test.js' | tap-arc"
   },
   "repository": {
     "type": "git",

src/check-references.js

@@ -11,6 +11,7 @@ const path = require('path')
 const pc = require('picocolors')
 const { validSpecificationPrefix, validAcceptanceCriteriaCode, ignoreFiles } = require('./lib')
 const { getCategoriesForSpec, increaseCodesForCategory, increaseCoveredForCategory, increaseAcceptableSpecsForCategory, increaseUncoveredForCategory, increaseFeatureCoveredForCategory, increaseSystemTestCoveredForCategory, increaseSpecCountForCategory, setCategories } = require('./lib/category')
+const { setFeatures, increaseAcceptableSpecsForFeature, increaseCodesForFeature, increaseCoveredForFeature, increaseFeatureCoveredForFeature, increaseSpecCountForFeature, increaseSystemTestCoveredForFeature, increaseUncoveredForFeature, specFeatures } = require('./lib/feature')
 const { Table } = require('console-table-printer')
 const { specPriorities } = require('./lib/priority')
 const sortBy = require('lodash.sortby')
@@ -124,16 +125,19 @@ function processReferences (specs, tests) {
           criteriaWithRefs.push(c)
           criteriaReferencedTotal++
           categories.forEach(c => increaseCoveredForCategory(c, 1))
+          increaseCoveredForFeature(c, 1)
 
           // Hacky hack: Limit these to 1 or 0 rather than a true tally.
           let criteriaAlreadyLoggedSystest = false
           let criteriaAlreadyLoggedFeature = false
           linksForAC.forEach(l => {
             if (!criteriaAlreadyLoggedSystest && l.match('system-tests')) {
+              increaseSystemTestCoveredForFeature(c, 1)
               categories.forEach(c => increaseSystemTestCoveredForCategory(c, 1))
               value.referencedBySystemTest++
               criteriaAlreadyLoggedSystest = true
             } else if (!criteriaAlreadyLoggedFeature && l.match('.feature')) {
+              increaseFeatureCoveredForFeature(c, 1)
               categories.forEach(c => increaseFeatureCoveredForCategory(c, 1))
               value.referencedByFeature++
               criteriaAlreadyLoggedFeature = true
@@ -150,6 +154,7 @@ function processReferences (specs, tests) {
       if (criteriaWithRefs.length !== value.criteria.length) {
         value.criteria.forEach(v => {
           if (!criteriaWithRefs.includes(v)) {
+            increaseUncoveredForFeature(v, 1)
             unreferencedCriteria.push(v)
             criteriaUnreferencedTotal++
           }
@@ -197,22 +202,29 @@ function processReferences (specs, tests) {
   }
 }
 
-function checkReferences (specsGlob, testsGlob, categoriesPath, ignoreGlob, showMystery = false, isVerbose = false, showCategoryStats = false, shouldShowFiles = false, shouldOutputCSV = false, shouldOutputJenkins = false, shouldShowFileStats = false, outputPath = './results') {
+function checkReferences (specsGlob, testsGlob, categoriesPath, ignoreGlob, featuresPath, showMystery = false, isVerbose = false, showCategoryStats = false, shouldShowFiles = false, shouldOutputCSV = false, shouldOutputJenkins = false, shouldShowFileStats = false, outputPath = './results') {
   verbose = isVerbose
   showFiles = shouldShowFiles
-
   const ignoreList = ignoreGlob ? glob.sync(ignoreGlob, {}) : []
   const specList = ignoreFiles(glob.sync(specsGlob, {}), ignoreList)
   const testList = ignoreFiles(glob.sync(testsGlob, {}), ignoreList, 'test')
   let categories
 
-  let specs, tests, specCategories
+  let specs, tests, features, specFeatures
   const exitCode = 0
 
+
   if (specList.length > 0 && testList.length > 0) {
     try {
+      // Categories gather spec files in to categories, and tally the number of codes in each category
       specCategories = JSON.parse(fs.readFileSync(categoriesPath))
       setCategories(specCategories)
+      // Features gather Acceptance Criteria across spec files or categories, and tally the numbers
+
+      if (featuresPath !== undefined) {
+        specFeatures = setFeatures(JSON.parse(fs.readFileSync(featuresPath)))
+      }
+
       specs = gatherSpecs(specList)
       tests = gatherTests(testList)
     } catch (e) {
@@ -269,6 +281,66 @@ function checkReferences (specsGlob, testsGlob, categoriesPath, ignoreGlob, show
       st.addRows(sortBy(specsTableRows, ['Priority', 'Coverage']))
       console.log(st.render())
     }
+
+    if (featuresPath) {
+      const totals = []
+      const milestoneNames = new Set()
+      const milestones = new Map()
+      Object.keys(specFeatures).forEach(key => milestoneNames.add(specFeatures[key].milestone))
+      milestoneNames.forEach(m => milestones.set(m, []))
+
+      Object.keys(specFeatures).filter(k => k !== 'Unknown').forEach(key => {
+        const c = specFeatures[key]
+        const coverage = (c.covered / (c.acs.length | 0) * 100).toFixed(1)
+
+        if (c.uncovered !== 0 && c.uncoveredAcs) {
+          console.log(
+            pc.red(`Uncovered ACs for ${key} (${c.milestone}): `) + 
+            Array.from(c.uncoveredAcs).join(', ')
+          )
+        }
+
+        milestones.get(c.milestone).push({
+          Feature: key,
+          Milestone: c.milestone || 0,
+          acs: c.acs.length || 0,
+          Covered: c.covered || 0,
+          'by/FeatTest': c.featureCovered || 0,
+          'by/SysTest': c.systemTestCovered || 0,
+          Uncovered: c.uncovered || 0,
+          Coverage: isNaN(coverage) ? '0%' : `${coverage}%`
+        })
+
+      })
+
+      milestones.forEach((featuresByMilestone, milestoneKey) => {
+        if (milestoneKey === 'unknown') {
+          return
+        }
+        const Covered = featuresByMilestone.reduce((acc, cur) => acc + cur.Covered, 0);
+        const acs = featuresByMilestone.reduce((acc, cur) => acc + cur.acs, 0);
+        const Coverage = `${(Covered / acs * 100).toFixed(1)}%`
+        totals.push({
+          Feature: `Total`,
+          Milestone: milestoneKey || '-',
+          acs,
+          Covered,
+          'by/FeatTest': featuresByMilestone.reduce((acc, cur) => acc + cur['by/FeatTest'], 0) || '-',
+          'by/SysTest': featuresByMilestone.reduce((acc, cur) => acc + cur['by/SysTest'], 0) || '-',
+          Uncovered: featuresByMilestone.reduce((acc, cur) => acc + cur.Uncovered, 0) || '-',
+          Coverage
+        })
+      })
+
+      const t = new Table()
+      t.addRows(milestones.get('deployment-1'));
+      t.addRows(milestones.get('deployment-2'));
+      t.addRows([{ Feature: '---', Milestone: '---', acs: '---', Covered: '---', 'by/FeatTest': '---', 'by/SysTest': '---', Uncovered: '---', Coverage: '---' }]);
+      t.addRows(totals);
+      const tableOutput = t.render()
+      console.log(tableOutput)
+    }
+
     if (showCategoryStats) {
       const shouldOutputImage = false
 
@@ -300,11 +372,11 @@ function checkReferences (specsGlob, testsGlob, categoriesPath, ignoreGlob, show
         }
 
         if (shouldOutputCSV) {
-          let csvOutput = Object.keys(categories[0]).join(',')
+          let categoriesCsvOutput = Object.keys(categories[0]).join(',')
           categories.forEach(c => {
-            csvOutput += `\r\n${Object.values(c).join(',')}`
+            categoriesCsvOutput += `\r\n${Object.values(c).join(',')}`
           })
-          fs.writeFileSync(`${outputPath}/approbation-categories.csv`, csvOutput)
+          fs.writeFileSync(`${outputPath}/approbation-categories.csv`, categoriesCsvOutput)
 
           if (shouldShowFileStats) {
             let csvOutputFiles = Object.keys(specsTableRows[0]).join(',')
@@ -315,10 +387,6 @@ function checkReferences (specsGlob, testsGlob, categoriesPath, ignoreGlob, show
           }
         }
 
-        if (shouldOutputImage) {
-          console.log(pc.red('Generating image not yet supported'))
-        }
-
         if (shouldOutputJenkins) {
           const skipCategories = ['Category', 'Specs', 'Acceptable']
           const jenkinsLine = Object.entries(categories.pop()).map(([key, value]) => skipCategories.indexOf(key) === -1 ? `*${key}*: ${value}` : '').join('  ').trim()