Emit JUnit report from tests

The test reports emitted by mocha-junit-reporter have <testcase> tags
with slightly weird `name` and `classname` properties, e.g.

<testsuite name="rest/defaults">
    <testcase name="rest/defaults Init with given environment" classname="Init with given environment">
    <testcase/>
</testsuite>

This looks a bit odd in the observability server web UI, which displays
the testcase.name and testcase.classname properties. I don’t know
whether it’d be better to do some pre-upload manipulation of the ably-js
reports, or to change the fields displayed in the web UI. Not going to
do anything about it now; it’s usable enough and I don’t want to mess up
the display of the ably-cocoa test reports.

Co-authored-by: Owen Pearson <[email protected]>

.gitignore

@@ -7,3 +7,5 @@ npm-debug.log
 build/
 react/
 docs/generated/
+junit/
+test/support/mocha_junit_reporter/build/

.prettierignore

@@ -1 +1,2 @@
 test/common/ably-common/
+test/support/mocha_junit_reporter/build/

Gruntfile.js

@@ -47,8 +47,8 @@ module.exports = function (grunt) {
     pkgVersion: grunt.file.readJSON('package.json').version,
     webpack: {
       all: Object.values(webpackConfig),
-      node: webpackConfig.node,
-      browser: [webpackConfig.browser, webpackConfig.browserMin],
+      node: [webpackConfig.node, webpackConfig.mochaJUnitReporterNode],
+      browser: [webpackConfig.browser, webpackConfig.browserMin, webpackConfig.mochaJUnitReporterBrowser],
     },
   };
 

package.json

@@ -46,6 +46,7 @@
   "devDependencies": {
     "@ably/vcdiff-decoder": "1.0.4",
     "@arethetypeswrong/cli": "^0.13.1",
+    "@babel/preset-env": "^7.23.6",
     "@testing-library/react": "^13.3.0",
     "@types/crypto-js": "^4.0.1",
     "@types/jmespath": "^0.15.2",
@@ -57,6 +58,7 @@
     "@vitejs/plugin-react": "^1.3.2",
     "async": "ably-forks/async#requirejs",
     "aws-sdk": "^2.1413.0",
+    "babel-loader": "^8.3.0",
     "chai": "^4.2.0",
     "copy-webpack-plugin": "^6.4.1",
     "cors": "^2.8.5",
@@ -100,7 +102,8 @@
     "vite": "^4.4.9",
     "vitest": "^0.18.0",
     "webpack": "^4.44.2",
-    "webpack-cli": "^4.2.0"
+    "webpack-cli": "^4.2.0",
+    "webpack-node-externals": "^3.0.0"
   },
   "engines": {
     "node": ">=5.10.x"

test/playwright.html

@@ -11,6 +11,7 @@
 
     <script src="../node_modules/chai/chai.js"></script>
     <script src="../node_modules/mocha/mocha.js"></script>
+    <script src="support/mocha_junit_reporter/build/browser.js"></script>
 
     <script src="support/playwrightSetup.js"></script>
 

test/support/junit_directory_path.js

@@ -0,0 +1,3 @@
+const path = require('path');
+
+module.exports = path.join(__dirname, '..', '..', 'junit');

test/support/mocha_junit_reporter/index.js

@@ -0,0 +1,2 @@
+const MochaJUnitReporter = require('mocha-junit-reporter');
+module.exports = MochaJUnitReporter;

test/support/mocha_junit_reporter/shims/fs.js

@@ -0,0 +1,6 @@
+module.exports = {
+  // mocha-junit-reporter calls this to check whether a report file already
+  // exists (so it can delete it if so), so just return false since we’re not
+  // going to write the report to the filesystem anyway
+  existsSync: () => false,
+};

test/support/mocha_reporter.js

@@ -0,0 +1,20 @@
+const Mocha = require('mocha');
+const MochaJUnitReporter = require('./mocha_junit_reporter/build/node');
+const path = require('path');
+const jUnitDirectoryPath = require('./junit_directory_path');
+
+/**
+ * Logs test results to the console (by extending the default `Spec` reporter) and also emits a JUnit XML file.
+ */
+class Reporter extends Mocha.reporters.Spec {
+  jUnitReporter;
+
+  constructor(runner, options) {
+    super(runner, options);
+    const jUnitFileName = `node-${process.version.split('.')[0]}.junit`;
+    const jUnitFilePath = path.join(jUnitDirectoryPath, jUnitFileName);
+    this.jUnitReporter = new MochaJUnitReporter(runner, { reporterOptions: { mochaFile: jUnitFilePath } });
+  }
+}
+
+module.exports = Reporter;

test/support/playwrightSetup.js

@@ -2,9 +2,29 @@ const { EVENT_RUN_END, EVENT_TEST_FAIL, EVENT_TEST_PASS, EVENT_SUITE_BEGIN, EVEN
 
 const { ok: passSymbol, err: failSymbol } = Mocha.reporters.Base.symbols;
 
+class InMemoryMochaJUnitReporter extends MochaJUnitReporter {
+  onGotReport;
+
+  // Bit of a hack, we override this internal method of MochaJUnitReporter to
+  // get access to the test report without trying to write it to disk (which we
+  // can’t since we’re in browser)
+  writeXmlToDisk(xml, filePath) {
+    this.onGotReport(xml);
+  }
+}
+
 class CustomEventReporter extends Mocha.reporters.HTML {
+  jUnitReporter;
+  jUnitReport;
+  testResultStats;
+
   constructor(runner) {
     super(runner);
+    this.junitReporter = new InMemoryMochaJUnitReporter(runner);
+    this.junitReporter.onGotReport = (report) => {
+      this.jUnitReport = report;
+      this.gotResults();
+    };
     this.indents = 0;
     this.failedTests = [];
 
@@ -30,16 +50,8 @@ class CustomEventReporter extends Mocha.reporters.HTML {
         if (this.failedTests.length > 0) {
           this.logToNodeConsole('\nfailed tests: \n' + this.failedTests.map((x) => ' - ' + x).join('\n') + '\n');
         }
-        runner.stats &&
-          window.dispatchEvent(
-            new CustomEvent('testResult', {
-              detail: {
-                pass: runner.stats && runner.stats.failures === 0,
-                passes: runner.stats.passes,
-                total: runner.stats.passes + runner.stats.failures,
-              },
-            })
-          );
+        this.testResultStats = runner.stats;
+        this.gotResults();
       });
   }
 
@@ -58,6 +70,23 @@ class CustomEventReporter extends Mocha.reporters.HTML {
   decreaseIndent() {
     this.indents--;
   }
+
+  gotResults() {
+    if (!this.testResultStats || !this.jUnitReport) {
+      return;
+    }
+
+    window.dispatchEvent(
+      new CustomEvent('testResult', {
+        detail: {
+          pass: this.testResultStats && this.testResultStats.failures === 0,
+          passes: this.testResultStats.passes,
+          total: this.testResultStats.passes + this.testResultStats.failures,
+          jUnitReport: this.jUnitReport,
+        },
+      })
+    );
+  }
 }
 
 mocha.reporter(CustomEventReporter);

test/support/runPlaywrightTests.js

@@ -3,6 +3,8 @@ const path = require('path');
 const serverProcess = require('child_process').fork(path.resolve(__dirname, '..', 'web_server'), {
   env: { PLAYWRIGHT_TEST: 1 },
 });
+const fs = require('fs');
+const jUnitDirectoryPath = require('./junit_directory_path');
 
 const port = process.env.PORT || 3000;
 const host = 'localhost';
@@ -35,6 +37,18 @@ const runTests = async (browserType) => {
     // Expose a function inside the playwright browser to exit with the right status code when tests pass/fail
     page.exposeFunction('onTestResult', ({ detail }) => {
       console.log(`${browserType.name()} tests complete: ${detail.passes}/${detail.total} passed`);
+
+      try {
+        if (!fs.existsSync(jUnitDirectoryPath)) {
+          fs.mkdirSync(jUnitDirectoryPath);
+        }
+        const filename = `playwright-${browserType.name()}.junit`;
+        fs.writeFileSync(path.join(jUnitDirectoryPath, filename), detail.jUnitReport, { encoding: 'utf-8' });
+      } catch (err) {
+        console.log('Failed to write JUnit report, exiting with code 2: ', err);
+        process.exit(2);
+      }
+
       if (detail.pass) {
         browser.close();
         resolve();

test/tasks/grunt-mocha.js

@@ -45,6 +45,8 @@ module.exports = function (grunt) {
         runTests += ' --fgrep ' + fgrep;
       }
 
+      runTests += ` --reporter ${path.join(__dirname, '..', 'support', 'mocha_reporter.js')}`;
+
       var done = this.async(),
         nodeExecutable = 'node';
 

webpack.config.js

@@ -5,6 +5,7 @@ const banner = require('./src/fragments/license');
 const CopyPlugin = require('copy-webpack-plugin');
 // This is needed for baseUrl to resolve correctly from tsconfig
 const TsconfigPathsPlugin = require('tsconfig-paths-webpack-plugin');
+const nodeExternals = require('webpack-node-externals');
 
 const baseConfig = {
   mode: 'production',
@@ -221,6 +222,87 @@ const commonJsNoEncryptionConfig = {
   },
 };
 
+/**
+ * We create a bundle that exposes the mocha-junit-reporter package. We do this for the following reasons:
+ *
+ * - Browser:
+ *
+ *   1. This package is designed for Node only and hence requires polyfills of Node libraries (e.g. `stream`, `path`) — webpack takes care of this for us.
+ *   2. The package is not compatible with RequireJS and hence we don’t have any easy way to directly load it in our tests — the webpack bundle exposes it as a global named MochaJUnitReporter.
+ *
+ * - Node: The library uses optional chaining syntax, which is not supported by Node 12.
+ */
+function createMochaJUnitReporterConfigs() {
+  const dir = path.join(__dirname, 'test', 'support', 'mocha_junit_reporter');
+
+  const baseConfig = {
+    mode: 'development',
+    entry: path.join(dir, 'index.js'),
+    module: {
+      rules: [
+        {
+          // The optional chaining syntax used by mocha-junit-reporter is not supported by:
+          //
+          // 1. webpack 4 (which we’re currently using)
+          // 2. Node 12 (see above)
+          //
+          // For these reasons, we transpile using Babel.
+          test: /\.js$/,
+          loader: 'babel-loader',
+          options: {
+            presets: [['@babel/preset-env']],
+          },
+        },
+      ],
+    },
+    externals: {
+      mocha: 'mocha.Mocha',
+    },
+    output: {
+      path: path.join(dir, 'build'),
+    },
+  };
+
+  const browserConfig = {
+    ...baseConfig,
+    externals: {
+      mocha: 'mocha.Mocha',
+    },
+    output: {
+      ...baseConfig.output,
+      filename: 'browser.js',
+      library: 'MochaJUnitReporter',
+    },
+    resolve: {
+      modules: [
+        // Webpack doesn’t provide a useful shim for the `fs` module, so we provide our own.
+        path.resolve(dir, 'shims'),
+        'node_modules',
+      ],
+    },
+  };
+
+  const nodeConfig = {
+    ...baseConfig,
+    target: 'node',
+    output: {
+      ...baseConfig.output,
+      filename: 'node.js',
+      libraryTarget: 'umd',
+    },
+    // Don’t bundle any packages except mocha-junit-reporter. Using the
+    // webpack-node-externals library which I saw mentioned on
+    // https://v4.webpack.js.org/configuration/externals/#function; there may
+    // be a simpler way of doing this but seems OK.
+    externals: [nodeExternals({ allowlist: 'mocha-junit-reporter' })],
+  };
+
+  return {
+    mochaJUnitReporterBrowser: browserConfig,
+    mochaJUnitReporterNode: nodeConfig,
+  };
+}
+
 module.exports = {
   node: nodeConfig,
   browser: browserConfig,
@@ -232,4 +314,5 @@ module.exports = {
   noEncryptionMin: noEncryptionMinConfig,
   commonJs: commonJsConfig,
   commonJsNoEncryption: commonJsNoEncryptionConfig,
+  ...createMochaJUnitReporterConfigs(),
 };