From 53432e568af92cf5fa888a096ea7119c7afc6b38 Mon Sep 17 00:00:00 2001 From: Timo Tijhof Date: Sun, 21 Feb 2021 02:45:20 +0000 Subject: [PATCH] [BREAKING CHANGE] Restore most consumer back-compat with v1 In light of the shift in direction per https://github.com/js-reporters/js-reporters/issues/133, I'm reverting (most of) cce0e4d846e17cfec5e8d1b5c4a6dd615549c2c5 so as to allow the next release to more similar to the previous, and to make upgrading easy, allowing most reporters to keep working with very minimal changes (if any). Instead, I'll focus on migrating consumers of js-reporters to use TAP tools directly where available, and to otherwise reduce use of js-reporters to purely the adapting and piping to TapReporter. * Revert `RunStart.testCounts` > `RunStart.counts` (idem RunEnd). * Revert `TestStart.suitName` > `TestStart.parentName` (idem TestEnd). * Revert Test allowing Test as child, restore Suite. This un-fixes https://github.com/js-reporters/js-reporters/issues/126, which will be declined. Frameworks adapted to TAP by js-reporters will not supported nested tests. Frameworks directly providing TAP 13 can one of several strategies to express relationships in a backwards-compatible manner, e.g. like we do in js-reporters by flattening with '>' symbol, or through indentation or through other manners proposed in https://github.com/TestAnything/testanything.github.io/pull/36. Refer to https://github.com/js-reporters/js-reporters/issues/133 for questions about how to support TAP. --- README.md | 2 +- lib/adapters/JasmineAdapter.js | 33 +++++----- lib/adapters/MochaAdapter.js | 30 ++++----- lib/adapters/QUnitAdapter.js | 43 ++++++------- lib/helpers.js | 8 +-- lib/reporters/SummaryReporter.js | 42 +++++++++++- package.json | 2 +- spec/cri-draft.adoc | 87 +++++++++++++++++++------ test/fixtures/unit.js | 30 ++++----- test/integration/adapters.js | 49 +++++++++++++- test/integration/reference-data.js | 100 +++++++++++------------------ test/unit/summary-reporter.js | 54 ++++++---------- 12 files changed, 287 insertions(+), 193 deletions(-) diff --git a/README.md b/README.md index 4ed049f..9ea5bd2 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ runner.on('testEnd', (test) => { }); runner.on('runEnd', (run) => { - const counts = run.counts; + const counts = run.testCounts; console.log('Testsuite status: %s', run.status); console.log('Total %d tests: %d passed, %d failed, %d skipped', diff --git a/lib/adapters/JasmineAdapter.js b/lib/adapters/JasmineAdapter.js index b42cd94..6f1ec11 100644 --- a/lib/adapters/JasmineAdapter.js +++ b/lib/adapters/JasmineAdapter.js @@ -15,6 +15,7 @@ module.exports = class JasmineAdapter extends EventEmitter { this.suiteChildren = {}; this.suiteEnds = []; + this.suiteStarts = {}; this.testStarts = {}; this.testEnds = {}; @@ -55,7 +56,7 @@ module.exports = class JasmineAdapter extends EventEmitter { return { name: testStart.name, - parentName: testStart.parentName, + suiteName: testStart.suiteName, fullName: testStart.fullName.slice(), status: (result.status === 'pending') ? 'skipped' : result.status, // TODO: Jasmine 3.4+ has result.duration, use it. @@ -85,15 +86,18 @@ module.exports = class JasmineAdapter extends EventEmitter { this.suiteChildren[result.id] = []; result.children.forEach((child) => { - this.testStarts[child.id] = { - name: child.description, - parentName: name, - fullName: [...fullName, child.description] - }; - if (child.id.indexOf('suite') === 0) { + this.suiteStarts[child.id] = { + name: child.description, + fullName: [...fullName, child.description] + }; this.processSuite(child, fullName.slice(), parentIds.slice()); } else { + this.testStarts[child.id] = { + name: child.description, + suiteName: name, + fullName: [...fullName, child.description] + }; // Update flat list of test children parentIds.forEach((id) => { this.suiteChildren[id].push(child.id); @@ -108,13 +112,10 @@ module.exports = class JasmineAdapter extends EventEmitter { const helperData = helpers.aggregateTests(tests); return { name: testStart.name, - parentName: testStart.parentName, fullName: testStart.fullName, // Jasmine has result.status, but does not propagate 'todo' or 'skipped' status: helperData.status, - runtime: result.duration || helperData.runtime, - errors: [], - assertions: [] + runtime: result.duration || helperData.runtime }; } @@ -131,14 +132,14 @@ module.exports = class JasmineAdapter extends EventEmitter { this.emit('runStart', { name: null, - counts: { + testCounts: { total: total } }); } onSuiteStarted (result) { - this.emit('testStart', this.testStarts[result.id]); + this.emit('suiteStart', this.suiteStarts[result.id]); } onSpecStarted (result) { @@ -152,9 +153,9 @@ module.exports = class JasmineAdapter extends EventEmitter { } onSuiteDone (result) { - const suiteEnd = this.createSuiteEnd(this.testStarts[result.id], result); + const suiteEnd = this.createSuiteEnd(this.suiteStarts[result.id], result); this.suiteEnds.push(suiteEnd); - this.emit('testEnd', suiteEnd); + this.emit('suiteEnd', suiteEnd); } onJasmineDone (doneInfo) { @@ -164,7 +165,7 @@ module.exports = class JasmineAdapter extends EventEmitter { this.emit('runEnd', { name: null, status: helperData.status, - counts: helperData.counts, + testCounts: helperData.testCounts, runtime: helperData.runtime }); } diff --git a/lib/adapters/MochaAdapter.js b/lib/adapters/MochaAdapter.js index 7ba7f3b..b1e254e 100644 --- a/lib/adapters/MochaAdapter.js +++ b/lib/adapters/MochaAdapter.js @@ -35,7 +35,6 @@ module.exports = class MochaAdapter extends EventEmitter { convertToSuiteStart (mochaSuite) { return { name: mochaSuite.title, - parentName: (mochaSuite.parent && !mochaSuite.parent.root) ? mochaSuite.parent.title : null, fullName: this.titlePath(mochaSuite) }; } @@ -47,25 +46,22 @@ module.exports = class MochaAdapter extends EventEmitter { return { name: mochaSuite.title, - parentName: (mochaSuite.parent && !mochaSuite.parent.root) ? mochaSuite.parent.title : null, fullName: this.titlePath(mochaSuite), status: helperData.status, - runtime: helperData.runtime, - errors: [], - assertions: [] + runtime: helperData.runtime }; } convertTest (mochaTest) { - let parentName; + let suiteName; let fullName; if (!mochaTest.parent.root) { - parentName = mochaTest.parent.title; + suiteName = mochaTest.parent.title; fullName = this.titlePath(mochaTest.parent); // Add also the test name. fullName.push(mochaTest.title); } else { - parentName = null; + suiteName = null; fullName = [mochaTest.title]; } @@ -85,7 +81,7 @@ module.exports = class MochaAdapter extends EventEmitter { return { name: mochaTest.title, - parentName, + suiteName, fullName, status, runtime, @@ -96,7 +92,7 @@ module.exports = class MochaAdapter extends EventEmitter { // It is a "test start". return { name: mochaTest.title, - parentName, + suiteName, fullName }; } @@ -130,7 +126,7 @@ module.exports = class MochaAdapter extends EventEmitter { }); this.emit('runStart', { name: null, - counts: { + testCounts: { total: total } }); @@ -138,7 +134,7 @@ module.exports = class MochaAdapter extends EventEmitter { onSuite (mochaSuite) { if (!mochaSuite.root) { - this.emit('testStart', this.convertToSuiteStart(mochaSuite)); + this.emit('suiteStart', this.convertToSuiteStart(mochaSuite)); } } @@ -175,11 +171,11 @@ module.exports = class MochaAdapter extends EventEmitter { onSuiteEnd (mochaSuite) { if (!mochaSuite.root) { - const testEnd = this.convertToSuiteEnd(mochaSuite); - this.emit('testEnd', testEnd); + const suiteEnd = this.convertToSuiteEnd(mochaSuite); + this.emit('suiteEnd', suiteEnd); this.finalCounts.total++; - this.finalCounts[testEnd.status]++; - this.finalRuntime += testEnd.runtime || 0; + this.finalCounts[suiteEnd.status]++; + this.finalRuntime += suiteEnd.runtime || 0; } } @@ -187,7 +183,7 @@ module.exports = class MochaAdapter extends EventEmitter { this.emit('runEnd', { name: null, status: this.finalCounts.failed > 0 ? 'failed' : 'passed', - counts: this.finalCounts, + testCounts: this.finalCounts, runtime: this.finalRuntime }); } diff --git a/lib/adapters/QUnitAdapter.js b/lib/adapters/QUnitAdapter.js index d11ced0..19b787c 100644 --- a/lib/adapters/QUnitAdapter.js +++ b/lib/adapters/QUnitAdapter.js @@ -28,10 +28,10 @@ module.exports = class QUnitAdapter extends EventEmitter { QUnit.done(this.onDone.bind(this)); } - prepTestEnd (parentName, parentNames, details) { + prepTestEnd (suiteName, parentNames, details) { const testEnd = this.testEnds[details.testId] = { name: details.name, - parentName: parentName, + suiteName: suiteName, fullName: [...parentNames, details.name], // Placeholders, populated by onTestDone() and onLog() status: null, @@ -45,22 +45,18 @@ module.exports = class QUnitAdapter extends EventEmitter { processModule (qunitModule) { const fullName = qunitModule.name.split(this.delim); const name = fullName.slice(-1)[0]; - const parentName = fullName.length >= 2 ? fullName.slice(-2, -1)[0] : null; const childTests = qunitModule.tests.map((details) => { return this.prepTestEnd(name, fullName, details); }); return { - testEnd: { + suiteEnd: { name, - parentName, fullName, // Placeholders, populated by emitTests() status: null, - runtime: null, - errors: [], - assertions: [] + runtime: null }, childTests, childModules: [] @@ -83,7 +79,7 @@ module.exports = class QUnitAdapter extends EventEmitter { modules = this.QUnit.config.modules; } - // Prepare all testEnd leafs + // Prepare all suiteEnd leafs modules = modules.map(this.processModule.bind(this)); // For CRI, each module will be represented as a wrapper test @@ -94,10 +90,10 @@ module.exports = class QUnitAdapter extends EventEmitter { // module and add the current module to it as a child, among the test leafs. const globalModules = []; modules.forEach((mod) => { - if (mod.testEnd.parentName !== null) { - const parentFullName = mod.testEnd.fullName.slice(0, -1); + if (mod.suiteEnd.fullName.length > 1) { + const parentFullName = mod.suiteEnd.fullName.slice(0, -1); modules.forEach((otherMod) => { - if (otherMod.testEnd.fullName.join(this.delim) === parentFullName.join(this.delim)) { + if (otherMod.suiteEnd.fullName.join(this.delim) === parentFullName.join(this.delim)) { otherMod.childModules.push(mod); } }); @@ -115,7 +111,10 @@ module.exports = class QUnitAdapter extends EventEmitter { }); const emitModule = (mod) => { - this.emit('testStart', helpers.createTestStart(mod.testEnd)); + this.emit('suiteStart', { + name: mod.suiteEnd.name, + fullName: mod.suiteEnd.fullName.slice() + }); mod.childTests.forEach((testEnd) => { this.emit('testStart', helpers.createTestStart(testEnd)); @@ -125,18 +124,18 @@ module.exports = class QUnitAdapter extends EventEmitter { // This is non-recursive and can be because we emit modules in the original // depth-first execution order. We fill in the status/runtime placeholders - // for the testEnd object of a nested module, and then later a parent module - // follows and sees that child testEnd object by reference and can propagate + // for the suiteEnd object of a nested module, and then later a parent module + // follows and sees that child suiteEnd object by reference and can propagate // and aggregate the information further. const helperData = helpers.aggregateTests([ ...mod.childTests, - ...mod.childModules.map(child => child.testEnd) + ...mod.childModules.map(child => child.suiteEnd) ]); - mod.testEnd.status = helperData.status; - mod.testEnd.runtime = helperData.runtime; + mod.suiteEnd.status = helperData.status; + mod.suiteEnd.runtime = helperData.runtime; - this.moduleEnds.push(mod.testEnd); - this.emit('testEnd', mod.testEnd); + this.moduleEnds.push(mod.suiteEnd); + this.emit('suiteEnd', mod.suiteEnd); }; this.globalModules.forEach(emitModule); } @@ -146,7 +145,7 @@ module.exports = class QUnitAdapter extends EventEmitter { this.emit('runStart', { name: null, - counts: { + testCounts: { total: this.totalBegin } }); @@ -199,7 +198,7 @@ module.exports = class QUnitAdapter extends EventEmitter { this.emit('runEnd', { name: null, status: helperData.status, - counts: helperData.counts, + testCounts: helperData.testCounts, runtime: details.runtime || null }); } diff --git a/lib/helpers.js b/lib/helpers.js index d7625dc..fb8c9e9 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -1,12 +1,12 @@ function aggregateTests (all) { - const counts = { + const testCounts = { passed: all.filter((test) => test.status === 'passed').length, failed: all.filter((test) => test.status === 'failed').length, skipped: all.filter((test) => test.status === 'skipped').length, todo: all.filter((test) => test.status === 'todo').length, total: all.length }; - const status = counts.failed ? 'failed' : 'passed'; + const status = testCounts.failed ? 'failed' : 'passed'; let runtime = 0; all.forEach((test) => { @@ -15,7 +15,7 @@ function aggregateTests (all) { return { status, - counts, + testCounts, runtime }; } @@ -23,7 +23,7 @@ function aggregateTests (all) { function createTestStart (testEnd) { return { name: testEnd.name, - parentName: testEnd.parentName, + suiteName: testEnd.suiteName, fullName: testEnd.fullName.slice() }; } diff --git a/lib/reporters/SummaryReporter.js b/lib/reporters/SummaryReporter.js index c8da679..21e6a0d 100644 --- a/lib/reporters/SummaryReporter.js +++ b/lib/reporters/SummaryReporter.js @@ -9,7 +9,7 @@ module.exports = class SummaryReporter extends EventEmitter { name: null, tests: [], status: null, - counts: null, + testCounts: null, runtime: null }; @@ -21,6 +21,40 @@ module.exports = class SummaryReporter extends EventEmitter { this.summary = null; + runner.on('suiteEnd', (suiteEnd) => { + const ownFull = suiteEnd.fullName.join('>'); + const suiteName = suiteEnd.fullName.length >= 2 ? suiteEnd.fullName.slice(-2, -1)[0] : null; + const parentFull = suiteEnd.fullName.length > 1 ? suiteEnd.fullName.slice(0, -1).join('>') : null; + + let children; + if (hasOwn.call(this.detachedChildren, ownFull)) { + children = this.detachedChildren[ownFull]; + delete this.detachedChildren[ownFull]; + } else { + children = []; + } + + const test = { + ...suiteEnd, + suiteName: suiteName, + errors: [], + assertions: [], + tests: children + }; + + if (parentFull === null) { + this.top.tests.push(test); + } else if (hasOwn.call(this.attachedTests, parentFull)) { + this.attachedTests[parentFull].tests.push(test); + } else if (hasOwn.call(this.detachedChildren, parentFull)) { + this.detachedChildren[parentFull].push(test); + } else { + this.detachedChildren[parentFull] = [test]; + } + + this.attachedTests[ownFull] = test; + }); + runner.on('testEnd', (testEnd) => { const ownFull = testEnd.fullName.join('>'); const parentFull = testEnd.fullName.length > 1 ? testEnd.fullName.slice(0, -1).join('>') : null; @@ -54,11 +88,15 @@ module.exports = class SummaryReporter extends EventEmitter { runner.once('runEnd', (runEnd) => { this.top.name = runEnd.name; this.top.status = runEnd.status; - this.top.counts = runEnd.counts; + this.top.testCounts = runEnd.testCounts; this.top.runtime = runEnd.runtime; this.summary = this.top; + const problems = Object.keys(this.detachedChildren).join(', '); + if (problems.length) { + console.error('detachedChildren:', problems); + } this.top = null; this.attachedTests = null; this.detachedChildren = null; diff --git a/package.json b/package.json index 72d936d..a488793 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "build": "rollup -c", "prepare": "npm run build", "lint": "semistandard", - "test": "npm run lint && npm run test-unit && npm run test-integration && npm run test-browser", + "test": "npm run test-unit && npm run test-integration && npm run test-browser && npm run lint", "test-unit": "qunit 'test/unit/*.js'", "test-integration": "qunit test/integration/adapters.js", "test-browser": "karma start", diff --git a/spec/cri-draft.adoc b/spec/cri-draft.adoc index 732b1bf..6d8ada7 100644 --- a/spec/cri-draft.adoc +++ b/spec/cri-draft.adoc @@ -14,7 +14,7 @@ Participate:: https://gitter.im/js-reporters/js-reporters[Chat room on Gitter] Last updated:: - 17 January 2021 + 20 Februrary 2021 Abstract:: This specification defines JavaScript APIs for reporting progress and results of executing software tests. @@ -45,15 +45,12 @@ Failed assertion:: An assertion that has evaluated to boolean false. [[test]] Test:: - A test is a named group containing zero or more assertions, and zero or more child tests. + + A test is a named group containing zero or more assertions. + + It is recommended that all tests contain assertions, but this is not required. For example, a testing framework that only records failed assertions (such as a testing framework that is decoupled from an assertion library and uses exceptions to discover failures), might not distinguish between a test with passing assertions and a test with no assertions. If a testing framework is generally aware assertions and if it considers absence of those an error, then it should ensure the test or test [[run]] is marked as failing. For example, by implicitly producing a failed assertion. + + - A test that contains at least one child test, is also known as a "parent test". + - + - In QUnit, a test may be defined by calling `QUnit.test()`, and a parent test is known as a "module", defined by calling `QUnit.module()`. + - In Mocha and Jasmine BDD, a test is known as a "spec", defined by calling `it()`, with parent tests defined by calling `describe()`. + - In JUnit and other xUnit-derivatives, tests are first grouped in a `TestCase` which are then further grouped under a ``. In the CRI standard, both of these are considered a test. + In QUnit, a test may be defined by calling `QUnit.test()`. + + In Mocha and Jasmine BDD, a test is known as a "spec", defined by calling `it()`. + (https://en.wikipedia.org/wiki/Test_case[Learn more]) + Skipped test:: @@ -66,15 +63,23 @@ Todo test:: + See also the `TODO` directive of the https://testanything.org/tap-version-13-specification.html#directives[TAP specification]. +[[suite]] Suite:: + A suite is a named group representing zero or more tests, and zero or more other suites. A suite that is part of another suite may also be called a "child suite". A suite that holds one or more child suites may also be called an "ancestor suite". + + (https://en.wikipedia.org/wiki/Test_case[Learn more]) + + + + In QUnit, a suite is known as a "module", defined by calling `QUnit.module()`. + + In Mocha and Jasmine BDD, a suite is defined by calling `describe()`. + + In JUnit and other xUnit-derivatives, tests are first grouped in a `TestCase` which are then further grouped under a ``. In the CRI standard, both of these are considered a suite. + [[run]] Run:: - A run is a single top-level group representing all tests that a producer is planning to report events about. + A run is a single top-level group representing all tests and test suites that a producer is planning to report events about. Reporter:: A JavaScript program that consumes information from a <>. For example, to render an HTML graphical user interface, to write command-line output in the https://testanything.org/[TAP] format, write results to a https://llg.cubic.org/docs/junit/[JUnit XML] artifact file, or serialize the information and transfer it over a socket to another server or process. [TIP] ===== -The use of "Test" (and "Parent test") as the main data structure was decided in https://github.com/js-reporters/js-reporters/issues/12[issue #12], and later revised in https://github.com/js-reporters/js-reporters/issues/126[issue #126]. +The use of "Suite" and "Test" as the main two data structues was decided in https://github.com/js-reporters/js-reporters/issues/12[issue #12], and later revised in https://github.com/js-reporters/js-reporters/issues/126[issue #126]. ===== == Events @@ -103,7 +108,7 @@ Read https://github.com/js-reporters/js-reporters/issues/62[issue #62] for the d === `runStart` event -The **runStart** event indicates the beginning of a <>. It must be emitted exactly once, and before any <>. +The **runStart** event indicates the beginning of a <>. It must be emitted exactly once, and before any <> or <>. Callback parameters: @@ -116,7 +121,7 @@ producer.on('runStart', (runStart) => { … }); === `runEnd` event -The **runEnd** event indicates the end of a <>. It must be emitted exactly once, after the last of any <>. +The **runEnd** event indicates the end of a <>. It must be emitted exactly once, after the last of any <> or <>. Callback parameters: @@ -127,9 +132,35 @@ Callback parameters: producer.on('runEnd', (runEnd) => { … }); ---- +=== `suiteStart` event + +The **suiteStart** event indicates the beginning of a <>. It must eventually be followed by a corresponding <>. + +Callback parameters: + +* <> **suiteStart**: Basic information about a suite. + +[source,javascript] +---- +producer.on('suiteStart', (suiteStart) => { … }); +---- + +=== `suiteEnd` event + +The **suiteEnd** event indicates the end of a <>. It must be emitted after its corresponding <>. + +Callback parameters: + +* <> **suiteEnd**: Basic information about a completed suite. + +[source,javascript] +---- +producer.on('suiteEnd', (suiteEnd) => { … }); +---- + === `testStart` event -The **testStart** event indicates the beginning of a <>. It must eventually be followed by a corresponding <>. A producer may emit several <> events before any corresponding <>, for example when there are child tests, or tests that run concurrently. +The **testStart** event indicates the beginning of a <>. It must eventually be followed by a corresponding <>. A producer may emit several <> events before any corresponding <>, for example when there are child tests, or tests that run concurrently. Callback parameters: @@ -147,7 +178,7 @@ If a producer has no real-time information about test execution, it may simply e === `testEnd` event -The **testEnd** event indicates the end of a <>. It must be emitted after its corresponding <>. +The **testEnd** event indicates the end of a <>. It must be emitted after its corresponding <>. Callback parameters: @@ -162,6 +193,24 @@ producer.on('testEnd', (testEnd) => { … }); The following data structures must be implemented as objects that have the specified fields as own properties. The objects are not required to be an instance of any specific class. They may be null-inherited objects, plain objects, or an instance of any public or private class. +=== SuiteStart + +`SuiteStart` object: + +* `string` **name**: Name of the suite. +* `Array` **fullName**: List of one or more strings, containing (in order) the names of any grandancestor suites, the name of the suite. + +=== SuiteEnd + +`SuiteEnd` object: + +* `string` **name**: Name of the suite. +* `Array` **fullName**: List of one or more strings, containing (in order) the names of any grandancestor suites, the name of the suite. +* `string` **status**: Aggregate result of all tests, one of: +** **failed** if at least one test has failed. +** **passed**, if there were no failed tests, which means there either were no tests, or tests only had passed, skipped, or todo statuses. +* `number|null` **runtime**: Optional duration of the suite in milliseconds. + === RunStart The plan for the <>. @@ -169,7 +218,7 @@ The plan for the <>. `RunStart` object: * `string|null` **name**: Name of the overall run, or `null` if the producer is unaware of a name. -* `Object` **counts**: Aggregate counts about tests. +* `Object` **testCounts**: Aggregate counts about tests. ** `number|null` **total**: Total number of tests the producer is expecting to emit events for, e.g. if there would be no unexpected failures. If may be `null` if the total is not known ahead of time. === RunEnd @@ -182,7 +231,7 @@ Summary of test results from the completed <>. * `string` **status**: Aggregate result of all tests, one of: ** **failed** if at least one test has failed. ** **passed**, if there were no failed tests, which means there either were no tests, or tests only had passed, skipped, or todo statuses. -* `Object` **counts**: Aggregate counts about tests. +* `Object` **testCounts**: Aggregate counts about tests. ** `number` **passed**: Number of passed tests. ** `number` **failed**: Number of failed tests. ** `number` **skipped**: Number of skipped tests. @@ -197,8 +246,8 @@ Basic information about a <>. `TestStart` object: * `string` **name**: Name of the test. -* `string|null` **parentName**: Name of the parent test, or `null` if it has no parent. -* `Array` **fullName**: List of one or more strings, containing (in order) the names of any grand parent tests, the name of any parent test, and the name of the test itself. +* `string|null` **suiteName**: Name of the suite the test belongs to, or `null` if it has no suite. +* `Array` **fullName**: List of one or more strings, containing (in order) the names of any grandancestor suites, the name of the suite, and the name of the test itself. === TestEnd @@ -207,8 +256,8 @@ Result of a completed <>. This is a superset of <>. `TestEnd` object: * `string` **name**: Name of the test. -* `string|null` **parentName**: Name of the parent test, or `null` if it has no parent. -* `Array` **fullName**: List of one or more strings, containing (in order) the names of any grand parent tests, the name of any parent test, and the name of the test itself. +* `string|null` **suiteName**: Name of the suite the test belongs to, or `null` if it has no suite. +* `Array` **fullName**: List of one or more strings, containing (in order) the names of any ancestor suites, the name of the suite, and the name of the test itself. * `string` **status**: Result of the test, one of: ** **passed**, if all assertions have passed, or if no assertions were recorded. ** **failed**, if at least one assertion has failed or if the test is todo and its assertions unexpectedly all passed. diff --git a/test/fixtures/unit.js b/test/fixtures/unit.js index ae2e6a9..efc37d3 100644 --- a/test/fixtures/unit.js +++ b/test/fixtures/unit.js @@ -14,12 +14,12 @@ function copyErrors (testEnd) { module.exports = { passingTestStart: { name: 'pass', - parentName: null, + suiteName: null, fullName: ['pass'] }, passingTest: { name: 'pass', - parentName: null, + suiteName: null, fullName: ['pass'], status: 'passed', runtime: 0, @@ -59,7 +59,7 @@ module.exports = { ], actualUndefinedTest: copyErrors({ name: 'Failing', - parentName: null, + suiteName: null, fullName: ['Failing'], status: 'failed', runtime: 0, @@ -72,7 +72,7 @@ module.exports = { }), actualInfinity: copyErrors({ name: 'Failing', - parentName: null, + suiteName: null, fullName: ['Failing'], status: 'failed', runtime: 0, @@ -85,7 +85,7 @@ module.exports = { }), actualStringChar: copyErrors({ name: 'Failing', - parentName: null, + suiteName: null, fullName: ['Failing'], status: 'failed', runtime: 0, @@ -98,7 +98,7 @@ module.exports = { }), actualStringNum: copyErrors({ name: 'Failing', - parentName: null, + suiteName: null, fullName: ['Failing'], status: 'failed', runtime: 0, @@ -111,7 +111,7 @@ module.exports = { }), actualStringBool: copyErrors({ name: 'Failing', - parentName: null, + suiteName: null, fullName: ['Failing'], status: 'failed', runtime: 0, @@ -124,7 +124,7 @@ module.exports = { }), actualStringOneTailLn: copyErrors({ name: 'Failing', - parentName: null, + suiteName: null, fullName: ['Failing'], status: 'failed', runtime: 0, @@ -144,7 +144,7 @@ module.exports = { ...`, actualStringTwoTailLn: copyErrors({ name: 'Failing', - parentName: null, + suiteName: null, fullName: ['Failing'], status: 'failed', runtime: 0, @@ -166,7 +166,7 @@ module.exports = { ...`, actualZero: copyErrors({ name: 'Failing', - parentName: null, + suiteName: null, fullName: ['Failing'], status: 'failed', runtime: 0, @@ -179,7 +179,7 @@ module.exports = { }), actualArray: copyErrors({ name: 'Failing', - parentName: null, + suiteName: null, fullName: ['Failing'], status: 'failed', runtime: 0, @@ -198,7 +198,7 @@ module.exports = { ...`, expectedUndefinedTest: copyErrors({ name: 'fail', - parentName: null, + suiteName: null, fullName: [], status: 'failed', runtime: 0, @@ -211,7 +211,7 @@ module.exports = { }), expectedFalsyTest: copyErrors({ name: 'fail', - parentName: null, + suiteName: null, fullName: [], status: 'failed', runtime: 0, @@ -224,7 +224,7 @@ module.exports = { }), skippedTest: { name: 'skip', - parentName: null, + suiteName: null, fullName: ['skip'], status: 'skipped', runtime: 0, @@ -233,7 +233,7 @@ module.exports = { }, todoTest: { name: 'todo', - parentName: null, + suiteName: null, fullName: ['todo'], status: 'todo', runtime: 0, diff --git a/test/integration/adapters.js b/test/integration/adapters.js index 7618629..ab959be 100644 --- a/test/integration/adapters.js +++ b/test/integration/adapters.js @@ -17,6 +17,13 @@ function collectDataFromRunner (collectedData, done, runner) { runner.on('runStart', (runStart) => { collectedData.push(['runStart', runStart]); }); + runner.on('suiteStart', (suiteStart) => { + collectedData.push(['suiteStart', suiteStart]); + }); + runner.on('suiteEnd', (suiteEnd) => { + normalizeSuiteEnd(suiteEnd); + collectedData.push(['suiteEnd', suiteEnd]); + }); runner.on('testStart', (testStart) => { collectedData.push(['testStart', testStart]); }); @@ -58,6 +65,12 @@ function normalizeTestEnd (test) { } } +function normalizeSuiteEnd (suiteEnd) { + if (Number.isFinite(suiteEnd.runtime)) { + suiteEnd.runtime = 42; + } +} + function normalizeRunEnd (runEnd) { if (Number.isFinite(runEnd.runtime)) { runEnd.runtime = 42; @@ -73,7 +86,7 @@ function fixExpectedData (adapter, expectedData) { data.assertions = []; } } - if (eventName === 'testEnd' || eventName === 'runEnd') { + if (eventName === 'testEnd' || eventName === 'suiteEnd' || eventName === 'runEnd') { if (Number.isFinite(data.runtime)) { data.runtime = 42; } @@ -140,6 +153,40 @@ QUnit.module('Adapters integration', function () { }); }); + test('Event "suiteStart" data', assert => { + const actuals = collectedData.filter(pair => pair[0] === 'suiteStart'); + const expecteds = expectedData.filter(pair => pair[0] === 'suiteStart'); + assert.propEqual( + actuals.map(expected => expected[1].name), + expecteds.map(pair => pair[1].name), + 'Suite names' + ); + expecteds.forEach((expected, i) => { + assert.propEqual( + actuals[i][1], + expected[1], + `Event data for suiteStart#${i}` + ); + }); + }); + + test('Event "suiteEnd" data', assert => { + const actuals = collectedData.filter(pair => pair[0] === 'suiteEnd'); + const expecteds = expectedData.filter(pair => pair[0] === 'suiteEnd'); + assert.propEqual( + actuals.map(expected => expected[1].name), + expecteds.map(pair => pair[1].name), + 'Suite names' + ); + expecteds.forEach((expected, i) => { + assert.propEqual( + actuals[i][1], + expected[1], + `Event data for suiteEnd#${i}` + ); + }); + }); + test('Event "runStart" data', assert => { const actuals = collectedData.filter(pair => pair[0] === 'runStart'); expectedData.filter(pair => pair[0] === 'runStart').forEach((expected, i) => { diff --git a/test/integration/reference-data.js b/test/integration/reference-data.js index 8b4ea67..45addbd 100644 --- a/test/integration/reference-data.js +++ b/test/integration/reference-data.js @@ -8,14 +8,14 @@ const passedAssertion = { const globalTestStart = { name: 'global test', - parentName: null, + suiteName: null, fullName: [ 'global test' ] }; const globalTestEnd = { name: 'global test', - parentName: null, + suiteName: null, fullName: [ 'global test' ], @@ -29,7 +29,7 @@ const globalTestEnd = { const passingTestStart1 = { name: 'should pass', - parentName: 'suite with passing test', + suiteName: 'suite with passing test', fullName: [ 'suite with passing test', 'should pass' @@ -37,14 +37,13 @@ const passingTestStart1 = { }; const passingSuiteStart = { name: 'suite with passing test', - parentName: null, fullName: [ 'suite with passing test' ] }; const passingTestEnd1 = { name: 'should pass', - parentName: 'suite with passing test', + suiteName: 'suite with passing test', fullName: [ 'suite with passing test', 'should pass' @@ -58,19 +57,16 @@ const passingTestEnd1 = { }; const passingSuiteEnd = { name: 'suite with passing test', - parentName: null, fullName: [ 'suite with passing test' ], status: 'passed', - runtime: 42, - errors: [], - assertions: [] + runtime: 42 }; const skippedTestStart1 = { name: 'should skip', - parentName: 'suite with skipped test', + suiteName: 'suite with skipped test', fullName: [ 'suite with skipped test', 'should skip' @@ -78,14 +74,13 @@ const skippedTestStart1 = { }; const skippedSuiteStart = { name: 'suite with skipped test', - parentName: null, fullName: [ 'suite with skipped test' ] }; const skippedTestEnd1 = { name: 'should skip', - parentName: 'suite with skipped test', + suiteName: 'suite with skipped test', fullName: [ 'suite with skipped test', 'should skip' @@ -97,19 +92,16 @@ const skippedTestEnd1 = { }; const skippedSuiteEnd = { name: 'suite with skipped test', - parentName: null, fullName: [ 'suite with skipped test' ], status: 'passed', - runtime: 0, - errors: [], - assertions: [] + runtime: 0 }; const failingTestStart1 = { name: 'should fail', - parentName: 'suite with failing test', + suiteName: 'suite with failing test', fullName: [ 'suite with failing test', 'should fail' @@ -117,14 +109,13 @@ const failingTestStart1 = { }; const failingSuiteStart = { name: 'suite with failing test', - parentName: null, fullName: [ 'suite with failing test' ] }; const failingTestEnd1 = { name: 'should fail', - parentName: 'suite with failing test', + suiteName: 'suite with failing test', fullName: [ 'suite with failing test', 'should fail' @@ -140,19 +131,16 @@ const failingTestEnd1 = { }; const failingSuiteEnd = { name: 'suite with failing test', - parentName: null, fullName: [ 'suite with failing test' ], status: 'failed', - runtime: 42, - errors: [], - assertions: [] + runtime: 42 }; const passingTestStart2 = { name: 'should pass', - parentName: 'suite with tests', + suiteName: 'suite with tests', fullName: [ 'suite with tests', 'should pass' @@ -160,7 +148,7 @@ const passingTestStart2 = { }; const passingTestEnd2 = { name: 'should pass', - parentName: 'suite with tests', + suiteName: 'suite with tests', fullName: [ 'suite with tests', 'should pass' @@ -174,7 +162,7 @@ const passingTestEnd2 = { }; const skippedTestStart2 = { name: 'should skip', - parentName: 'suite with tests', + suiteName: 'suite with tests', fullName: [ 'suite with tests', 'should skip' @@ -182,7 +170,7 @@ const skippedTestStart2 = { }; const failingTestStart2 = { name: 'should fail', - parentName: 'suite with tests', + suiteName: 'suite with tests', fullName: [ 'suite with tests', 'should fail' @@ -190,14 +178,13 @@ const failingTestStart2 = { }; const testSuiteStart = { name: 'suite with tests', - parentName: null, fullName: [ 'suite with tests' ] }; const skippedTestEnd2 = { name: 'should skip', - parentName: 'suite with tests', + suiteName: 'suite with tests', fullName: [ 'suite with tests', 'should skip' @@ -209,7 +196,7 @@ const skippedTestEnd2 = { }; const failingTestEnd2 = { name: 'should fail', - parentName: 'suite with tests', + suiteName: 'suite with tests', fullName: [ 'suite with tests', 'should fail' @@ -225,19 +212,16 @@ const failingTestEnd2 = { }; const testSuiteEnd = { name: 'suite with tests', - parentName: null, fullName: [ 'suite with tests' ], status: 'failed', - runtime: 84, - errors: [], - assertions: [] + runtime: 84 }; const outerTestStart = { name: 'outer test', - parentName: 'outer suite', + suiteName: 'outer suite', fullName: [ 'outer suite', 'outer test' @@ -245,7 +229,7 @@ const outerTestStart = { }; const outerTestEnd = { name: 'outer test', - parentName: 'outer suite', + suiteName: 'outer suite', fullName: [ 'outer suite', 'outer test' @@ -259,7 +243,7 @@ const outerTestEnd = { }; const innerTestStart = { name: 'inner test', - parentName: 'inner suite', + suiteName: 'inner suite', fullName: [ 'outer suite', 'inner suite', @@ -268,7 +252,6 @@ const innerTestStart = { }; const innerSuiteStart = { name: 'inner suite', - parentName: 'outer suite', fullName: [ 'outer suite', 'inner suite' @@ -276,7 +259,7 @@ const innerSuiteStart = { }; const innerTestEnd = { name: 'inner test', - parentName: 'inner suite', + suiteName: 'inner suite', fullName: [ 'outer suite', 'inner suite', @@ -291,46 +274,39 @@ const innerTestEnd = { }; const innerSuiteEnd = { name: 'inner suite', - parentName: 'outer suite', fullName: [ 'outer suite', 'inner suite' ], status: 'passed', - runtime: 42, - errors: [], - assertions: [] + runtime: 42 }; const outerSuiteStart = { name: 'outer suite', - parentName: null, fullName: [ 'outer suite' ] }; const outerSuiteEnd = { name: 'outer suite', - parentName: null, fullName: [ 'outer suite' ], status: 'passed', - runtime: 84, - errors: [], - assertions: [] + runtime: 84 }; const runStart = { name: null, - counts: { + testCounts: { total: 15 } }; const runEnd = { name: null, status: 'failed', - counts: { + testCounts: { passed: 9, failed: 4, skipped: 2, @@ -346,38 +322,38 @@ module.exports = [ ['testStart', globalTestStart], ['testEnd', globalTestEnd], - ['testStart', passingSuiteStart], + ['suiteStart', passingSuiteStart], ['testStart', passingTestStart1], ['testEnd', passingTestEnd1], - ['testEnd', passingSuiteEnd], + ['suiteEnd', passingSuiteEnd], - ['testStart', skippedSuiteStart], + ['suiteStart', skippedSuiteStart], ['testStart', skippedTestStart1], ['testEnd', skippedTestEnd1], - ['testEnd', skippedSuiteEnd], + ['suiteEnd', skippedSuiteEnd], - ['testStart', failingSuiteStart], + ['suiteStart', failingSuiteStart], ['testStart', failingTestStart1], ['testEnd', failingTestEnd1], - ['testEnd', failingSuiteEnd], + ['suiteEnd', failingSuiteEnd], - ['testStart', testSuiteStart], + ['suiteStart', testSuiteStart], ['testStart', passingTestStart2], ['testEnd', passingTestEnd2], ['testStart', skippedTestStart2], ['testEnd', skippedTestEnd2], ['testStart', failingTestStart2], ['testEnd', failingTestEnd2], - ['testEnd', testSuiteEnd], + ['suiteEnd', testSuiteEnd], - ['testStart', outerSuiteStart], + ['suiteStart', outerSuiteStart], ['testStart', outerTestStart], ['testEnd', outerTestEnd], - ['testStart', innerSuiteStart], + ['suiteStart', innerSuiteStart], ['testStart', innerTestStart], ['testEnd', innerTestEnd], - ['testEnd', innerSuiteEnd], - ['testEnd', outerSuiteEnd], + ['suiteEnd', innerSuiteEnd], + ['suiteEnd', outerSuiteEnd], ['runEnd', runEnd] ]; diff --git a/test/unit/summary-reporter.js b/test/unit/summary-reporter.js index 205adc6..b549637 100644 --- a/test/unit/summary-reporter.js +++ b/test/unit/summary-reporter.js @@ -7,13 +7,13 @@ const JsReporters = require('../../index.js'); function playUpwardRun (emitter) { emitter.emit('runStart', { name: null, - counts: { + testCounts: { total: 2 } }); emitter.emit('testEnd', { name: 'foo', - parentName: 'Inner suite', + suiteName: 'Inner suite', fullName: ['Outer suite', 'Inner suite', 'foo'], status: 'passed', runtime: 42, @@ -24,7 +24,7 @@ function playUpwardRun (emitter) { }); emitter.emit('testEnd', { name: 'bar', - parentName: 'Inner suite', + suiteName: 'Inner suite', fullName: ['Outer suite', 'Inner suite', 'bar'], status: 'passed', runtime: 42, @@ -33,28 +33,22 @@ function playUpwardRun (emitter) { { passed: true } ] }); - emitter.emit('testEnd', { + emitter.emit('suiteEnd', { name: 'Inner suite', - parentName: 'Outer suite', fullName: ['Outer suite', 'Inner suite'], status: 'passed', - runtime: 42, - errors: [], - assertions: [] + runtime: 42 }); - emitter.emit('testEnd', { + emitter.emit('suiteEnd', { name: 'Outer suite', - parentName: null, fullName: ['Outer suite'], status: 'passed', - runtime: 42, - errors: [], - assertions: [] + runtime: 42 }); emitter.emit('runEnd', { name: null, status: 'passed', - counts: { + testCounts: { passed: 4, failed: 0, skipped: 0, @@ -70,31 +64,25 @@ function playUpwardRun (emitter) { function playDownwardRun (emitter) { emitter.emit('runStart', { name: null, - counts: { + testCounts: { total: 2 } }); - emitter.emit('testEnd', { + emitter.emit('suiteEnd', { name: 'Outer suite', - parentName: null, fullName: ['Outer suite'], status: 'passed', - runtime: 42, - errors: [], - assertions: [] + runtime: 42 }); - emitter.emit('testEnd', { + emitter.emit('suiteEnd', { name: 'Inner suite', - parentName: 'Outer suite', fullName: ['Outer suite', 'Inner suite'], status: 'passed', - runtime: 42, - errors: [], - assertions: [] + runtime: 42 }); emitter.emit('testEnd', { name: 'foo', - parentName: 'Inner suite', + suiteName: 'Inner suite', fullName: ['Outer suite', 'Inner suite', 'foo'], status: 'passed', runtime: 42, @@ -105,7 +93,7 @@ function playDownwardRun (emitter) { }); emitter.emit('testEnd', { name: 'bar', - parentName: 'Inner suite', + suiteName: 'Inner suite', fullName: ['Outer suite', 'Inner suite', 'bar'], status: 'passed', runtime: 42, @@ -117,7 +105,7 @@ function playDownwardRun (emitter) { emitter.emit('runEnd', { name: null, status: 'passed', - counts: { + testCounts: { passed: 4, failed: 0, skipped: 0, @@ -133,7 +121,7 @@ const expectedSummary = { tests: [ { name: 'Outer suite', - parentName: null, + suiteName: null, fullName: ['Outer suite'], status: 'passed', runtime: 42, @@ -142,7 +130,7 @@ const expectedSummary = { tests: [ { name: 'Inner suite', - parentName: 'Outer suite', + suiteName: 'Outer suite', fullName: ['Outer suite', 'Inner suite'], status: 'passed', runtime: 42, @@ -151,7 +139,7 @@ const expectedSummary = { tests: [ { name: 'foo', - parentName: 'Inner suite', + suiteName: 'Inner suite', fullName: ['Outer suite', 'Inner suite', 'foo'], status: 'passed', runtime: 42, @@ -163,7 +151,7 @@ const expectedSummary = { }, { name: 'bar', - parentName: 'Inner suite', + suiteName: 'Inner suite', fullName: ['Outer suite', 'Inner suite', 'bar'], status: 'passed', runtime: 42, @@ -179,7 +167,7 @@ const expectedSummary = { } ], status: 'passed', - counts: { + testCounts: { passed: 4, failed: 0, skipped: 0,