From 1c4d62a819a2b6149cb2f77685e4e975dc75223e Mon Sep 17 00:00:00 2001 From: amsterget Date: Thu, 27 Aug 2020 20:52:44 +0300 Subject: [PATCH] Several bugfixes and code cleanup(#57) --- README.md | 24 ++- modules/constants.js | 24 +++ modules/context.js | 43 +++-- modules/cucumber-reportportal-formatter.js | 179 ++++++++++++--------- modules/documents-storage.js | 10 +- modules/itemFinders.js | 39 ++--- modules/utils.js | 14 +- package.json | 3 +- 8 files changed, 211 insertions(+), 125 deletions(-) diff --git a/README.md b/README.md index f527961..c136b4d 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ Example: ## Step reporting configuration -By defaut, this agent report the following structure: +By default, this agent reports the following structure: - feature - SUITE - scenario - TEST @@ -177,6 +177,28 @@ To report your steps as logs, you need to pass an additional parameter to the ag This will report your your steps with logs to a log level without creating statistics for every step. +## Reporting skipped cucumber steps as failed + +By default, cucumber marks steps which follow a failed step as `skipped`. +When `scenarioBasedStatistics` is set to `false` (the default behavior) +Report Portal reports these steps as failures to investigate. + +To change this behavior and instead mark skipped steps which follow a failed step as `cancelled`, +you need to add an additional parameter to the agent config: `"reportSkippedCucumberStepsOnFailedTest": false` + +```json +{ + "token": "${rp.token}", + "endpoint": "${rp.endpoint}/api/v1", + "launch": "${rp.launch}", + "project": "${rp.your_project}", + "takeScreenshot": "onFailure", + "reportSkippedCucumberStepsOnFailedTest": false +} +``` + +Steps which are marked as `skipped` that do not follow a failed step will continue to mark the step and the scenario as `skipped`. + ## Attachments Attachments are being reported as logs. You can either just attach a file using cucumber's `this.attach` or specify log level and message: diff --git a/modules/constants.js b/modules/constants.js index 9ac2581..01903de 100644 --- a/modules/constants.js +++ b/modules/constants.js @@ -24,6 +24,7 @@ const STATUSES = { CANCELLED: 'cancelled', INFO: 'info', WARN: 'warn', + STARTED: 'started', PENDING: 'pending', NOT_IMPLEMENTED: 'not_implemented', UNDEFINED: 'undefined', @@ -61,6 +62,28 @@ const RP_EVENTS = { STATUS: 'rp/status', }; +// @see https://github.com/Automattic/cli-table#custom-styles +const TABLE_CONFIG = { + chars: { + top: '', + 'top-left': '', + 'top-mid': '', + 'top-right': '', + mid: '', + 'left-mid': '', + 'mid-mid': '', + 'right-mid': '', + bottom: '', + 'bottom-left': '', + 'bottom-mid': '', + 'bottom-right': '', + }, + style: { + head: [], + border: [], + }, +}; + module.exports = { AFTER_HOOK_URI_TO_SKIP, RP_ENTITY_LAUNCH, @@ -68,4 +91,5 @@ module.exports = { LOG_LEVELS, CUCUMBER_EVENTS, RP_EVENTS, + TABLE_CONFIG, }; diff --git a/modules/context.js b/modules/context.js index 9ef6f43..05c9aea 100644 --- a/modules/context.js +++ b/modules/context.js @@ -15,6 +15,7 @@ */ const { cleanContext } = require('./utils'); +const itemFinders = require('./itemFinders'); class Context { constructor() { @@ -31,14 +32,15 @@ class Context { findStep(event) { let stepObj = null; - const stepSourceLocation = this.context.stepDefinitions.steps[event.index]; + const stepDefinition = this.context.stepDefinitions.steps[event.index]; - if (stepSourceLocation.sourceLocation) { - this.context.isBeforeHook = false; + if (stepDefinition.hookType) { + stepObj = { keyword: stepDefinition.hookType }; + } else { this.context.scenario.steps.forEach((step) => { if ( - stepSourceLocation.sourceLocation.uri === event.testCase.sourceLocation.uri && - stepSourceLocation.sourceLocation.line === step.location.line + stepDefinition.sourceLocation.uri === event.testCase.sourceLocation.uri && + stepDefinition.sourceLocation.line === step.location.line ) { stepObj = step; } @@ -47,25 +49,38 @@ class Context { if (this.context.background) { this.context.background.steps.forEach((step) => { if ( - stepSourceLocation.sourceLocation.uri === event.testCase.sourceLocation.uri && - stepSourceLocation.sourceLocation.line === step.location.line + stepDefinition.sourceLocation.uri === event.testCase.sourceLocation.uri && + stepDefinition.sourceLocation.line === step.location.line ) { stepObj = step; } }); } - } else { - stepObj = { keyword: this.context.isBeforeHook ? 'Before' : 'After' }; } return stepObj; } - countFailedScenarios(uri) { - if (this.context.failedScenarios[uri]) { - this.context.failedScenarios[uri]++; - } else { - this.context.failedScenarios[uri] = 1; + countTotalScenarios(feature, featureUri) { + let total = feature.children.length; + feature.children.forEach((child) => { + if (child.examples) { + child.examples.forEach((ex) => { + total += ex.tableBody.length - 1; + }); + } + }); + this.context.background = itemFinders.findBackground(feature); + if (this.context.background) { + total -= 1; } + + this.context.scenariosCount[featureUri] = { total, done: 0 }; + } + + incrementFailedScenariosCount(uri) { + this.context.failedScenarios[uri] = this.context.failedScenarios[uri] + ? this.context.failedScenarios[uri] + 1 + : 1; } resetContext() { diff --git a/modules/cucumber-reportportal-formatter.js b/modules/cucumber-reportportal-formatter.js index 9336b36..4c0248b 100644 --- a/modules/cucumber-reportportal-formatter.js +++ b/modules/cucumber-reportportal-formatter.js @@ -16,6 +16,7 @@ const { Formatter } = require('cucumber'); const ReportPortalClient = require('@reportportal/client-javascript'); +const Table = require('cli-table3'); const utils = require('./utils'); const Context = require('./context'); const DocumentStorage = require('./documents-storage'); @@ -28,6 +29,7 @@ const { LOG_LEVELS, CUCUMBER_EVENTS, RP_EVENTS, + TABLE_CONFIG, } = require('./constants'); const createRPFormatterClass = (config) => { @@ -54,42 +56,18 @@ const createRPFormatterClass = (config) => { } registerListeners(eventBroadcaster) { - eventBroadcaster.on( - CUCUMBER_EVENTS.GHERKIN_DOCUMENT, - this.onGherkinDocument.bind(this), - ); - eventBroadcaster.on( - CUCUMBER_EVENTS.PICKLE_ACCEPTED, - this.onPickleAccepted.bind(this), - ); - eventBroadcaster.on( - CUCUMBER_EVENTS.TEST_CASE_PREPARED, - this.onTestCasePrepared.bind(this), - ); - eventBroadcaster.on( - CUCUMBER_EVENTS.TEST_CASE_STARTED, - this.onTestCaseStarted.bind(this), - ); - eventBroadcaster.on( - CUCUMBER_EVENTS.TEST_STEP_STARTED, - this.onTestStepStarted.bind(this), - ); - eventBroadcaster.on( - CUCUMBER_EVENTS.TEST_STEP_FINISHED, - this.onTestStepFinished.bind(this), - ); + eventBroadcaster.on(CUCUMBER_EVENTS.GHERKIN_DOCUMENT, this.onGherkinDocument.bind(this)); + eventBroadcaster.on(CUCUMBER_EVENTS.PICKLE_ACCEPTED, this.onPickleAccepted.bind(this)); + eventBroadcaster.on(CUCUMBER_EVENTS.TEST_CASE_PREPARED, this.onTestCasePrepared.bind(this)); + eventBroadcaster.on(CUCUMBER_EVENTS.TEST_CASE_STARTED, this.onTestCaseStarted.bind(this)); + eventBroadcaster.on(CUCUMBER_EVENTS.TEST_STEP_STARTED, this.onTestStepStarted.bind(this)); + eventBroadcaster.on(CUCUMBER_EVENTS.TEST_STEP_FINISHED, this.onTestStepFinished.bind(this)); eventBroadcaster.on( CUCUMBER_EVENTS.TEST_STEP_ATTACHMENT, this.onTestStepAttachment.bind(this), ); - eventBroadcaster.on( - CUCUMBER_EVENTS.TEST_CASE_FINISHED, - this.onTestCaseFinished.bind(this), - ); - eventBroadcaster.on( - CUCUMBER_EVENTS.TEST_RUN_FINISHED, - this.onTestRunFinished.bind(this), - ); + eventBroadcaster.on(CUCUMBER_EVENTS.TEST_CASE_FINISHED, this.onTestCaseFinished.bind(this)); + eventBroadcaster.on(CUCUMBER_EVENTS.TEST_RUN_FINISHED, this.onTestRunFinished.bind(this)); } onGherkinDocument(event) { @@ -113,37 +91,19 @@ const createRPFormatterClass = (config) => { } onPickleAccepted(event) { - const isPickleCached = this.documentsStorage.isAcceptedPickleCached(event); - - if (!isPickleCached) { - this.documentsStorage.cacheAcceptedPickle(event); + const featureUri = utils.getUri(event.uri); + if (!this.documentsStorage.isFeatureDataCached(featureUri)) { + this.documentsStorage.createCachedFeature(featureUri); const featureDocument = itemFinders.findFeature( this.documentsStorage.gherkinDocuments, event, ); - const featureUri = utils.getUri(event.uri); const description = featureDocument.description ? featureDocument.description : featureUri; const { name } = featureDocument; const itemAttributes = utils.createAttributes(featureDocument.tags); - let total = featureDocument.children.length; - let parameters = []; - featureDocument.children.forEach((child) => { - if (child.examples) { - child.examples.forEach((ex) => { - total += ex.tableBody.length - 1; - parameters = parameters.concat(utils.getParameters(ex.tableHeader, ex.tableBody)); - }); - } - }); - - this.contextState.context.background = itemFinders.findBackground(featureDocument); - if (this.contextState.context.background) { - total -= 1; - } - - this.contextState.context.scenariosCount[featureUri] = { total, done: 0 }; + this.contextState.context.countTotalScenarios(featureDocument, featureUri); // BeforeFeature const featureId = this.reportportal.startTestItem( @@ -152,20 +112,27 @@ const createRPFormatterClass = (config) => { startTime: this.reportportal.helpers.now(), type: this.isScenarioBasedStatistics ? 'TEST' : 'SUITE', codeRef: utils.formatCodeRef(event.uri, name), - parameters, description, attributes: itemAttributes, }, this.contextState.context.launchId, ).tempId; - this.documentsStorage.pickleDocuments[event.uri].featureId = featureId; + this.documentsStorage.featureData[utils.getUri(event.uri)].featureId = featureId; } } onTestCasePrepared(event) { this.contextState.context.stepDefinitions = event; - this.contextState.context.isBeforeHook = true; + let hookType = 'Before'; + this.contextState.context.stepDefinitions.steps.forEach((step) => { + if (step.sourceLocation) { + hookType = 'After'; + return; + } + // eslint-disable-next-line no-param-reassign + step.hookType = hookType; + }); } onTestCaseStarted(event) { @@ -177,21 +144,23 @@ const createRPFormatterClass = (config) => { this.documentsStorage.gherkinDocuments, event.sourceLocation, ); + this.contextState.context.scenarioStatus = STATUSES.STARTED; this.contextState.context.background = itemFinders.findBackground(featureDocument); const featureTags = featureDocument.tags; - const pickle = this.documentsStorage.pickleDocuments[utils.getUri(event.sourceLocation.uri)]; const keyword = this.contextState.context.scenario.keyword ? this.contextState.context.scenario.keyword : this.contextState.context.scenario.type; let name = [keyword, this.contextState.context.scenario.name].join(': '); - const pickleTags = pickle.tags - ? pickle.tags.filter((tag) => !featureTags.find(utils.createTagComparator(tag))) + const eventTags = this.contextState.context.scenario.tags + ? this.contextState.context.scenario.tags.filter( + (tag) => !featureTags.find(utils.createTagComparator(tag)), + ) : []; - const itemAttributes = utils.createAttributes(pickleTags); + const itemAttributes = utils.createAttributes(eventTags); const description = this.contextState.context.scenario.description || [utils.getUri(event.sourceLocation.uri), event.sourceLocation.line].join(':'); - const { featureId } = this.documentsStorage.pickleDocuments[event.sourceLocation.uri]; + const { featureId } = this.documentsStorage.featureData[event.sourceLocation.uri]; if (this.contextState.context.lastScenarioDescription !== name) { this.contextState.context.lastScenarioDescription = name; @@ -221,7 +190,7 @@ const createRPFormatterClass = (config) => { } onTestStepStarted(event) { - this.contextState.context.stepStatus = 'failed'; + this.contextState.context.stepStatus = STATUSES.FAILED; this.contextState.context.stepId = null; this.contextState.context.stepSourceLocation = this.contextState.context.stepDefinitions.steps[ @@ -243,9 +212,40 @@ const createRPFormatterClass = (config) => { event, ); - const name = this.contextState.context.step.text + let description; + let name = this.contextState.context.step.text ? `${this.contextState.context.step.keyword} ${this.contextState.context.step.text}` : this.contextState.context.step.keyword; + + if (this.contextState.context.step.argument) { + let stepArguments; + if (this.contextState.context.step.argument.content) { + stepArguments = `"""\n${this.contextState.context.step.argument.content}\n"""`; + } + + if (this.contextState.context.step.argument.rows) { + const rows = this.contextState.context.step.argument.rows.map((row) => + row.cells.map((cell) => { + this.contextState.context.scenario.parameters.forEach((parameter) => { + if (cell.value === `<${parameter.key}>`) { + // eslint-disable-next-line no-param-reassign + cell.value = utils.replaceParameter(cell.value, parameter.key, parameter.value); + } + }); + return cell.value; + }), + ); + const datatable = new Table(TABLE_CONFIG); + datatable.push(...rows); + stepArguments = datatable.toString(); + } + if (this.isScenarioBasedStatistics) { + name += `\n${stepArguments}`; + } else { + description = stepArguments; + } + } + let type = 'STEP'; let isHook = false; if (this.contextState.context.step.keyword === 'Before') { @@ -265,6 +265,7 @@ const createRPFormatterClass = (config) => { this.contextState.context.stepId = this.reportportal.startTestItem( { name, + description, startTime: this.reportportal.helpers.now(), type, codeRef, @@ -293,7 +294,9 @@ const createRPFormatterClass = (config) => { switch (event.result.status) { case STATUSES.PASSED: { this.contextState.context.stepStatus = STATUSES.PASSED; - this.contextState.context.scenarioStatus = STATUSES.PASSED; + if (this.contextState.context.scenarioStatus !== STATUSES.FAILED) { + this.contextState.context.scenarioStatus = STATUSES.PASSED; + } break; } case STATUSES.PENDING: { @@ -304,7 +307,7 @@ const createRPFormatterClass = (config) => { }); this.contextState.context.stepStatus = STATUSES.NOT_IMPLEMENTED; this.contextState.context.scenarioStatus = STATUSES.FAILED; - this.contextState.countFailedScenarios(event.testCase.sourceLocation.uri); + this.contextState.incrementFailedScenariosCount(event.testCase.sourceLocation.uri); break; } case STATUSES.UNDEFINED: { @@ -315,7 +318,7 @@ const createRPFormatterClass = (config) => { }); this.contextState.context.stepStatus = STATUSES.NOT_FOUND; this.contextState.context.scenarioStatus = STATUSES.FAILED; - this.contextState.countFailedScenarios(event.testCase.sourceLocation.uri); + this.contextState.incrementFailedScenariosCount(event.testCase.sourceLocation.uri); break; } case STATUSES.AMBIGUOUS: { @@ -327,7 +330,7 @@ const createRPFormatterClass = (config) => { }); this.contextState.context.stepStatus = STATUSES.NOT_FOUND; this.contextState.context.scenarioStatus = STATUSES.FAILED; - this.contextState.countFailedScenarios(event.testCase.sourceLocation.uri); + this.contextState.incrementFailedScenariosCount(event.testCase.sourceLocation.uri); break; } case STATUSES.SKIPPED: { @@ -335,11 +338,30 @@ const createRPFormatterClass = (config) => { if (this.contextState.context.scenarioStatus === STATUSES.FAILED) { this.contextState.context.scenarioStatus = STATUSES.SKIPPED; } + + this.contextState.context.stepStatus = STATUSES.SKIPPED; + if ( + this.contextState.context.scenarioStatus === STATUSES.STARTED || + this.contextState.context.scenarioStatus === STATUSES.PASSED + ) { + this.contextState.context.scenarioStatus = STATUSES.SKIPPED; + } else { + this.contextState.context.scenarioStatus = STATUSES.FAILED; + if ( + // eslint-disable-next-line no-prototype-builtins + config.hasOwnProperty('reportSkippedCucumberStepsOnFailedTest') && + !config.reportSkippedCucumberStepsOnFailedTest + ) { + this.contextState.context.stepStatus = STATUSES.CANCELLED; + } + } + break; } case STATUSES.FAILED: { this.contextState.context.stepStatus = STATUSES.FAILED; - this.contextState.countFailedScenarios(event.testCase.sourceLocation.uri); + this.contextState.context.scenarioStatus = STATUSES.FAILED; + this.contextState.incrementFailedScenariosCount(event.testCase.sourceLocation.uri); const errorMessage = `${ this.contextState.context.stepDefinition.uri }\n ${event.result.exception.toString()}`; @@ -501,32 +523,29 @@ const createRPFormatterClass = (config) => { if (!this.isScenarioBasedStatistics && event.result.retried) { return; } - const isFailed = event.result.status.toUpperCase() !== 'PASSED'; + const isFailed = event.result.status.toUpperCase() !== STATUSES.PASSED; // ScenarioResult this.reportportal.finishTestItem(this.contextState.context.scenarioId, { status: isFailed ? STATUSES.FAILED : STATUSES.PASSED, endTime: this.reportportal.helpers.now(), }); - this.contextState.context.scenarioStatus = STATUSES.FAILED; this.contextState.context.scenarioId = null; - const featureUri = event.sourceLocation.uri; + if (!event.result.retried) { - this.contextState.context.scenariosCount[featureUri].done += 1; + this.contextState.context.scenariosCount[featureUri].done++; } + const { total, done } = this.contextState.context.scenariosCount[featureUri]; if (done === total) { const featureStatus = this.contextState.context.failedScenarios[featureUri] > 0 ? STATUSES.FAILED : STATUSES.PASSED; - this.reportportal.finishTestItem( - this.documentsStorage.pickleDocuments[featureUri].featureId, - { - status: featureStatus, - endTime: this.reportportal.helpers.now(), - }, - ); + this.reportportal.finishTestItem(this.documentsStorage.featureData[featureUri].featureId, { + status: featureStatus, + endTime: this.reportportal.helpers.now(), + }); } } diff --git a/modules/documents-storage.js b/modules/documents-storage.js index ae62234..2600bba 100644 --- a/modules/documents-storage.js +++ b/modules/documents-storage.js @@ -17,19 +17,19 @@ class DocumentsStorage { constructor() { this.gherkinDocuments = {}; - this.pickleDocuments = {}; + this.featureData = {}; } cacheDocument(gherkinDocument) { this.gherkinDocuments[gherkinDocument.uri] = gherkinDocument.document; } - cacheAcceptedPickle(event) { - this.pickleDocuments[event.uri] = event.pickle; + createCachedFeature(uri) { + this.featureData[uri] = {}; } - isAcceptedPickleCached(event) { - return !!this.pickleDocuments[event.uri]; + isFeatureDataCached(uri) { + return !!this.featureData[uri]; } } diff --git a/modules/itemFinders.js b/modules/itemFinders.js index defbc8f..e7aa021 100644 --- a/modules/itemFinders.js +++ b/modules/itemFinders.js @@ -20,12 +20,12 @@ function createSteps(header, row, steps) { return steps.map((step) => { const modified = { ...step, parameters: [] }; - header.cells.forEach((varable, index) => { - const isParameterPresents = modified.text.indexOf(`<${varable.value}>`) !== -1; - modified.text = modified.text.replace(`<${varable.value}>`, row.cells[index].value); + header.cells.forEach((variable, index) => { + const isParameterPresents = modified.text.indexOf(`<${variable.value}>`) !== -1; + modified.text = utils.replaceParameter(modified.text, variable.value, row.cells[index].value); if (isParameterPresents) { - modified.parameters.push({ key: varable.value, value: row.cells[index].value }); + modified.parameters.push({ key: variable.value, value: row.cells[index].value }); } }); @@ -33,32 +33,37 @@ function createSteps(header, row, steps) { }); } -function createScenarioFromOutlineExample(outline, example, location) { - const found = example.tableBody.find((row) => row.location.line === location.line); - const parameters = utils.getParameters(example.tableHeader, found); +function createScenarioFromOutlineExample(outline, example, row) { + const parameters = utils.getParameters(example.tableHeader, row); + let outlineName = outline.name; - if (!found) return null; + parameters.forEach((param) => { + outlineName = utils.replaceParameter(outlineName, param.key, param.value); + }); return { type: 'Scenario', - steps: createSteps(example.tableHeader, found, outline.steps), + tags: example.tags, + location: row.location, + keyword: 'Scenario', + name: outlineName, + steps: createSteps(example.tableHeader, row, outline.steps), parameters, - name: outline.name, - location: found.location, description: outline.description, }; } function createScenarioFromOutline(outline, location) { + let foundRow; const foundExample = outline.examples.find((example) => { - const foundRow = example.tableBody.find((row) => row.location.line === location.line); + foundRow = example.tableBody.find((row) => row.location.line === location.line); return !!foundRow; }); - if (!foundExample) return null; + if (!foundRow) return null; - return createScenarioFromOutlineExample(outline, foundExample, location); + return createScenarioFromOutlineExample(outline, foundExample, foundRow); } function findOutlineScenario(outlines, location) { @@ -68,11 +73,7 @@ function findOutlineScenario(outlines, location) { } function findBackground(feature) { - const background = feature.children - ? feature.children.find((child) => child.type === 'Background') - : null; - - return background; + return feature.children ? feature.children.find((child) => child.type === 'Background') : null; } function findFeature(documents, location) { diff --git a/modules/utils.js b/modules/utils.js index 49c5fb0..0a4e8d6 100644 --- a/modules/utils.js +++ b/modules/utils.js @@ -14,7 +14,7 @@ * limitations under the License. */ -const Path = require('path'); +const path = require('path'); const getJSON = (json) => { try { @@ -28,7 +28,7 @@ const getJSON = (json) => { return false; }; -const getUri = (uri) => uri.replace(process.cwd() + Path.sep, ''); +const getUri = (uri) => uri.replace(process.cwd() + path.sep, ''); const cleanContext = () => ({ outlineRow: 0, @@ -48,7 +48,6 @@ const cleanContext = () => ({ stepSourceLocation: null, stepDefinitions: null, stepDefinition: null, - isBeforeHook: true, itemsParams: {}, }); @@ -78,8 +77,8 @@ const createTagComparator = (tagA) => (tagB) => const isScenarioBasedStatistics = (config) => typeof config.scenarioBasedStatistics === 'boolean' ? config.scenarioBasedStatistics : false; -const formatCodeRef = (path, itemName) => { - const codeRef = path.replace(/\\/g, '/'); +const formatCodeRef = (pathName, itemName) => { + const codeRef = pathName.replace(/\\/g, '/'); return itemName ? `${codeRef}/${itemName}` : codeRef; }; @@ -104,6 +103,10 @@ const getParameters = (header, body) => { })); }; +function replaceParameter(originalString, name, value) { + return originalString.replace(`<${name}>`, value); +} + const getStepType = (keyword) => { let type; @@ -132,5 +135,6 @@ module.exports = { getStepType, getParameters, formatCodeRef, + replaceParameter, cleanContext, }; diff --git a/package.json b/package.json index 6d5a737..098201d 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,8 @@ "main": "./modules/index", "spec": "cucumber version >=4.x <=6.x", "dependencies": { - "@reportportal/client-javascript": "^5.0.0" + "@reportportal/client-javascript": "5.0.0", + "cli-table3": "^0.6.0" }, "devDependencies": { "chromedriver": "^83.0.0",