From 5b049be1a8e5e89c7564370b4b823db9a2c2136b Mon Sep 17 00:00:00 2001 From: Stalgia Grigg Date: Mon, 16 Dec 2024 13:17:42 -0800 Subject: [PATCH 01/18] First pass at Failing Assertions table on the Reports page --- .../FailingAssertionsSummary/index.jsx | 60 +++++++++++++++++++ .../Reports/SummarizeTestPlanReport.jsx | 12 +++- client/hooks/useFailingAssertions.js | 25 ++++++++ 3 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 client/components/FailingAssertionsSummary/index.jsx create mode 100644 client/hooks/useFailingAssertions.js diff --git a/client/components/FailingAssertionsSummary/index.jsx b/client/components/FailingAssertionsSummary/index.jsx new file mode 100644 index 000000000..90647ab57 --- /dev/null +++ b/client/components/FailingAssertionsSummary/index.jsx @@ -0,0 +1,60 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import { Table } from 'react-bootstrap'; +import { Link } from 'react-router-dom'; +import { useFailingAssertions } from '../../hooks/useFailingAssertions'; +import { TestPlanReportPropType } from '../common/proptypes'; + +const FailingAssertionsSummary = ({ testPlanReport, atName }) => { + const failingAssertions = useFailingAssertions(testPlanReport); + const { metrics } = testPlanReport; + + if (failingAssertions.length === 0) return null; + + return ( +
+

+ Summary of Failing Assertions ({metrics.mustAssertionsFailedCount} must,{' '} + {metrics.shouldAssertionsFailedCount} should) +

+

+ {metrics.assertionsFailedCount} assertions failed for{' '} + {metrics.commandsCount} commands in {metrics.testsFailedCount} tests. +

+ + + + + + + + + + + + + {failingAssertions.map((assertion, index) => ( + + + + + + + + ))} + +
Test NameCommandAssertion PriorityAssertion{atName} Response
+ + {assertion.testTitle} + + {assertion.scenarioCommands}{assertion.priority}{assertion.assertionText}{assertion.output}
+
+ ); +}; + +FailingAssertionsSummary.propTypes = { + testPlanReport: TestPlanReportPropType.isRequired, + atName: PropTypes.string.isRequired +}; + +export default FailingAssertionsSummary; diff --git a/client/components/Reports/SummarizeTestPlanReport.jsx b/client/components/Reports/SummarizeTestPlanReport.jsx index e71db42ac..dafafbf34 100644 --- a/client/components/Reports/SummarizeTestPlanReport.jsx +++ b/client/components/Reports/SummarizeTestPlanReport.jsx @@ -23,6 +23,7 @@ import { TestPlanReportPropType, TestPlanVersionPropType } from '../common/proptypes'; +import FailingAssertionsSummary from '../FailingAssertionsSummary'; const ResultsContainer = styled.div` padding: 1em 1.75em; @@ -172,6 +173,15 @@ const SummarizeTestPlanReport = ({ testPlanVersion, testPlanReports }) => { ); }; + const renderFailingAssertionsSummary = () => { + return ( + + ); + }; + return ( @@ -246,7 +256,7 @@ const SummarizeTestPlanReport = ({ testPlanVersion, testPlanReports }) => { {renderVersionsSummaryTable()} {renderResultsForTargetTable()} - + {renderFailingAssertionsSummary()} {testPlanReport.finalizedTestResults.map((testResult, index) => { const test = testResult.test; diff --git a/client/hooks/useFailingAssertions.js b/client/hooks/useFailingAssertions.js new file mode 100644 index 000000000..368d059f7 --- /dev/null +++ b/client/hooks/useFailingAssertions.js @@ -0,0 +1,25 @@ +import { useMemo } from 'react'; + +export const useFailingAssertions = testPlanReport => { + return useMemo(() => { + if (!testPlanReport?.finalizedTestResults) { + return []; + } + + return testPlanReport.finalizedTestResults.flatMap(testResult => { + return testResult.scenarioResults.flatMap(scenarioResult => { + return scenarioResult.assertionResults + .filter(assertionResult => !assertionResult.passed) + .map(assertionResult => ({ + testTitle: testResult.test.title, + scenarioCommands: scenarioResult.scenario.commands + .map(cmd => cmd.text) + .join(', '), + assertionText: assertionResult.assertion.text, + priority: assertionResult.assertion.priority, + output: scenarioResult.output + })); + }); + }); + }, [testPlanReport]); +}; From ff46f32a4eaa56075ee6e901aa58f5a0ab497026 Mon Sep 17 00:00:00 2001 From: Stalgia Grigg Date: Mon, 16 Dec 2024 16:19:43 -0800 Subject: [PATCH 02/18] Add summary to TestNavigator --- .../CandidateTestPlanRun/index.jsx | 528 ++++++++++-------- .../CandidateTestPlanRun/queries.js | 1 + client/components/TestRun/TestNavigator.jsx | 19 + client/components/TestRun/TestRun.css | 52 +- client/components/TestRun/index.jsx | 4 +- client/hooks/useUrlTestIndex.js | 18 +- 6 files changed, 371 insertions(+), 251 deletions(-) diff --git a/client/components/CandidateReview/CandidateTestPlanRun/index.jsx b/client/components/CandidateReview/CandidateTestPlanRun/index.jsx index a75525389..fb6adbf86 100644 --- a/client/components/CandidateReview/CandidateTestPlanRun/index.jsx +++ b/client/components/CandidateReview/CandidateTestPlanRun/index.jsx @@ -33,6 +33,7 @@ import createIssueLink, { import RunHistory from '../../common/RunHistory'; import { useUrlTestIndex } from '../../../hooks/useUrlTestIndex'; import NotApprovedModal from '../CandidateModals/NotApprovedModal'; +import FailingAssertionsSummary from '../../FailingAssertionsSummary'; const CandidateTestPlanRun = () => { const { atId, testPlanVersionId } = useParams(); @@ -59,7 +60,10 @@ const CandidateTestPlanRun = () => { const [firstTimeViewing, setFirstTimeViewing] = useState(false); const [viewedTests, setViewedTests] = useState([]); const [testsLength, setTestsLength] = useState(0); - const [currentTestIndex, setCurrentTestIndex] = useUrlTestIndex(testsLength); + const [currentTestIndex, setCurrentTestIndex] = useUrlTestIndex({ + minTestIndex: -1, + maxTestIndex: testsLength + }); const [showTestNavigator, setShowTestNavigator] = useState(true); const [isFirstTest, setIsFirstTest] = useState(true); const [isLastTest, setIsLastTest] = useState(false); @@ -78,7 +82,11 @@ const CandidateTestPlanRun = () => { const handleTestClick = async index => { setCurrentTestIndex(index); - if (index === 0) { + if (index === -1) { + // Summary view + setIsFirstTest(false); + setIsLastTest(false); + } else if (index === 0) { setIsFirstTest(true); setIsLastTest(false); } else if (index === tests.length - 1) { @@ -89,6 +97,7 @@ const CandidateTestPlanRun = () => { setIsLastTest(false); } }; + const handleNextTestClick = async () => { navigateTests( false, @@ -116,6 +125,7 @@ const CandidateTestPlanRun = () => { }; const updateTestViewed = async () => { + if (!currentTest) return; const userPreviouslyViewedTest = viewedTests.includes(currentTest.id); if (!userPreviouslyViewedTest) { setFirstTimeViewing(true); @@ -309,14 +319,14 @@ const CandidateTestPlanRun = () => { issue => issue.isCandidateReview && issue.feedbackType === 'CHANGES_REQUESTED' && - issue.testNumberFilteredByAt === currentTest.seq + issue.testNumberFilteredByAt === currentTest?.seq ); const feedbackIssues = testPlanReport.issues?.filter( issue => issue.isCandidateReview && issue.feedbackType === 'FEEDBACK' && - issue.testNumberFilteredByAt === currentTest.seq + issue.testNumberFilteredByAt === currentTest?.seq ); const issue = { @@ -325,10 +335,10 @@ const CandidateTestPlanRun = () => { testPlanTitle: testPlanVersion.title, testPlanDirectory: testPlanVersion.testPlan.directory, versionString: testPlanVersion.versionString, - testTitle: currentTest.title, - testRowNumber: currentTest.rowNumber, - testSequenceNumber: currentTest.seq, - testRenderedUrl: currentTest.renderedUrl, + testTitle: currentTest?.title, + testRowNumber: currentTest?.rowNumber, + testSequenceNumber: currentTest?.seq, + testRenderedUrl: currentTest?.renderedUrl, atName: testPlanReport.at.name }; @@ -352,7 +362,7 @@ const CandidateTestPlanRun = () => { isCandidateReviewChangesRequested: false, testPlanTitle: testPlanVersion.title, versionString: testPlanVersion.versionString, - testRowNumber: currentTest.rowNumber, + testRowNumber: currentTest?.rowNumber, username: data.me.username, atName: testPlanReport.at.name }; @@ -381,163 +391,180 @@ const CandidateTestPlanRun = () => { 'https://github.com/FreedomScientific/VFO-standards-support/issues'; } - const heading = ( -
-
- Viewing Test {currentTest.title}, Test {currentTest.seq} of{' '} - {tests.length} - {currentTest.seq === tests.length ? 'You are on the last test.' : ''} + const getHeading = () => { + return ( +
+
+ Viewing Test {currentTest.title}, Test {currentTest.seq} of{' '} + {tests.length} + {currentTest.seq === tests.length ? 'You are on the last test.' : ''} +
+ + Reviewing Test {currentTest.seq} of {tests.length}: + +

+ {`${currentTest.seq}. ${currentTest.title}`}{' '} + using {`${at}`}{' '} + {`${testPlanReport?.latestAtVersionReleasedAt?.name ?? ''}`} + {viewedTests.includes(currentTest.id) && !firstTimeViewing && ' '} + {viewedTests.includes(currentTest.id) && !firstTimeViewing && ( + + Previously Viewed + + )} +

- - Reviewing Test {currentTest.seq} of {tests.length}: - -

- {`${currentTest.seq}. ${currentTest.title}`}{' '} - using {`${at}`}{' '} - {`${testPlanReport?.latestAtVersionReleasedAt?.name ?? ''}`} - {viewedTests.includes(currentTest.id) && !firstTimeViewing && ' '} - {viewedTests.includes(currentTest.id) && !firstTimeViewing && ( - - Previously Viewed - - )} -

-
- ); + ); + }; - const testInfo = ( -
-
-
- Candidate Test Plan:{' '} - - {`${ - testPlanVersion.title || testPlanVersion.testPlan?.directory || '' - } ${testPlanVersion.versionString}`} - + const getTestInfo = () => { + return ( + -
-
- Review status by {at} Representative: {`${reviewStatusText} `} +
+
+ Review status by {at} Representative:{' '} + {`${reviewStatusText} `} +
-
-
-
- Target Completion Date: - {targetCompletionDate} +
+
+ Target Completion Date: + {targetCompletionDate} +
-
- ); + ); + }; - const feedback = testPlanReport.issues.filter( - issue => - issue.isCandidateReview && issue.testNumberFilteredByAt == currentTest.seq - ).length > 0 && ( -
-

- Feedback from{' '} - {at} Representative -

-
    - {[changesRequestedIssues, feedbackIssues].map((list, index) => { - if (list.length > 0) { - const uniqueAuthors = [...new Set(list.map(issue => issue.author))]; - const differentAuthors = !( - uniqueAuthors.length === 1 && - uniqueAuthors[0] === data.me.username - ); - return ( - - ); - } - })} -
-
- ); + const getFeedback = () => { + return ( + testPlanReport.issues.filter( + issue => + issue.isCandidateReview && + issue.testNumberFilteredByAt == currentTest.seq + ).length > 0 && ( +
+

+ Feedback from{' '} + {at} Representative +

+
    + {[changesRequestedIssues, feedbackIssues].map((list, index) => { + if (list.length > 0) { + const uniqueAuthors = [ + ...new Set(list.map(issue => issue.author)) + ]; + const differentAuthors = !( + uniqueAuthors.length === 1 && + uniqueAuthors[0] === data.me.username + ); + return ( + + ); + } + })} +
+
+ ) + ); + }; - const results = ( -
-

{currentTest.title}

- `Test Results for ${testPlanReport.browser.name}` - ), - 'Run History' - ]} - onClick={[ - () => setShowInstructions(!showInstructions), - ...showBrowserClicks, - () => setShowRunHistory(!showRunHistory) - ]} - expanded={[showInstructions, ...showBrowserBools, showRunHistory]} - disclosureContainerView={[ - , - ...testPlanReports.map(testPlanReport => { - const testResult = - testPlanReport.finalizedTestResults[currentTestIndex]; - - const { - assertionsPassedCount, - mustAssertionsFailedCount, - shouldAssertionsFailedCount, - mayAssertionsFailedCount - } = getMetrics({ testResult }); - - const mustShouldAssertionsFailedCount = - mustAssertionsFailedCount + shouldAssertionsFailedCount; - - return ( - <> -

- Test Results ( - {assertionsPassedCount} passed,  - {mustShouldAssertionsFailedCount} failed,  - {mayAssertionsFailedCount} unsupported) -

- - - ); - }), - - ]} - stacked - > -
- ); + const getResults = () => { + return ( +
+

{currentTest.title}

+ + `Test Results for ${testPlanReport.browser.name}` + ), + 'Run History' + ]} + onClick={[ + () => setShowInstructions(!showInstructions), + ...showBrowserClicks, + () => setShowRunHistory(!showRunHistory) + ]} + expanded={[showInstructions, ...showBrowserBools, showRunHistory]} + disclosureContainerView={[ + , + ...testPlanReports.map(testPlanReport => { + const testResult = + testPlanReport.finalizedTestResults[currentTestIndex]; + + const { + assertionsPassedCount, + mustAssertionsFailedCount, + shouldAssertionsFailedCount, + mayAssertionsFailedCount + } = getMetrics({ testResult }); + + const mustShouldAssertionsFailedCount = + mustAssertionsFailedCount + shouldAssertionsFailedCount; + + return ( + <> +

+ Test Results ( + {assertionsPassedCount} passed,  + {mustShouldAssertionsFailedCount} failed,  + {mayAssertionsFailedCount} unsupported) +

+ + + ); + }), + + ]} + stacked + > +
+ ); + }; return ( @@ -557,98 +584,109 @@ const CandidateTestPlanRun = () => { /> - {heading} - {testInfo} - - - - {feedback} - {results} - -
    + ) : ( + <> + {getHeading()} + {getTestInfo()} + + + -
  • - -
    • -
  • - -
    • -
  • - +
    • +
  • + +
    • +
  • + +
    • +
+ + + +
+

Test Review Options

+
    - Finish - - -
+
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + + Email us if you need help + +
    • + +
    +     - -
    -

    Test Review Options

    - -
    - -     - + + )} diff --git a/client/components/CandidateReview/CandidateTestPlanRun/queries.js b/client/components/CandidateReview/CandidateTestPlanRun/queries.js index f591896e2..7b833189b 100644 --- a/client/components/CandidateReview/CandidateTestPlanRun/queries.js +++ b/client/components/CandidateReview/CandidateTestPlanRun/queries.js @@ -62,6 +62,7 @@ export const CANDIDATE_REPORTS_QUERY = gql` ) { id vendorReviewStatus + metrics issues { ...IssueFieldsAll } diff --git a/client/components/TestRun/TestNavigator.jsx b/client/components/TestRun/TestNavigator.jsx index 1dc01c9de..b4e40f838 100644 --- a/client/components/TestRun/TestNavigator.jsx +++ b/client/components/TestRun/TestNavigator.jsx @@ -61,6 +61,25 @@ const TestNavigator = ({ aria-labelledby="test-navigator-heading" className="test-navigator-list" > + {isVendor && ( + + )} {tests.map((test, index) => { let resultClassName = isReadOnly ? 'missing' : 'not-started'; let resultStatus = isReadOnly ? 'Missing' : 'Not Started'; diff --git a/client/components/TestRun/TestRun.css b/client/components/TestRun/TestRun.css index 4731f8d00..557836d3d 100644 --- a/client/components/TestRun/TestRun.css +++ b/client/components/TestRun/TestRun.css @@ -54,6 +54,7 @@ button.test-navigator-toggle:focus { margin-top: 1em; font-size: 0.9em; padding-left: 3.25em; + counter-reset: list-counter; } .test-name-wrapper { @@ -62,7 +63,7 @@ button.test-navigator-toggle:focus { list-style: unset; } -.test-name-wrapper:before { +.test-name-wrapper:not(.summary):before { content: ''; height: 100%; left: -2.8em; @@ -290,6 +291,55 @@ button.test-navigator-toggle:focus { left: 9px; } +.test-name-wrapper.summary { + border-bottom: 1px solid #dee2e6; + margin-bottom: 1rem; + padding-bottom: 0.5rem; + list-style: none; /* Hide the number for just this item */ + counter-increment: none; +} + +/* Reset counter after summary */ +.test-name-wrapper.summary + .test-name-wrapper { + counter-reset: list-counter; +} + +/* Resume counting for non-summary items */ +.test-name-wrapper:not(.summary) { + counter-increment: list-counter; +} + +.test-name-wrapper.summary .progress-indicator { + background: #295fa6; +} + +.test-name-wrapper.summary .progress-indicator:before { + display: inline-block; + text-rendering: auto; + -webkit-font-smoothing: antialiased; + font-family: 'Font Awesome 5 Free'; + font-weight: 900; + content: '\f03a'; /* List icon */ + color: white; + font-size: 10px; + position: relative; + top: -3px; + left: 4px; +} + +.test-name-wrapper.summary .test-name { + font-weight: normal; +} + +.test-name-wrapper.summary .test-name[aria-current='true'] { + font-weight: bold; +} + +/* Reset the counter after the summary */ +.test-name-wrapper.summary + li { + counter-reset: list-counter; +} + .test-iframe-container { padding: 0; } diff --git a/client/components/TestRun/index.jsx b/client/components/TestRun/index.jsx index b6ca26d0c..6bce14f08 100644 --- a/client/components/TestRun/index.jsx +++ b/client/components/TestRun/index.jsx @@ -124,7 +124,9 @@ const TestRun = () => { const [testPlanReport, setTestPlanReport] = useState({}); const [testPlanVersion, setTestPlanVersion] = useState(); const [currentTest, setCurrentTest] = useState({}); - const [currentTestIndex, setCurrentTestIndex] = useUrlTestIndex(tests.length); + const [currentTestIndex, setCurrentTestIndex] = useUrlTestIndex({ + maxTestIndex: tests.length + }); const [currentTestAtVersionId, setCurrentTestAtVersionId] = useState(''); const [currentTestBrowserVersionId, setCurrentTestBrowserVersionId] = useState(''); diff --git a/client/hooks/useUrlTestIndex.js b/client/hooks/useUrlTestIndex.js index 68802665e..4d312ad19 100644 --- a/client/hooks/useUrlTestIndex.js +++ b/client/hooks/useUrlTestIndex.js @@ -1,18 +1,22 @@ import { useLocation, useNavigate } from 'react-router-dom'; import { useEffect, useState } from 'react'; -export const useUrlTestIndex = maxTestIndex => { +export const useUrlTestIndex = ({ minTestIndex = 0, maxTestIndex }) => { const location = useLocation(); const navigate = useNavigate(); - const [currentTestIndex, setCurrentTestIndex] = useState(0); + const [currentTestIndex, setCurrentTestIndex] = useState(minTestIndex); const getTestIndex = () => { + if (location.hash === '#summary') { + return -1; + } + // Remove the '#' character const fragment = parseInt(location.hash.slice(1), 10) || 1; if (!maxTestIndex || maxTestIndex < 1) { - // If the max test index is less than 1, return 0 - return 0; + // If the max test index is less than 1, return the min test index + return minTestIndex; } // Subtract 1 to make it 0-indexed // and clamp to the max test index @@ -24,6 +28,12 @@ export const useUrlTestIndex = maxTestIndex => { }, [location.hash, maxTestIndex]); const updateTestIndex = index => { + // Special case for summary + if (index === -1) { + navigate(`${location.pathname}#summary`, { replace: true }); + return; + } + // Add 1 to make it 1-indexed const newIndex = index + 1; navigate(`${location.pathname}#${newIndex}`, { replace: true }); From 8b9709a57394a4b55ba686a18d8d9dc4d6ff0a6a Mon Sep 17 00:00:00 2001 From: Stalgia Grigg Date: Mon, 16 Dec 2024 17:48:29 -0800 Subject: [PATCH 03/18] Navigation for summary view in Candidate Review --- .../CandidateTestPlanRun/index.jsx | 353 ++++++++++-------- client/utils/navigateTests.js | 23 ++ 2 files changed, 212 insertions(+), 164 deletions(-) diff --git a/client/components/CandidateReview/CandidateTestPlanRun/index.jsx b/client/components/CandidateReview/CandidateTestPlanRun/index.jsx index fb6adbf86..01aa05091 100644 --- a/client/components/CandidateReview/CandidateTestPlanRun/index.jsx +++ b/client/components/CandidateReview/CandidateTestPlanRun/index.jsx @@ -74,6 +74,8 @@ const CandidateTestPlanRun = () => { const [showRunHistory, setShowRunHistory] = useState(false); const [showBrowserClicks, setShowBrowserClicks] = useState([]); + const isSummaryView = currentTestIndex === -1; + const isLaptopOrLarger = useMediaQuery({ query: '(min-width: 792px)' }); @@ -392,6 +394,13 @@ const CandidateTestPlanRun = () => { } const getHeading = () => { + if (isSummaryView) { + return ( +
    +

    Summary of Failing Assertions

    +
    + ); + } return (
    @@ -449,6 +458,9 @@ const CandidateTestPlanRun = () => { }; const getFeedback = () => { + if (isSummaryView) { + return null; + } return ( testPlanReport.issues.filter( issue => @@ -495,73 +507,82 @@ const CandidateTestPlanRun = () => { ); }; - const getResults = () => { + const getContent = () => { return (
    -

    {currentTest.title}

    - - `Test Results for ${testPlanReport.browser.name}` - ), - 'Run History' - ]} - onClick={[ - () => setShowInstructions(!showInstructions), - ...showBrowserClicks, - () => setShowRunHistory(!showRunHistory) - ]} - expanded={[showInstructions, ...showBrowserBools, showRunHistory]} - disclosureContainerView={[ - , - ...testPlanReports.map(testPlanReport => { - const testResult = - testPlanReport.finalizedTestResults[currentTestIndex]; - - const { - assertionsPassedCount, - mustAssertionsFailedCount, - shouldAssertionsFailedCount, - mayAssertionsFailedCount - } = getMetrics({ testResult }); - - const mustShouldAssertionsFailedCount = - mustAssertionsFailedCount + shouldAssertionsFailedCount; - - return ( - <> -

    - Test Results ( - {assertionsPassedCount} passed,  - {mustShouldAssertionsFailedCount} failed,  - {mayAssertionsFailedCount} unsupported) -

    - - - ); - }), - - ]} - stacked - >     + {isSummaryView ? ( + + ) : ( + <> +

    {currentTest.title}

    + + `Test Results for ${testPlanReport.browser.name}` + ), + 'Run History' + ]} + onClick={[ + () => setShowInstructions(!showInstructions), + ...showBrowserClicks, + () => setShowRunHistory(!showRunHistory) + ]} + expanded={[showInstructions, ...showBrowserBools, showRunHistory]} + disclosureContainerView={[ + , + ...testPlanReports.map(testPlanReport => { + const testResult = + testPlanReport.finalizedTestResults[currentTestIndex]; + + const { + assertionsPassedCount, + mustAssertionsFailedCount, + shouldAssertionsFailedCount, + mayAssertionsFailedCount + } = getMetrics({ testResult }); + + const mustShouldAssertionsFailedCount = + mustAssertionsFailedCount + shouldAssertionsFailedCount; + + return ( + <> +

    + Test Results ( + {assertionsPassedCount} passed,  + {mustShouldAssertionsFailedCount} failed,  + {mayAssertionsFailedCount} unsupported) +

    + + + ); + }), + + ]} + stacked + >     + + )}
    ); }; @@ -584,109 +605,113 @@ const CandidateTestPlanRun = () => { /> - {currentTestIndex === -1 ? ( - - ) : ( - <> - {getHeading()} - {getTestInfo()} - - - - {getFeedback()} - - {getResults()} - - -
      -
    • - -
      • -
    • - -
      • -
    • - -
      • -
    -     - - + + + {getFeedback()} + {getContent()} + +
      -
      -

      Test Review Options

      -
        + + + )} + {isSummaryView ? ( +
      • + +
        • + ) : ( +
      • + +
        • + )} +
      • +
      -
      - + Finish + + +
    - - )} + +
    +

    Test Review Options

    + +
    + +     +     diff --git a/client/utils/navigateTests.js b/client/utils/navigateTests.js index 7c8346f16..aaa2dfab8 100644 --- a/client/utils/navigateTests.js +++ b/client/utils/navigateTests.js @@ -6,6 +6,22 @@ export const navigateTests = ( setIsFirstTest = () => {}, setIsLastTest = () => {} ) => { + // If currentTest is undefined, we are on the summary view + if (!currentTest) { + if (!previous) { + const firstTest = tests.find(t => t.seq === 1); + setCurrentTestIndex(firstTest.index); + setIsFirstTest(true); + setIsLastTest(false); + return { + currentIndex: firstTest.index, + isFirstTest: true, + isLastTest: false + }; + } + // No previous from summary + return { currentIndex: -1, isFirstTest: false, isLastTest: false }; + } // assume navigation forward if previous is false let newTestIndex = currentTest.seq; if (!previous) { @@ -15,6 +31,13 @@ export const navigateTests = ( } else { // previous const newTestIndexToEval = currentTest.seq - 1; + // Go to summary when going previous from first test + if (newTestIndexToEval === 0) { + setCurrentTestIndex(-1); + setIsFirstTest(false); + setIsLastTest(false); + return { currentIndex: -1, isFirstTest: false, isLastTest: false }; + } if (newTestIndexToEval >= 1 && newTestIndexToEval <= tests.length) newTestIndex = newTestIndexToEval; } From c14f4ed886e9c6482f07bd9da6ceb7a2cea1098f Mon Sep 17 00:00:00 2001 From: Stalgia Grigg Date: Mon, 16 Dec 2024 18:11:07 -0800 Subject: [PATCH 04/18] Decouple FailingAssertionsSummaryTable and Heading --- .../CandidateTestPlanRun/index.jsx | 64 +++++++++++-------- .../FailingAssertionsSummary/Heading.jsx | 19 ++++++ .../{index.jsx => Table.jsx} | 14 ++-- .../Reports/SummarizeTestPlanReport.jsx | 4 +- client/components/common/proptypes/index.js | 35 +++++++++- 5 files changed, 96 insertions(+), 40 deletions(-) create mode 100644 client/components/FailingAssertionsSummary/Heading.jsx rename client/components/FailingAssertionsSummary/{index.jsx => Table.jsx} (80%) diff --git a/client/components/CandidateReview/CandidateTestPlanRun/index.jsx b/client/components/CandidateReview/CandidateTestPlanRun/index.jsx index 01aa05091..3b60bf19f 100644 --- a/client/components/CandidateReview/CandidateTestPlanRun/index.jsx +++ b/client/components/CandidateReview/CandidateTestPlanRun/index.jsx @@ -33,7 +33,8 @@ import createIssueLink, { import RunHistory from '../../common/RunHistory'; import { useUrlTestIndex } from '../../../hooks/useUrlTestIndex'; import NotApprovedModal from '../CandidateModals/NotApprovedModal'; -import FailingAssertionsSummary from '../../FailingAssertionsSummary'; +import FailingAssertionsSummaryTable from '../../FailingAssertionsSummary/Table'; +import FailingAssertionsSummaryHeading from '../../FailingAssertionsSummary/Heading'; const CandidateTestPlanRun = () => { const { atId, testPlanVersionId } = useParams(); @@ -394,34 +395,41 @@ const CandidateTestPlanRun = () => { } const getHeading = () => { - if (isSummaryView) { - return ( -
    -

    Summary of Failing Assertions

    -
    - ); - } return (
    -
    - Viewing Test {currentTest.title}, Test {currentTest.seq} of{' '} - {tests.length} - {currentTest.seq === tests.length ? 'You are on the last test.' : ''} -
    - - Reviewing Test {currentTest.seq} of {tests.length}: - -

    - {`${currentTest.seq}. ${currentTest.title}`}{' '} - using {`${at}`}{' '} - {`${testPlanReport?.latestAtVersionReleasedAt?.name ?? ''}`} - {viewedTests.includes(currentTest.id) && !firstTimeViewing && ' '} - {viewedTests.includes(currentTest.id) && !firstTimeViewing && ( - - Previously Viewed - - )} -

    + {isSummaryView ? ( + <> + Candidate Test Plan Review + + + ) : ( + <> +
    + Viewing Test {currentTest.title}, Test {currentTest.seq} of{' '} + {tests.length} + {currentTest.seq === tests.length + ? 'You are on the last test.' + : ''} +
    + + Reviewing Test {currentTest.seq} of {tests.length}: + +

    + {`${currentTest.seq}. ${currentTest.title}`}{' '} + using {`${at}`}{' '} + {`${testPlanReport?.latestAtVersionReleasedAt?.name ?? ''}`} + {viewedTests.includes(currentTest.id) && !firstTimeViewing && ' '} + {viewedTests.includes(currentTest.id) && !firstTimeViewing && ( + + Previously Viewed + + )} +

    + + )}
    ); }; @@ -511,7 +519,7 @@ const CandidateTestPlanRun = () => { return (
    {isSummaryView ? ( - diff --git a/client/components/FailingAssertionsSummary/Heading.jsx b/client/components/FailingAssertionsSummary/Heading.jsx new file mode 100644 index 000000000..0bcba1ab2 --- /dev/null +++ b/client/components/FailingAssertionsSummary/Heading.jsx @@ -0,0 +1,19 @@ +import React from 'react'; +import { TestPlanReportMetricsPropType } from '../common/proptypes'; +import PropTypes from 'prop-types'; + +const FailingAssertionsSummaryHeading = ({ metrics, as: Element = 'h2' }) => { + return ( + + Summary of Failing Assertions ({metrics.mustAssertionsFailedCount} must,{' '} + {metrics.shouldAssertionsFailedCount} should) + + ); +}; + +FailingAssertionsSummaryHeading.propTypes = { + metrics: TestPlanReportMetricsPropType.isRequired, + as: PropTypes.elementType +}; + +export default FailingAssertionsSummaryHeading; diff --git a/client/components/FailingAssertionsSummary/index.jsx b/client/components/FailingAssertionsSummary/Table.jsx similarity index 80% rename from client/components/FailingAssertionsSummary/index.jsx rename to client/components/FailingAssertionsSummary/Table.jsx index 90647ab57..fc6f83930 100644 --- a/client/components/FailingAssertionsSummary/index.jsx +++ b/client/components/FailingAssertionsSummary/Table.jsx @@ -5,18 +5,14 @@ import { Link } from 'react-router-dom'; import { useFailingAssertions } from '../../hooks/useFailingAssertions'; import { TestPlanReportPropType } from '../common/proptypes'; -const FailingAssertionsSummary = ({ testPlanReport, atName }) => { +const FailingAssertionsSummaryTable = ({ testPlanReport, atName }) => { const failingAssertions = useFailingAssertions(testPlanReport); const { metrics } = testPlanReport; if (failingAssertions.length === 0) return null; return ( -
    -

    - Summary of Failing Assertions ({metrics.mustAssertionsFailedCount} must,{' '} - {metrics.shouldAssertionsFailedCount} should) -

    + <>

    {metrics.assertionsFailedCount} assertions failed for{' '} {metrics.commandsCount} commands in {metrics.testsFailedCount} tests. @@ -48,13 +44,13 @@ const FailingAssertionsSummary = ({ testPlanReport, atName }) => { ))} -

    + ); }; -FailingAssertionsSummary.propTypes = { +FailingAssertionsSummaryTable.propTypes = { testPlanReport: TestPlanReportPropType.isRequired, atName: PropTypes.string.isRequired }; -export default FailingAssertionsSummary; +export default FailingAssertionsSummaryTable; diff --git a/client/components/Reports/SummarizeTestPlanReport.jsx b/client/components/Reports/SummarizeTestPlanReport.jsx index dafafbf34..f64023222 100644 --- a/client/components/Reports/SummarizeTestPlanReport.jsx +++ b/client/components/Reports/SummarizeTestPlanReport.jsx @@ -23,7 +23,7 @@ import { TestPlanReportPropType, TestPlanVersionPropType } from '../common/proptypes'; -import FailingAssertionsSummary from '../FailingAssertionsSummary'; +import FailingAssertionsSummaryTable from '../FailingAssertionsSummary/Table'; const ResultsContainer = styled.div` padding: 1em 1.75em; @@ -175,7 +175,7 @@ const SummarizeTestPlanReport = ({ testPlanVersion, testPlanReports }) => { const renderFailingAssertionsSummary = () => { return ( - diff --git a/client/components/common/proptypes/index.js b/client/components/common/proptypes/index.js index 27d0fad2c..fcf197471 100644 --- a/client/components/common/proptypes/index.js +++ b/client/components/common/proptypes/index.js @@ -208,6 +208,39 @@ export const TestPlanReportConflictPropType = PropTypes.shape({ ).isRequired }); +export const TestPlanReportMetricsPropType = PropTypes.shape({ + __typename: PropTypes.string, + assertionsPassedCount: PropTypes.number, + assertionsFailedCount: PropTypes.number, + mustAssertionsPassedCount: PropTypes.number, + mustAssertionsCount: PropTypes.number, + mustAssertionsFailedCount: PropTypes.number, + shouldAssertionsPassedCount: PropTypes.number, + shouldAssertionsCount: PropTypes.number, + shouldAssertionsFailedCount: PropTypes.number, + mayAssertionsPassedCount: PropTypes.number, + mayAssertionsCount: PropTypes.number, + mayAssertionsFailedCount: PropTypes.number, + testsPassedCount: PropTypes.number, + testsCount: PropTypes.number, + testsFailedCount: PropTypes.number, + unexpectedBehaviorCount: PropTypes.number, + severeImpactPassedAssertionCount: PropTypes.number, + severeImpactFailedAssertionCount: PropTypes.number, + moderateImpactPassedAssertionCount: PropTypes.number, + moderateImpactFailedAssertionCount: PropTypes.number, + commandsCount: PropTypes.number, + mustFormatted: PropTypes.oneOfType([PropTypes.bool, PropTypes.string]), + shouldFormatted: PropTypes.oneOfType([PropTypes.bool, PropTypes.string]), + mayFormatted: PropTypes.oneOfType([PropTypes.bool, PropTypes.string]), + unexpectedBehaviorsFormatted: PropTypes.oneOfType([ + PropTypes.bool, + PropTypes.string + ]), + supportLevel: PropTypes.string, + supportPercent: PropTypes.number +}); + export const TestPlanReportStatusPropType = PropTypes.shape({ __typename: PropTypes.string, isRequired: PropTypes.bool, @@ -217,7 +250,7 @@ export const TestPlanReportStatusPropType = PropTypes.shape({ exactAtVersion: AtVersionPropType, testPlanReport: PropTypes.shape({ id: PropTypes.string.isRequired, - metrics: PropTypes.object, + metrics: TestPlanReportMetricsPropType, isFinal: PropTypes.bool, markedFinalAt: PropTypes.string, issues: PropTypes.arrayOf(IssuePropType), From 1d728b718fa4399b9b3db1d42cc05c9baaaa1e19 Mon Sep 17 00:00:00 2001 From: Stalgia Grigg Date: Tue, 17 Dec 2024 09:10:11 -0800 Subject: [PATCH 05/18] Correct links in FailingAssertionsSummaryTable --- .../CandidateTestPlanRun/index.jsx | 1 + .../FailingAssertionsSummary/Table.jsx | 15 +++++--- .../Reports/SummarizeTestPlanReport.jsx | 15 +++++--- client/hooks/useFailingAssertions.js | 34 +++++++++++-------- 4 files changed, 42 insertions(+), 23 deletions(-) diff --git a/client/components/CandidateReview/CandidateTestPlanRun/index.jsx b/client/components/CandidateReview/CandidateTestPlanRun/index.jsx index 3b60bf19f..a07b2010f 100644 --- a/client/components/CandidateReview/CandidateTestPlanRun/index.jsx +++ b/client/components/CandidateReview/CandidateTestPlanRun/index.jsx @@ -522,6 +522,7 @@ const CandidateTestPlanRun = () => { `#${assertion.testIndex + 1}`} /> ) : ( <> diff --git a/client/components/FailingAssertionsSummary/Table.jsx b/client/components/FailingAssertionsSummary/Table.jsx index fc6f83930..c73395b06 100644 --- a/client/components/FailingAssertionsSummary/Table.jsx +++ b/client/components/FailingAssertionsSummary/Table.jsx @@ -5,7 +5,12 @@ import { Link } from 'react-router-dom'; import { useFailingAssertions } from '../../hooks/useFailingAssertions'; import { TestPlanReportPropType } from '../common/proptypes'; -const FailingAssertionsSummaryTable = ({ testPlanReport, atName }) => { +const FailingAssertionsSummaryTable = ({ + testPlanReport, + atName, + getLinkUrl = assertion => `#result-${assertion.testId}`, + LinkComponent = Link +}) => { const failingAssertions = useFailingAssertions(testPlanReport); const { metrics } = testPlanReport; @@ -32,9 +37,9 @@ const FailingAssertionsSummaryTable = ({ testPlanReport, atName }) => { {failingAssertions.map((assertion, index) => ( - + {assertion.testTitle} - + {assertion.scenarioCommands} {assertion.priority} @@ -50,7 +55,9 @@ const FailingAssertionsSummaryTable = ({ testPlanReport, atName }) => { FailingAssertionsSummaryTable.propTypes = { testPlanReport: TestPlanReportPropType.isRequired, - atName: PropTypes.string.isRequired + atName: PropTypes.string.isRequired, + getLinkUrl: PropTypes.func, + LinkComponent: PropTypes.elementType }; export default FailingAssertionsSummaryTable; diff --git a/client/components/Reports/SummarizeTestPlanReport.jsx b/client/components/Reports/SummarizeTestPlanReport.jsx index f64023222..ea5695549 100644 --- a/client/components/Reports/SummarizeTestPlanReport.jsx +++ b/client/components/Reports/SummarizeTestPlanReport.jsx @@ -24,6 +24,7 @@ import { TestPlanVersionPropType } from '../common/proptypes'; import FailingAssertionsSummaryTable from '../FailingAssertionsSummary/Table'; +import FailingAssertionsSummaryHeading from '../FailingAssertionsSummary/Heading'; const ResultsContainer = styled.div` padding: 1em 1.75em; @@ -175,10 +176,16 @@ const SummarizeTestPlanReport = ({ testPlanVersion, testPlanReports }) => { const renderFailingAssertionsSummary = () => { return ( - + <> + + + `/report/${testPlanVersion.id}/targets/${testPlanReport.id}#result-${assertion.testResultId}` + } + /> + ); }; diff --git a/client/hooks/useFailingAssertions.js b/client/hooks/useFailingAssertions.js index 368d059f7..a6b1bf6f6 100644 --- a/client/hooks/useFailingAssertions.js +++ b/client/hooks/useFailingAssertions.js @@ -6,20 +6,24 @@ export const useFailingAssertions = testPlanReport => { return []; } - return testPlanReport.finalizedTestResults.flatMap(testResult => { - return testResult.scenarioResults.flatMap(scenarioResult => { - return scenarioResult.assertionResults - .filter(assertionResult => !assertionResult.passed) - .map(assertionResult => ({ - testTitle: testResult.test.title, - scenarioCommands: scenarioResult.scenario.commands - .map(cmd => cmd.text) - .join(', '), - assertionText: assertionResult.assertion.text, - priority: assertionResult.assertion.priority, - output: scenarioResult.output - })); - }); - }); + return testPlanReport.finalizedTestResults.flatMap( + (testResult, testIndex) => { + return testResult.scenarioResults.flatMap(scenarioResult => { + return scenarioResult.assertionResults + .filter(assertionResult => !assertionResult.passed) + .map(assertionResult => ({ + testResultId: testResult.id, + testIndex, + testTitle: testResult.test.title, + scenarioCommands: scenarioResult.scenario.commands + .map(cmd => cmd.text) + .join(', '), + assertionText: assertionResult.assertion.text, + priority: assertionResult.assertion.priority, + output: scenarioResult.output + })); + }); + } + ); }, [testPlanReport]); }; From b94aae28fc44dc13da8b796c75c3bf16b8f24ef4 Mon Sep 17 00:00:00 2001 From: Stalgia Grigg Date: Tue, 17 Dec 2024 16:49:21 -0800 Subject: [PATCH 06/18] Update snapshot --- .../CandidateTestPlanRun/index.jsx | 4 ++-- .../saved/_candidate-test-plan_24_1.html | 18 ++++++++++------- .../e2e/snapshots/saved/_data-management.html | 20 +++++++++---------- .../saved/_report_67_targets_20.html | 3 +++ 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/client/components/CandidateReview/CandidateTestPlanRun/index.jsx b/client/components/CandidateReview/CandidateTestPlanRun/index.jsx index a07b2010f..5f8fd5c5e 100644 --- a/client/components/CandidateReview/CandidateTestPlanRun/index.jsx +++ b/client/components/CandidateReview/CandidateTestPlanRun/index.jsx @@ -644,7 +644,7 @@ const CandidateTestPlanRun = () => {
    • -
  • +
  • diff --git a/client/tests/e2e/snapshots/saved/_data-management.html b/client/tests/e2e/snapshots/saved/_data-management.html index fc0f5d489..0ac106f45 100644 --- a/client/tests/e2e/snapshots/saved/_data-management.html +++ b/client/tests/e2e/snapshots/saved/_data-management.html @@ -259,7 +259,7 @@

    -

    • @@ -1183,7 +1183,7 @@

    Test Plans Status Summary

    R&D -

    Complete Nov 20, 2024

    +

    Complete Dec 17, 2024

    @@ -2067,7 +2067,7 @@

    Test Plans Status Summary

    V24.11.20    V24.12.17
    +

    + Summary of Failing Assertions (0 must, 1 should) +

    Test 1: Navigate forwards to a mixed checkbox in reading From c9b1fc6e4f7808b97cb2ce47fdd98e7638cb5afa Mon Sep 17 00:00:00 2001 From: Stalgia Grigg Date: Tue, 17 Dec 2024 17:38:01 -0800 Subject: [PATCH 07/18] styles for tablet view --- .../CandidateTestPlanRun/index.jsx | 12 +++++++----- .../FailingAssertionsSummary.css | 17 +++++++++++++++++ .../FailingAssertionsSummary/Table.jsx | 1 + 3 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 client/components/FailingAssertionsSummary/FailingAssertionsSummary.css diff --git a/client/components/CandidateReview/CandidateTestPlanRun/index.jsx b/client/components/CandidateReview/CandidateTestPlanRun/index.jsx index 5f8fd5c5e..c0652a38d 100644 --- a/client/components/CandidateReview/CandidateTestPlanRun/index.jsx +++ b/client/components/CandidateReview/CandidateTestPlanRun/index.jsx @@ -519,11 +519,13 @@ const CandidateTestPlanRun = () => { return (
    {isSummaryView ? ( - `#${assertion.testIndex + 1}`} - /> +
    + `#${assertion.testIndex + 1}`} + /> +
    ) : ( <>

    {currentTest.title}

    diff --git a/client/components/FailingAssertionsSummary/FailingAssertionsSummary.css b/client/components/FailingAssertionsSummary/FailingAssertionsSummary.css new file mode 100644 index 000000000..fa8c72e20 --- /dev/null +++ b/client/components/FailingAssertionsSummary/FailingAssertionsSummary.css @@ -0,0 +1,17 @@ +.failing-assertions-summary-table-container { + width: 100%; + max-width: 100%; + padding: 0; +} + +/* Add these rules */ +.failing-assertions-summary-table-container table { + table-layout: fixed; + width: 100%; +} + +.failing-assertions-summary-table-container th, +.failing-assertions-summary-table-container td { + word-wrap: break-word; + overflow-wrap: break-word; +} diff --git a/client/components/FailingAssertionsSummary/Table.jsx b/client/components/FailingAssertionsSummary/Table.jsx index c73395b06..f81f1b928 100644 --- a/client/components/FailingAssertionsSummary/Table.jsx +++ b/client/components/FailingAssertionsSummary/Table.jsx @@ -4,6 +4,7 @@ import { Table } from 'react-bootstrap'; import { Link } from 'react-router-dom'; import { useFailingAssertions } from '../../hooks/useFailingAssertions'; import { TestPlanReportPropType } from '../common/proptypes'; +import './FailingAssertionsSummary.css'; const FailingAssertionsSummaryTable = ({ testPlanReport, From a42c7d77e6514138a8233ef22fb1b0b6a7189aec Mon Sep 17 00:00:00 2001 From: Stalgia Grigg Date: Tue, 17 Dec 2024 17:50:54 -0800 Subject: [PATCH 08/18] Add e2e tests for summary view in CandidateTestPlanRun --- client/hooks/useFailingAssertions.js | 32 ++++--- client/tests/e2e/CandidateReview.e2e.test.js | 97 ++++++++++++++++++++ 2 files changed, 116 insertions(+), 13 deletions(-) diff --git a/client/hooks/useFailingAssertions.js b/client/hooks/useFailingAssertions.js index a6b1bf6f6..0e064b4b5 100644 --- a/client/hooks/useFailingAssertions.js +++ b/client/hooks/useFailingAssertions.js @@ -9,19 +9,25 @@ export const useFailingAssertions = testPlanReport => { return testPlanReport.finalizedTestResults.flatMap( (testResult, testIndex) => { return testResult.scenarioResults.flatMap(scenarioResult => { - return scenarioResult.assertionResults - .filter(assertionResult => !assertionResult.passed) - .map(assertionResult => ({ - testResultId: testResult.id, - testIndex, - testTitle: testResult.test.title, - scenarioCommands: scenarioResult.scenario.commands - .map(cmd => cmd.text) - .join(', '), - assertionText: assertionResult.assertion.text, - priority: assertionResult.assertion.priority, - output: scenarioResult.output - })); + return ( + scenarioResult.assertionResults + .filter(assertionResult => !assertionResult.passed) + // We only want to show MUST and SHOULD assertions + .filter( + assertionResult => assertionResult.assertion.priority !== 'MAY' + ) + .map(assertionResult => ({ + testResultId: testResult.id, + testIndex, + testTitle: testResult.test.title, + scenarioCommands: scenarioResult.scenario.commands + .map(cmd => cmd.text) + .join(', '), + assertionText: assertionResult.assertion.text, + priority: assertionResult.assertion.priority, + output: scenarioResult.output + })) + ); }); } ); diff --git a/client/tests/e2e/CandidateReview.e2e.test.js b/client/tests/e2e/CandidateReview.e2e.test.js index 98c7811c1..3784b555d 100644 --- a/client/tests/e2e/CandidateReview.e2e.test.js +++ b/client/tests/e2e/CandidateReview.e2e.test.js @@ -110,4 +110,101 @@ describe('Candidate Review when signed in as vendor', () => { expect(afterClickTestResultDisclosureDisplay).not.toBe('none'); }); }); + + it('shows summary view as first page and navigates correctly', async () => { + await getPage( + { role: 'vendor', url: '/candidate-review' }, + async (page, { consoleErrors }) => { + await page.waitForSelector('h1 ::-p-text(Candidate Review)'); + + // Select first possible link from the table in the Candidate Test Plans + // column + await page.click('table[aria-label] a'); + + await page.waitForSelector('nav#test-navigator-nav ol'); + await page.waitForSelector('h1 ::-p-text(1.)'); + await page.waitForSelector('h1[class="current-test-title"]'); + + await page.click('a[href="#summary"]'); + + // Check navigation buttons in summary view + const previousButton = await page.$('button::-p-text(Previous Test)'); + expect(previousButton).toBeNull(); // No Previous button on summary + + const nextButton = await page.waitForSelector( + 'button::-p-text(Begin Review)' + ); + await nextButton.click(); + + // Should navigate to first test + await page.waitForSelector('h1::-p-text(1.)'); + // Check summary link aria-current attribute + const summaryLinkAriaCurrent = await page.$eval( + 'a[href="#summary"]', + el => el.getAttribute('aria-current') + ); + expect(summaryLinkAriaCurrent).toBe('false'); + + // Previous button should go back to summary from first test + const previousFromFirstTest = await page.waitForSelector( + 'button::-p-text(Summary)' + ); + await previousFromFirstTest.click(); + + // Should be back on summary + await page.waitForSelector('a[href="#summary"][aria-current="true"]'); + + expect(consoleErrors).toHaveLength(0); + } + ); + }); + + it('shows failing assertions summary content', async () => { + await getPage( + { role: 'vendor', url: '/candidate-review' }, + async (page, { consoleErrors }) => { + await page.waitForSelector('h1 ::-p-text(Candidate Review)'); + + // Select first possible link from the table in the Candidate Test Plans + // column + await page.click('table[aria-label] a'); + + await page.waitForSelector('nav#test-navigator-nav ol'); + await page.waitForSelector('h1 ::-p-text(1.)'); + await page.waitForSelector('h1[class="current-test-title"]'); + + // Check for summary specific content + await page.click('a[href="#summary"]'); + await page.waitForSelector( + 'h1::-p-text(Summary of Failing Assertions)' + ); + + // Check for table + await page.waitForSelector( + 'table[aria-labelledby="failing-assertions-heading"]' + ); + + // Get the text from the first link in the table + const firstLinkText = await page.$eval( + 'table[aria-labelledby="failing-assertions-heading"] a', + el => el.textContent + ); + + // Click on the first link in the table + await page.click( + 'table[aria-labelledby="failing-assertions-heading"] a' + ); + + // Ensure we are on the correct test + await page.waitForSelector('nav#test-navigator-nav ol'); + + // Check if the text from the first link in the table is in the h1 + // So that we know we are on the correct test + const h1Text = await text(page, 'h1'); + expect(h1Text).toContain(firstLinkText); + + expect(consoleErrors).toHaveLength(0); + } + ); + }); }); From b64b3e0cae744b1bcea28e0f95948b565b5bc3f5 Mon Sep 17 00:00:00 2001 From: Stalgia Grigg Date: Tue, 17 Dec 2024 17:55:40 -0800 Subject: [PATCH 09/18] Add candidate test plan review summary to snapshots --- .../Reports/SummarizeTestPlanReport.jsx | 7 + .../_candidate-test-plan_24_1#summary.html | 423 ++++++++++++++++++ client/tests/e2e/snapshots/utils.js | 1 + 3 files changed, 431 insertions(+) create mode 100644 client/tests/e2e/snapshots/saved/_candidate-test-plan_24_1#summary.html diff --git a/client/components/Reports/SummarizeTestPlanReport.jsx b/client/components/Reports/SummarizeTestPlanReport.jsx index ea5695549..463a99390 100644 --- a/client/components/Reports/SummarizeTestPlanReport.jsx +++ b/client/components/Reports/SummarizeTestPlanReport.jsx @@ -175,6 +175,13 @@ const SummarizeTestPlanReport = ({ testPlanVersion, testPlanReports }) => { }; const renderFailingAssertionsSummary = () => { + if ( + testPlanReport.metrics.mustAssertionsFailedCount === 0 && + testPlanReport.metrics.shouldAssertionsFailedCount === 0 + ) { + return null; + } + return ( <> diff --git a/client/tests/e2e/snapshots/saved/_candidate-test-plan_24_1#summary.html b/client/tests/e2e/snapshots/saved/_candidate-test-plan_24_1#summary.html new file mode 100644 index 000000000..ac7568985 --- /dev/null +++ b/client/tests/e2e/snapshots/saved/_candidate-test-plan_24_1#summary.html @@ -0,0 +1,423 @@ + + + + + + + + + Candidate Test Run Page | ARIA-AT + + +
    +
    + +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    +
    +
    + Candidate Test Plan Review +

    + Summary of Failing Assertions (2 must, 1 should) +

    +
    +
    +
    +
    + Candidate Test Plan: + Modal Dialog Example V22.05.26 +
    +
    +
    +
    + Review status by JAWS Representative: In Progress +
    +
    +
    +
    + Target Completion Date: January 2, 2023 +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +

    + 3 assertions failed for 26 commands in 2 tests. +

    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Test NameCommandAssertion PriorityAssertionJAWS Response
    + Close a modal dialog in reading mode + EscapeMUSTRole 'button' is conveyedautomatically seeded sample output
    + Close a modal dialog in interaction + mode + EscapeMUSTRole 'button' is conveyedautomatically seeded sample output
    +
    +
    +
    +
    +
    +
      +
    • + +
      • +
    • + +
      • +
    +
    +
    +
    +
    +

    Test Review Options

    + +
    +
    +
    +
    +
    +
    +
    +
    +
    + +
    + + + diff --git a/client/tests/e2e/snapshots/utils.js b/client/tests/e2e/snapshots/utils.js index 749a0963b..0c4aad1c7 100644 --- a/client/tests/e2e/snapshots/utils.js +++ b/client/tests/e2e/snapshots/utils.js @@ -14,6 +14,7 @@ const snapshotRoutes = [ '/run/2', '/data-management/meter', '/candidate-test-plan/24/1', + '/candidate-test-plan/24/1#summary', '/test-queue/2/conflicts', '/report/67/targets/20' ]; From c9e92ebb39653e3bf811bf04329838a13c2d2b98 Mon Sep 17 00:00:00 2001 From: Stalgia Grigg Date: Tue, 17 Dec 2024 18:04:06 -0800 Subject: [PATCH 10/18] Cleanup --- .../CandidateTestPlanRun/CandidateTestPlanRun.css | 5 +++++ .../CandidateReview/CandidateTestPlanRun/index.jsx | 2 +- .../FailingAssertionsSummary.css | 1 - client/components/TestRun/TestRun.css | 11 ----------- .../saved/_candidate-test-plan_24_1#summary.html | 2 +- .../snapshots/saved/_candidate-test-plan_24_1.html | 2 +- 6 files changed, 8 insertions(+), 15 deletions(-) diff --git a/client/components/CandidateReview/CandidateTestPlanRun/CandidateTestPlanRun.css b/client/components/CandidateReview/CandidateTestPlanRun/CandidateTestPlanRun.css index 82cb873aa..67172a8ef 100644 --- a/client/components/CandidateReview/CandidateTestPlanRun/CandidateTestPlanRun.css +++ b/client/components/CandidateReview/CandidateTestPlanRun/CandidateTestPlanRun.css @@ -152,3 +152,8 @@ position: relative; top: -5px; } + +.begin-review-button-container { + margin-left: auto; + flex-grow: 0; +} diff --git a/client/components/CandidateReview/CandidateTestPlanRun/index.jsx b/client/components/CandidateReview/CandidateTestPlanRun/index.jsx index c0652a38d..f500afa12 100644 --- a/client/components/CandidateReview/CandidateTestPlanRun/index.jsx +++ b/client/components/CandidateReview/CandidateTestPlanRun/index.jsx @@ -643,7 +643,7 @@ const CandidateTestPlanRun = () => { )} {isSummaryView ? ( -
  • +
  • diff --git a/client/tests/e2e/snapshots/saved/_candidate-test-plan_24_1.html b/client/tests/e2e/snapshots/saved/_candidate-test-plan_24_1.html index 1d6a34f6f..c65de7151 100644 --- a/client/tests/e2e/snapshots/saved/_candidate-test-plan_24_1.html +++ b/client/tests/e2e/snapshots/saved/_candidate-test-plan_24_1.html @@ -709,7 +709,7 @@

    Previous Test

    • -
  • +
  • From 79adb6d9e401f86320c2ae6e705674d9599b72dc Mon Sep 17 00:00:00 2001 From: Stalgia Grigg Date: Thu, 9 Jan 2025 09:05:31 -0800 Subject: [PATCH 11/18] Update client/hooks/useFailingAssertions.js Co-authored-by: Howard Edwards --- client/hooks/useFailingAssertions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/hooks/useFailingAssertions.js b/client/hooks/useFailingAssertions.js index 0e064b4b5..af4e1802a 100644 --- a/client/hooks/useFailingAssertions.js +++ b/client/hooks/useFailingAssertions.js @@ -14,7 +14,7 @@ export const useFailingAssertions = testPlanReport => { .filter(assertionResult => !assertionResult.passed) // We only want to show MUST and SHOULD assertions .filter( - assertionResult => assertionResult.assertion.priority !== 'MAY' + assertionResult => assertionResult.assertion.priority !== 'MAY' && assertionResult.assertion.priority !== 'EXCLUDE' ) .map(assertionResult => ({ testResultId: testResult.id, From 043e55bff2b5deb3733ec9095519d4a027ac910b Mon Sep 17 00:00:00 2001 From: Stalgia Grigg Date: Wed, 22 Jan 2025 10:12:32 -0800 Subject: [PATCH 12/18] Proper button state for single test review, hide summary when appropriate --- .../CandidateReview/CandidateTestPlanRun/index.jsx | 14 +++++++++----- client/components/TestRun/TestNavigator.jsx | 6 +++++- client/hooks/useFailingAssertions.js | 4 +++- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/client/components/CandidateReview/CandidateTestPlanRun/index.jsx b/client/components/CandidateReview/CandidateTestPlanRun/index.jsx index f500afa12..37fcb1d60 100644 --- a/client/components/CandidateReview/CandidateTestPlanRun/index.jsx +++ b/client/components/CandidateReview/CandidateTestPlanRun/index.jsx @@ -1,4 +1,4 @@ -import React, { useEffect, useRef, useState } from 'react'; +import React, { useEffect, useRef, useState, useMemo } from 'react'; import { useQuery, useMutation } from '@apollo/client'; import TestNavigator from '../../TestRun/TestNavigator'; import InstructionsRenderer from './InstructionsRenderer'; @@ -66,7 +66,7 @@ const CandidateTestPlanRun = () => { maxTestIndex: testsLength }); const [showTestNavigator, setShowTestNavigator] = useState(true); - const [isFirstTest, setIsFirstTest] = useState(true); + const [isFirstTest, setIsFirstTest] = useState(currentTestIndex === 0); const [isLastTest, setIsLastTest] = useState(false); const [feedbackModalShowing, setFeedbackModalShowing] = useState(false); const [confirmationModal, setConfirmationModal] = useState(null); @@ -83,6 +83,9 @@ const CandidateTestPlanRun = () => { const toggleTestNavigator = () => setShowTestNavigator(!showTestNavigator); + const hasFailingAssertionsSummary = useMemo(() => { + return data?.testPlanReports[0]?.metrics?.assertionsFailedCount > 0; + }, [data?.testPlanReports]); const handleTestClick = async index => { setCurrentTestIndex(index); if (index === -1) { @@ -227,11 +230,11 @@ const CandidateTestPlanRun = () => { if (data) { updateVendorStatus(); updateTestViewed(); - if (currentTestIndex !== 0) setIsFirstTest(false); + setIsFirstTest(currentTestIndex === 0); if (tests?.length === 1) setIsLastTest(true); if (currentTestIndex + 1 === tests?.length) setIsLastTest(true); } - }, [reviewStatus, currentTestIndex]); + }, [reviewStatus, currentTestIndex, data]); useEffect(() => { // Prevent a plan with only 1 test from immediately pushing the focus to the @@ -631,7 +634,8 @@ const CandidateTestPlanRun = () => { aria-labelledby="test-toolbar-heading" className="test-run-toolbar mt-1" > - {isSummaryView ? null : ( + {isSummaryView || + (isFirstTest && !hasFailingAssertionsSummary) ? null : (
  • From 72ba042a69104b61fdfe9d4b1da93700d3164b50 Mon Sep 17 00:00:00 2001 From: Stalgia Grigg Date: Thu, 30 Jan 2025 11:17:13 -0800 Subject: [PATCH 14/18] Use must and should for total tests, update snapshots --- client/components/FailingAssertionsSummary/Table.jsx | 6 ++++-- .../saved/_candidate-test-plan_24_1#summary.html | 2 +- client/tests/e2e/snapshots/saved/_data-management.html | 8 ++++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/client/components/FailingAssertionsSummary/Table.jsx b/client/components/FailingAssertionsSummary/Table.jsx index f81f1b928..773c7260d 100644 --- a/client/components/FailingAssertionsSummary/Table.jsx +++ b/client/components/FailingAssertionsSummary/Table.jsx @@ -20,8 +20,10 @@ const FailingAssertionsSummaryTable = ({ return ( <>

    - {metrics.assertionsFailedCount} assertions failed for{' '} - {metrics.commandsCount} commands in {metrics.testsFailedCount} tests. + {failingAssertions.length} assertions failed for{' '} + {metrics.mustAssertionsFailedCount + + metrics.shouldAssertionsFailedCount}{' '} + commands in {metrics.testsFailedCount} tests.

    diff --git a/client/tests/e2e/snapshots/saved/_candidate-test-plan_24_1#summary.html b/client/tests/e2e/snapshots/saved/_candidate-test-plan_24_1#summary.html index 0b44e992f..074e09b8e 100644 --- a/client/tests/e2e/snapshots/saved/_candidate-test-plan_24_1#summary.html +++ b/client/tests/e2e/snapshots/saved/_candidate-test-plan_24_1#summary.html @@ -297,7 +297,7 @@

    - 3 assertions failed for 26 commands in 2 tests. + 2 assertions failed for 3 commands in 2 tests.

    Test Plans Status Summary
    R&D -

    Complete Dec 17, 2024

    +

    Complete Jan 23, 2025

    		 @@ -2067,7 +2067,7 @@

    Test Plans Status Summary

    V24.12.17V25.01.23
    R&D -

    Complete Dec 17, 2024

    +

    Complete Jan 23, 2025

    		 @@ -2122,7 +2122,7 @@

    Test Plans Status Summary

    V24.12.17V25.01.23