Re-enable pa11y (Shopify#1456)

Vilango · May 13, 2019 · 6df87a7 · 6df87a7
1 parent 72b0c51
commit 6df87a7
Show file tree

Hide file tree

Showing 7 changed files with 356 additions and 109 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -54,3 +54,4 @@ workflows:
     jobs:
       - check
       - percy
+      - a11y
diff --git a/UNRELEASED.md b/UNRELEASED.md
@@ -30,6 +30,8 @@ Use [the changelog guidelines](https://git.io/polaris-changelog-guidelines) to f
 
 ### Development workflow
 
+- Updated the a11y shitlist and re-enabled the pa11y job in CI. The job always passes for now, as a way for us to judge whether it is stable and can be made a required check. ([#1456](https://github.com/Shopify/polaris-react/pull/1456))
+
 ### Dependency upgrades
 
 ### Code quality

diff --git a/a11y_shitlist.json b/a11y_shitlist.json
diff --git a/scripts/pa11y-utilities.js b/scripts/pa11y-utilities.js
@@ -8,16 +8,16 @@ function shitlistCheck(results, immutableShitlist) {
   });
 
   const filteredResults = results.map((result) => {
-    if (mutableShitlist[result.pageUrl]) {
+    if (mutableShitlist[result.exampleID]) {
       result.issues = result.issues.filter((issue) => {
         const issueHash = hash(issue);
-        const matchIndex = mutableShitlist[result.pageUrl].findIndex(
+        const matchIndex = mutableShitlist[result.exampleID].findIndex(
           (shitlistedResult) => {
             return hash(shitlistedResult) === issueHash;
           },
         );
         if (matchIndex >= 0) {
-          mutableShitlist[result.pageUrl].splice(matchIndex, 1);
+          mutableShitlist[result.exampleID].splice(matchIndex, 1);
         }
         return matchIndex === -1;
       });
@@ -28,7 +28,7 @@ function shitlistCheck(results, immutableShitlist) {
   Object.keys(mutableShitlist).forEach((key) => {
     if (mutableShitlist[key].length) {
       remainingIssues.push({
-        pageUrl: key,
+        exampleID: key,
         issues: mutableShitlist[key],
       });
     }

diff --git a/scripts/pa11y.js b/scripts/pa11y.js
@@ -58,34 +58,35 @@ async function runPa11y() {
     .goto(iframePath)
     .then(() => page.evaluate(() => window.__STORYBOOK_CLIENT_API__.raw()));
 
-  const queryStrings = stories.reduce((memo, story) => {
+  const storyIDs = stories.reduce((memo, story) => {
     // There is no need to test the Playground, or the "All Examples" stories
     const isSkippedStory =
       story.kind === 'Playground|Playground' || story.name === 'All Examples';
 
     if (!isSkippedStory) {
-      memo.push(`id=${encodeURIComponent(story.id)}`);
+      memo.push(`${encodeURIComponent(story.id)}`);
     }
     return memo;
   }, []);
 
-  queryStrings.forEach((queryString) => {
+  storyIDs.forEach((queryString) => {
     const currentBrowser = browsers[browserIndex % NUMBER_OF_BROWSERS];
     browserIndex++;
     currentBrowser.taken = currentBrowser.taken.then(async () => {
       console.log('Testing ', queryString);
-      results.push(
-        await pa11y(`${iframePath}?${queryString}`, {
-          browser: currentBrowser.browser,
-          ignore: [
-            // Missing lang attribute on <html> tag
-            // Storybook does not include this property so ignore it
-            'WCAG2AA.Principle3.Guideline3_1.3_1_1.H57.2',
-            // Color contrast failures
-            'WCAG2AA.Principle1.Guideline1_4.1_4_3.G18.Fail',
-          ],
-        }),
-      );
+      const result = await pa11y(`${iframePath}?id=${queryString}`, {
+        browser: currentBrowser.browser,
+        ignore: [
+          // Missing lang attribute on <html> tag
+          // Storybook does not include this property so ignore it
+          'WCAG2AA.Principle3.Guideline3_1.3_1_1.H57.2',
+          // Color contrast failures
+          'WCAG2AA.Principle1.Guideline1_4.1_4_3.G18.Fail',
+        ],
+      });
+      result.exampleID = queryString;
+      delete result.pageUrl;
+      results.push(result);
     });
   });
 
@@ -122,7 +123,7 @@ Please edit the file a11y_shitlist.json to remove them and run these tests again
       console.log(
         '------------------------------------------------------------------------',
       );
-      console.log(issue.pageUrl);
+      console.log(issue.exampleID);
       console.log(
         '------------------------------------------------------------------------',
       );
@@ -144,7 +145,7 @@ The following issues were discovered and need to be fixed before this code can b
       console.log(
         '------------------------------------------------------------------------',
       );
-      console.log(result.pageUrl);
+      console.log(result.exampleID);
       console.log(
         '------------------------------------------------------------------------',
       );

diff --git a/src/components/Sheet/README.md b/src/components/Sheet/README.md
@@ -207,6 +207,7 @@ class SheetExample extends React.Component {
                   </div>
                   <Scrollable style={{padding: '1.6rem', height: '100%'}}>
                     <ChoiceList
+                      name="salesChannelsList"
                       choices={salesChannels}
                       selected={selected}
                       allowMultiple

diff --git a/tests/pa11y-utilities.test.js b/tests/pa11y-utilities.test.js
@@ -17,7 +17,7 @@ describe('shitlistCheck', () => {
   it('filters errors on the shitlist', () => {
     const issueList = [
       {
-        pageUrl: '伊藤美来',
+        exampleID: '伊藤美来',
         issues: [{code: 'ショッキングブルー'}, {code: '風の戦士'}],
       },
     ];
@@ -36,7 +36,7 @@ describe('shitlistCheck', () => {
   it('leaves errors not on the shitlist', () => {
     const issueList = [
       {
-        pageUrl: '伊藤美来',
+        exampleID: '伊藤美来',
         issues: [{code: '美来の色探し'}, {code: '風の戦士'}],
       },
     ];
@@ -49,7 +49,7 @@ describe('shitlistCheck', () => {
   it('removes entries with no issues from the list', () => {
     const issueList = [
       {
-        pageUrl: '虹香',
+        exampleID: '虹香',
         issues: [],
       },
     ];
@@ -62,7 +62,7 @@ describe('shitlistCheck', () => {
   it('returns a list with errors that werent found', () => {
     const issueList = [
       {
-        pageUrl: '伊藤美来',
+        exampleID: '伊藤美来',
         issues: [{code: 'ショッキングブルー'}],
       },
     ];
-Original file line number
+Diff line change
@@ Expand Up / @@ -54,3 +54,4 @@ workflows: @@
         jobs:
           - check
           - percy
+          - a11y