Add support for automated Storybook accessibility testing (#4820)

* quick convert to csf 3

* cleanup and disabling aria-hidden rule

* adding storybook test runner

* sample failures for test to catch

* Revert "sample failures for test to catch"

This reverts commit 36fa68a.

* experimenting with interaction test

* poking around to try and ignore aria rule for specific selector

* test catching render errors and reuse play functions

* using story axe config instead of manually setting it in main

* fix lint and add keyboard interaction test

* experiementing with jest-axe

* testing interaction panel with simpler component

* update yarn.lock

* comments and cleanup

* removing axe-core stuff

this is a working commit before version bumping storybook interactions, for some reason it is breaking

* disabling some false positives and run against all stories

* fix storyName warning from storybook test runner

* adding partial DnD interactions test

* fixing yarn.lock after rebase

* fixing package.json from rebase

* update and cleanup tests + update to data test ids

* fix/ignore various story only aXe issues (Card,Dialog, Landmark, DnD)

some are fixed by adding ids/labels, other are ignored because they are story only tests cases and not part of the actual component accessibility

* fix/ignore various story only aXe issues (ListBox, NumberField, Table, Toast, RAC)

* fix/ignore various story only aXe issues (FocusScope, List, Checkbox, Meter, SideNav

* fixing the rest of the failing false positive tests

* fix lint

Author: LFDanLu

.storybook/test-runner.js

@@ -0,0 +1,43 @@
+const {configureAxe, checkA11y, injectAxe} = require('axe-playwright');
+const {getStoryContext} = require('@storybook/test-runner');
+
+
+/*
+* See https://storybook.js.org/docs/react/writing-tests/test-runner#test-hook-api-experimental
+* to learn more about the test-runner hooks API.
+*/
+module.exports = {
+  async preRender(page) {
+    await injectAxe(page);
+  },
+  async postRender(page, context) {
+    // Grab accessibility settings from the story itself
+    const storyContext = await getStoryContext(page, context);
+    if (storyContext.parameters?.a11y?.disable) {
+      return;
+    }
+
+    await configureAxe(page, {
+      // TODO: Ideally would have a selector target for the storybook's sb main body element
+      rules: [
+        {
+          id: 'color-contrast',
+          selector: 'body *:not([data-a11y-ignore="color-contrast"])'
+        },
+        {
+          id: 'aria-hidden-focus',
+          selector: 'body *:not([data-a11y-ignore="aria-hidden-focus"])',
+        },
+        ...(storyContext.parameters?.a11y?.config?.rules ?? [])
+      ]
+    });
+
+    await checkA11y(page, '#root', {
+      detailedReport: true,
+      detailedReportOptions: {
+        html: true,
+      },
+      axeOptions: storyContext.parameters?.a11y?.options,
+    });
+  },
+};

package.json

@@ -22,6 +22,7 @@
     "build:docs": "DOCS_ENV=staging parcel build 'packages/@react-{spectrum,aria,stately}/*/docs/*.mdx' 'packages/react-aria-components/docs/*.mdx' 'packages/@internationalized/*/docs/*.mdx' 'packages/dev/docs/pages/**/*.mdx'",
     "test": "cross-env STRICT_MODE=1 yarn jest",
     "test-loose": "yarn jest",
+    "test-storybook": "test-storybook --url http://localhost:9003  --browsers chromium --no-cache",
     "build": "make build",
     "test:ssr": "cross-env STRICT_MODE=1 yarn jest --config jest.ssr.config.js",
     "ci-test": "cross-env STRICT_MODE=1 yarn jest --maxWorkers=2 && cross-env STRICT_MODE=1 yarn test:ssr --runInBand",
@@ -80,10 +81,14 @@
     "@storybook/addon-a11y": "^6.5.12",
     "@storybook/addon-actions": "^6.5.12",
     "@storybook/addon-controls": "^6.5.12",
+    "@storybook/addon-links": "^6.5.12",
     "@storybook/addons": "^6.5.12",
     "@storybook/api": "^6.5.12",
     "@storybook/components": "^6.5.12",
+    "@storybook/jest": "^0.0.10",
     "@storybook/react": "^6.5.12",
+    "@storybook/test-runner": "^0.9.0",
+    "@storybook/testing-library": "^0.0.13",
     "@storybook/testing-react": "^1.3.0",
     "@testing-library/dom": "^9.2.0",
     "@testing-library/jest-dom": "^5.16.5",
@@ -95,6 +100,7 @@
     "@typescript-eslint/parser": "^5.40.0",
     "autoprefixer": "^9.6.0",
     "axe-core": "^4.6.3",
+    "axe-playwright": "^1.1.11",
     "babel-plugin-istanbul": "^6.0.0",
     "babel-plugin-macros": "^3.0.1",
     "babel-plugin-react-remove-properties": "^0.3.0",

packages/@react-aria/dnd/stories/DraggableListBox.tsx

@@ -37,7 +37,7 @@ export function DraggableListBox(props) {
   useDraggableCollection({}, dragState, ref);
 
   return (
-    <ul {...listBoxProps} ref={ref} className={dndStyles['draggable-listbox']}>
+    <ul {...listBoxProps} ref={ref} className={dndStyles['draggable-listbox']} aria-label="example draggable listbox">
       {[...state.collection].map((item) => (
         <Option
           key={item.key}

packages/@react-aria/dnd/stories/dnd.stories.tsx

@@ -58,9 +58,9 @@ export const Default = () => (
   <Flex direction="column" gap="size-200" alignItems="center">
     <Draggable />
     <Droppable />
-    <input />
+    <input aria-label="test input 1" />
     <Droppable type="text/html" />
-    <input />
+    <input aria-label="test input 2" />
     <Droppable />
   </Flex>
 );
@@ -74,7 +74,14 @@ export const NestedDropRegions = {
       </Droppable>
     </Flex>
   ),
-  name: 'nested drop regions'
+  name: 'nested drop regions',
+  parameters: {
+    a11y: {
+      config: {
+        rules: [{id: 'nested-interactive', enabled: false}]
+      }
+    }
+  }
 };
 
 export const DraggableListbox = {

packages/@react-aria/focus/stories/FocusScope.stories.tsx

@@ -94,12 +94,11 @@ export function Example({isPortaled, contain}: StoryProps) {
 
   return (
     <div>
-      <input />
-
+      <input aria-label="input before" />
       <button type="button" onClick={() => setOpen(true)}>
         Open dialog
       </button>
-      <input />
+      <input aria-label="input after" />
       {open && <NestedDialog onClose={() => setOpen(false)} isPortaled={isPortaled} contain={contain} />}
 
       <div id={dialogsRoot} />

packages/@react-aria/interactions/stories/useFocusRing.stories.tsx

@@ -39,7 +39,15 @@ export default {
 
 export const SearchTableview = {
   render: () => <SearchExample />,
-  name: 'search + tableview'
+  name: 'search + tableview',
+  parameters: {
+    a11y: {
+      config: {
+        // Fails due to TableView's known issue, ignoring here since it isn't pertinent to the story
+        rules: [{id: 'aria-required-children', selector: '*:not([role="grid"])'}]
+      }
+    }
+  }
 };
 
 function SearchExample() {
@@ -48,6 +56,7 @@ function SearchExample() {
   return (
     <div>
       <SearchField
+        aria-label="table searchfield"
         onChange={(value) => {
           const newItems = manyRows.filter((item) =>
             item['C0'].toLowerCase().includes(value.toLowerCase())

packages/@react-aria/landmark/stories/Landmark.stories.tsx

@@ -219,6 +219,7 @@ function NestedExample() {
   );
 }
 
+// TODO: known accessiblity failure https://github.com/adobe/react-spectrum/wiki/Known-accessibility-false-positives#tableview
 function TableExample() {
   return (
     <div>
@@ -428,23 +429,53 @@ export const NestedLandmarks = {
 };
 
 export const TableLandmark = {
-  render: TableTemplate
+  render: TableTemplate,
+  parameters: {
+    a11y: {
+      config: {
+        // Fails due to TableView's known issue, ignoring here since it isn't pertinent to the story
+        rules: [{id: 'aria-required-children', selector: '*:not([role="grid"])'}]
+      }
+    }
+  }
 };
 
 export const ApplicationWithLandmarks = {
-  render: ApplicationTemplate
+  render: ApplicationTemplate,
+  parameters: {
+    a11y: {
+      config: {
+        // Fails due to TableView's known issue, ignoring here since it isn't pertinent to the story
+        rules: [{id: 'aria-required-children', selector: '*:not([role="grid"])'}]
+      }
+    }
+  }
 };
 
 export const DuplicateRolesWithLabels = {
   render: DuplicateRolesWithLabelsTemplate
 };
 
 export const DuplicateRolesWithNoLabels = {
-  render: DuplicateRolesWithNoLabelsTemplate
+  render: DuplicateRolesWithNoLabelsTemplate,
+  parameters: {
+    a11y: {
+      config: {
+        rules: [{id: 'landmark-unique', enabled: false}]
+      }
+    }
+  }
 };
 
 export const DuplicateRolesWithSameLabels = {
-  render: DuplicateRolesWithSameLabelsTemplate
+  render: DuplicateRolesWithSameLabelsTemplate,
+  parameters: {
+    a11y: {
+      config: {
+        rules: [{id: 'landmark-unique', enabled: false}]
+      }
+    }
+  }
 };
 
 export const OneWithNoFocusableChildren = {
@@ -455,4 +486,14 @@ export const AllWithNoFocusableChildren = {
   render: AllWithNoFocusableChildrenExampleTemplate
 };
 
-export {IframeExample};
+export const IframeExampleStory = {
+  render: IframeExample,
+  parameters: {
+    a11y: {
+      config: {
+        rules: [{id: 'aria-allowed-role', enabled: false}]
+      }
+    }
+  },
+  name: 'iframe example'
+};

packages/@react-aria/menu/stories/useMenu.stories.tsx

@@ -43,7 +43,7 @@ export const DoubleMenuFiresOnInteractOutside = {
         <Item key="cut">Cut</Item>
         <Item key="paste">Paste</Item>
       </MenuButton>
-      <input />
+      <input aria-label="input after" />
     </div>
   ),
   name: 'double menu fires onInteractOutside'

packages/@react-aria/selection/stories/List.tsx

@@ -45,7 +45,7 @@ export function List<T extends object>(props: ListProps<T>) {
   });
 
   return (
-    <ul ref={ref} {...listProps} >
+    <ul ref={ref} {...listProps} role="listbox" aria-label={props['aria-label'] ?? 'test listbox'}>
       {[...state.collection].map((item) => (
         <ListItem key={item.key} item={item} state={state} />
       ))}

packages/@react-aria/selection/stories/useSelectableList.stories.tsx

@@ -125,7 +125,17 @@ const options = [
 ];
 
 export default {
-  title: 'useSelectableList'
+  title: 'useSelectableList',
+  parameters: {
+    a11y: {
+      config: {
+        rules: [
+          // Ignore landmark accessibility failure since the list is to test for selection/scrolling only
+          {id: 'list', enabled: false}
+        ]
+      }
+    }
+  }
 };
 
 export const StaticUlStaticSubUl = () => (

packages/@react-aria/table/stories/useTable.stories.tsx

@@ -106,12 +106,28 @@ export const ScrollTesting = {
 
 export const ActionTesting = {
   render: Template,
-  args: {selectionBehavior: 'replace', selectionStyle: 'highlight', onAction: action('onAction')}
+  args: {selectionBehavior: 'replace', selectionStyle: 'highlight', onAction: action('onAction')},
+  parameters: {
+    a11y: {
+      config: {
+        // False positive, tabbing into the table is handled by us and will focus the row
+        rules: [{id: 'scrollable-region-focusable', enabled: false}]
+      }
+    }
+  }
 };
 
 export const BackwardCompatActionTesting = {
   render: TemplateBackwardsCompat,
-  args: {selectionBehavior: 'replace', selectionStyle: 'highlight', onAction: action('onAction')}
+  args: {selectionBehavior: 'replace', selectionStyle: 'highlight', onAction: action('onAction')},
+  parameters: {
+    a11y: {
+      config: {
+        // False positive, tabbing into the table is handled by us and will focus the row
+        rules: [{id: 'scrollable-region-focusable', enabled: false}]
+      }
+    }
+  }
 };
 
 export const TableWithResizingNoProps = {

packages/@react-spectrum/actionbar/stories/ActionBar.stories.tsx

@@ -17,7 +17,6 @@ import {Example} from './Example';
 import React from 'react';
 import {useViewportSize} from '@react-aria/utils';
 
-
 export default {
   title: 'ActionBar',
   component: ActionBar,
@@ -39,10 +38,19 @@ export default {
 export type ActionBarStory = ComponentStoryObj<any>;
 
 export const Default: ActionBarStory = {
-  render: (args) => <Example {...args} />
+  render: (args) => <Example {...args} />,
+  parameters: {
+    a11y: {
+      config: {
+        // Fails due to TableView's known issue, ignoring here since it isn't pertinent to the story
+        rules: [{id: 'aria-required-children', selector: '*:not([role="grid"])'}]
+      }
+    }
+  }
 };
 
 export const FullWidthStory: ActionBarStory = {
+  ...Default,
   render: (args) => <FullWidth {...args} />
 };