Merge pull request #124 from storybookjs/tom/sb-270-update-test-runne…

…r-to-deal-with-docs

Fetch `index.json` and refer to "index json" mode.

README.md

@@ -11,7 +11,7 @@ Read the announcement: [Interaction Testing with Storybook](https://storybook.js
 - [CLI Options](#cli-options)
 - [Configuration](#configuration)
 - [Running against a deployed Storybook](#running-against-a-deployed-storybook)
-  - [Stories.json mode](#storiesjson-mode)
+  - [Index.json mode](#indexjson-mode)
 - [Running in CI](#running-in-ci)
   - [1. Running against deployed Storybooks on Github Actions deployment](#1-running-against-deployed-storybooks-on-github-actions-deployment)
   - [2. Running against locally built Storybooks in CI](#2-running-against-locally-built-storybooks-in-ci)
@@ -116,8 +116,8 @@ Usage: test-storybook [options]
 | Options                         | Description                                                                                                                      |
 | ------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- |
 | `--help`                        | Output usage information <br/>`test-storybook --help`                                                                            |
-| `-s`, `--stories-json`          | Run in stories json mode. Automatically detected (requires a compatible Storybook) <br/>`test-storybook --stories-json`          |
-| `--no-stories-json`             | Disables stories json mode <br/>`test-storybook --no-stories-json`                                                               |
+| `-i`, `--index-json`            | Run in index json mode. Automatically detected (requires a compatible Storybook) <br/>`test-storybook --index-json`              |
+| `--no-index-json`               | Disables index json mode <br/>`test-storybook --no-index-json`                                                                   |
 | `-c`, `--config-dir [dir-name]` | Directory where to load Storybook configurations from <br/>`test-storybook -c .storybook`                                        |
 | `--watch`                       | Run in watch mode <br/>`test-storybook --watch`                                                                                  |
 | `--coverage`                    | Indicates that test coverage information should be collected and reported in the output <br/>`test-storybook --coverage`         |
@@ -153,23 +153,38 @@ Or by using the `--url` flag:
 yarn test-storybook --url https://the-storybook-url-here.com
 ```
 
-### Stories.json mode
+### Index.json mode
 
-By default, the test runner transforms your story files into tests. It also supports a secondary "stories.json mode" which runs directly against your Storybook's `stories.json`, a static index of all the stories.
+By default, the test runner transforms your story files into tests. It also supports a secondary "index.json mode" which runs directly against your Storybook's index data, which dependending on your Storybook version is located in a `stories.json` or `index.json`, a static index of all the stories.
 
-This is particularly useful for running against a deployed storybook because `stories.json` is guaranteed to be in sync with the Storybook you are testing. In the default, story file-based mode, your local story files may be out of sync--or you might not even have access to the source code. Furthermore, it is not possible to run the test-runner directly against `.mdx` stories, and stories.json mode must be used.
+This is particularly useful for running against a deployed storybook because `index.json` is guaranteed to be in sync with the Storybook you are testing. In the default, story file-based mode, your local story files may be out of sync – or you might not even have access to the source code. Furthermore, it is not possible to run the test-runner directly against `.mdx` stories, and `index.json` mode must be used.
 
-To run in stories.json mode, first make sure your Storybook has a v3 `stories.json` file. You can navigate to:
+<!-- TODO: switch details to 6.4 once Storybook 7.0 becomes default -->
+
+<details>
+<summary>If you're using Storybook 7.0</summary>
+
+To run in `index.json` mode, first make sure your Storybook has a v4 `index.json` file. You can find it when navigating to:
 
 ```
-https://your-storybook-url-here.com/stories.json
+https://your-storybook-url-here.com/index.json
 ```
 
-It should be a JSON file and the first key should be `"v": 3` followed by a key called `"stories"` containing a map of story IDs to JSON objects.
+It should be a JSON file and the first key should be `"v": 4` followed by a key called `"entries"` containing a map of story IDs to JSON objects.
+
+In Storybok 7.0, `index.json` is enabled by default, unless you are using the `storiesOf()` syntax, in which case it is not supported.
+
+</details>
+
+On Storybook 6.4 and 6.5, to run in `index.json` mode, first make sure your Storybook has a file called `stories.json` that has `"v": 3`, available at:
+
+```
+https://your-storybook-url-here.com/stories.json
+```
 
-If your Storybook does not have a `stories.json` file, you can generate one provided:
+If your Storybook does not have a `stories.json` file, you can generate one, provided:
 
-- You are running SB6.4 or above
+- You are running Storybook 6.4 or above
 - You are not using `storiesOf` stories
 
 To enable `stories.json` in your Storybook, set the `buildStoriesJson` feature flag in `.storybook/main.js`:
@@ -180,21 +195,21 @@ module.exports = {
 };
 ```
 
-Once you have a valid `stories.json` file, your Storybook will be compatible with the "stories.json mode".
+Once you have a valid `stories.json` file, your Storybook will be compatible with the "index.json mode".
 
-By default, the test runner will detect whether your Storybook URL is local or remote, and if it is remote, it will run in "stories.json mode" automatically. To disable it, you can pass the `--no-stories-json` flag:
+By default, the test runner will detect whether your Storybook URL is local or remote, and if it is remote, it will run in "index.json mode" automatically. To disable it, you can pass the `--no-index-json` flag:
 
 ```bash
-yarn test-storybook --no-stories-json
+yarn test-storybook --no-index-json
 ```
 
-If you are running tests against a local Storybook but for some reason want to run in "stories.json mode", you can pass the `--stories-json` flag:
+If you are running tests against a local Storybook but for some reason want to run in "index.json mode", you can pass the `--index-json` flag:
 
 ```bash
-yarn test-storybook --stories-json
+yarn test-storybook --index-json
 ```
 
-> **NOTE:** stories.json mode is not compatible with watch mode.
+> **NOTE:** index.json mode is not compatible with watch mode.
 
 ## Running in CI
 

bin/test-storybook.js

@@ -29,11 +29,11 @@ const log = (message) => console.log(`[test-storybook] ${message}`);
 const error = (message) => console.error(`[test-storybook] ${message}`);
 
 // Clean up tmp files globally in case of control-c
-let storiesJsonTmpDir;
+let indexTmpDir;
 const cleanup = () => {
-  if (storiesJsonTmpDir) {
-    log(`Cleaning up ${storiesJsonTmpDir}`);
-    fs.rmSync(storiesJsonTmpDir, { recursive: true, force: true });
+  if (indexTmpDir) {
+    log(`Cleaning up ${indexTmpDir}`);
+    fs.rmSync(indexTmpDir, { recursive: true, force: true });
   }
 };
 
@@ -67,7 +67,7 @@ async function reportCoverage() {
   // --check-coverage if we want to break if coverage reaches certain threshold
   // .nycrc will be respected for thresholds etc. https://www.npmjs.com/package/nyc#coverage-thresholds
   execSync(`npx nyc report --reporter=text -t ${coverageFolder} --report-dir ${coverageFolder}`, { stdio: 'inherit' })
-} 
+}
 
 const onProcessEnd = () => {
   cleanup();
@@ -97,19 +97,21 @@ function sanitizeURL(url) {
 
 const checkForIncompatibilities = () => {
   try {
-    const jestVersion = require('jest/package.json').version
+    const jestVersion = require('jest/package.json').version;
     if (semver.gte(jestVersion, '28.0.0')) {
       error(dedent`We detected that your project is using Jest 28.0.0 or higher, which is currently incompatible with the test runner.
       
       You can find more info at: https://github.com/storybookjs/test-runner#errors-with-jest-28
-      `)
-      process.exit(1)
+      `);
+      process.exit(1);
     }
   } catch (err) {
-    error('We detected that Jest is not installed in your project. Please install it and run test-storybook again.')
-    process.exit(1)
+    error(
+      'We detected that Jest is not installed in your project. Please install it and run test-storybook again.'
+    );
+    process.exit(1);
   }
-}
+};
 
 async function executeJestPlaywright(args) {
   const jest = require('jest');
@@ -142,25 +144,56 @@ async function checkStorybook(url) {
   }
 }
 
-async function fetchStoriesJson(url) {
+async function getIndexJson(url) {
+  const indexJsonUrl = new URL('index.json', url).toString();
   const storiesJsonUrl = new URL('stories.json', url).toString();
+
+  const [indexRes, storiesRes] = await Promise.all([
+    fetch(indexJsonUrl),
+    fetch(storiesJsonUrl)
+  ]);
+
+  if (indexRes.ok) {
+    try {
+      const json = await indexRes.text();
+      return JSON.parse(json);
+    } catch (err) { }
+  }
+
+  if(storiesRes.ok) {
+    try {
+      const json = await storiesRes.text();
+      return JSON.parse(json);
+    } catch (err) { }
+  }
+
+  throw new Error(dedent`
+    Failed to fetch index data from the project.
+
+    Make sure that either of these URLs are available with valid data in your Storybook:
+    ${
+      // TODO: switch order once index.json becomes more common than stories.json
+      storiesJsonUrl
+    }
+    ${indexJsonUrl}
+
+    More info: https://github.com/storybookjs/test-runner/blob/main/README.md#indexjson-mode
+  `);
+}
+
+async function getIndexTempDir(url) {
   let tmpDir;
   try {
-    const res = await fetch(storiesJsonUrl);
-    const json = await res.text();
-    const titleIdToTest = transformPlaywrightJson(json);
+    const indexJson = await getIndexJson(url)
+    const titleIdToTest = transformPlaywrightJson(indexJson);
 
     tmpDir = tempy.directory();
     Object.entries(titleIdToTest).forEach(([titleId, test]) => {
       const tmpFile = path.join(tmpDir, `${titleId}.test.js`);
       fs.writeFileSync(tmpFile, test);
     });
   } catch (err) {
-    console.error(`Failed to fetch stories.json from ${storiesJsonUrl}`);
-    console.error(
-      'More info: https://github.com/storybookjs/test-runner/blob/main/README.md#storiesjson-mode\n'
-    );
-    console.error(err);
+    error(err);
     process.exit(1);
   }
   return tmpDir;
@@ -217,16 +250,16 @@ const main = async () => {
   const { hostname } = new URL(targetURL);
 
   const isLocalStorybookIp = await isLocalhostIp(hostname, true);
-  const shouldRunStoriesJson = runnerOptions.storiesJson !== false && !isLocalStorybookIp;
-  if (shouldRunStoriesJson) {
+  const shouldRunIndexJson = runnerOptions.indexJson !== false && !isLocalStorybookIp;
+  if (shouldRunIndexJson) {
     log(
-      'Detected a remote Storybook URL, running in stories json mode. To disable this, run the command again with --no-stories-json'
+      'Detected a remote Storybook URL, running in index json mode. To disable this, run the command again with --no-index-json\n'
     );
   }
 
-  if (runnerOptions.storiesJson || shouldRunStoriesJson) {
-    storiesJsonTmpDir = await fetchStoriesJson(targetURL);
-    process.env.TEST_ROOT = storiesJsonTmpDir;
+  if (runnerOptions.indexJson || shouldRunIndexJson) {
+    indexTmpDir = await getIndexTempDir(targetURL);
+    process.env.TEST_ROOT = indexTmpDir;
     process.env.TEST_MATCH = '**/*.test.js';
   }
 

src/playwright/transformPlaywrightJson.ts

@@ -1,18 +1,16 @@
 import * as t from '@babel/types';
 import generate from '@babel/generator';
-import { toId } from '@storybook/csf';
+import { ComponentTitle, StoryId, StoryName, toId } from '@storybook/csf';
 
 import { testPrefixer } from './transformPlaywright';
 
-type Story = { id: string; name: string; title: string; parameters?: Record<string, any> };
-
-const makeTest = (story: Story): t.Statement => {
+const makeTest = (entry: V4Entry): t.Statement => {
   const result: any = testPrefixer({
-    name: t.stringLiteral(story.name),
-    title: t.stringLiteral(story.title),
-    id: t.stringLiteral(story.id),
+    name: t.stringLiteral(entry.name),
+    title: t.stringLiteral(entry.title),
+    id: t.stringLiteral(entry.id),
     // FIXME
-    storyExport: t.identifier(story.id),
+    storyExport: t.identifier(entry.id),
   });
   const stmt = result[1] as t.ExpressionStatement;
   return t.expressionStatement(
@@ -29,28 +27,63 @@ const makeDescribe = (title: string, stmts: t.Statement[]) => {
   );
 };
 
-// FIXME: this should be a CSF helper
-const isDocsOnly = (stories: Story[]) => stories.length === 1 && stories[0].name === 'Page';
+type V4Entry = { type?: 'story' | 'docs'; id: StoryId; name: StoryName; title: ComponentTitle };
+type V4Index = {
+  v: 4;
+  entries: Record<StoryId, V4Entry>;
+};
+
+type V3Story = Omit<V4Entry, 'type'> & { parameters?: Record<string, any> };
+type V3StoriesIndex = {
+  v: 3;
+  stories: Record<StoryId, V3Story>;
+};
+const isV3DocsOnly = (stories: V3Story[]) => stories.length === 1 && stories[0].name === 'Page';
+
+function v3TitleMapToV4TitleMap(titleIdToStories: Record<string, V3Story[]>) {
+  return Object.fromEntries(
+    Object.entries(titleIdToStories).map(([id, stories]) => [
+      id,
+      stories.map(
+        ({ parameters, ...story }) =>
+          ({
+            type: isV3DocsOnly(stories) ? 'docs' : 'story',
+            ...story,
+          } as V4Entry)
+      ),
+    ])
+  );
+}
+
+function groupByTitleId<T extends { title: ComponentTitle }>(entries: T[]) {
+  return entries.reduce((acc, entry) => {
+    const titleId = toId(entry.title);
+    acc[titleId] = acc[titleId] || [];
+    acc[titleId].push(entry);
+    return acc;
+  }, {} as { [key: string]: T[] });
+}
 
 /**
  * Generate one test file per component so that Jest can
  * run them in parallel.
  */
-export const transformPlaywrightJson = (src: string) => {
-  const json = JSON.parse(src);
-  if (json.v !== 3) {
-    throw new Error(`Unsupported version ${json.v}`);
+export const transformPlaywrightJson = (index: Record<string, any>) => {
+  let titleIdToEntries: Record<string, V4Entry[]>;
+  if (index.v === 3) {
+    const titleIdToStories = groupByTitleId<V3Story>(
+      Object.values((index as V3StoriesIndex).stories)
+    );
+    titleIdToEntries = v3TitleMapToV4TitleMap(titleIdToStories);
+  } else if (index.v === 4) {
+    titleIdToEntries = groupByTitleId<V4Entry>(Object.values((index as V4Index).entries));
+  } else {
+    throw new Error(`Unsupported version ${index.v}`);
   }
-  const stories = Object.values(json.stories) as Story[];
-  const titleIdToStories = stories.reduce((acc, story) => {
-    const titleId = toId(story.title);
-    acc[titleId] = acc[titleId] || [];
-    acc[titleId].push(story);
-    return acc;
-  }, {} as { [key: string]: Story[] });
 
-  const titleIdToTest = Object.entries(titleIdToStories).reduce((acc, [titleId, stories]) => {
-    if (!isDocsOnly(stories)) {
+  const titleIdToTest = Object.entries(titleIdToEntries).reduce((acc, [titleId, entries]) => {
+    const stories = entries.filter((s) => s.type !== 'docs');
+    if (stories.length) {
       const storyTests = stories.map((story) => makeDescribe(story.name, [makeTest(story)]));
       const program = t.program([makeDescribe(stories[0].title, storyTests)]);
 

src/util/getCliOptions.test.ts

@@ -3,7 +3,7 @@ import * as cliHelper from './getParsedCliOptions';
 
 describe('getCliOptions', () => {
   it('returns custom options if passed', () => {
-    const customConfig = { configDir: 'custom', storiesJson: true };
+    const customConfig = { configDir: 'custom', indexJson: true };
     jest
       .spyOn(cliHelper, 'getParsedCliOptions')
       .mockReturnValue({ options: customConfig, extraArgs: [] });

src/util/getCliOptions.ts

@@ -3,7 +3,7 @@ import type { BrowserType } from 'jest-playwright-preset';
 
 type CliOptions = {
   runnerOptions: {
-    storiesJson?: boolean;
+    indexJson?: boolean;
     url?: string;
     configDir?: string;
     eject?: boolean;
@@ -16,7 +16,7 @@ type CliOptions = {
 type StorybookRunnerCommand = keyof CliOptions['runnerOptions'];
 
 const STORYBOOK_RUNNER_COMMANDS: StorybookRunnerCommand[] = [
-  'storiesJson',
+  'indexJson',
   'configDir',
   'browsers',
   'eject',

src/util/getParsedCliOptions.ts

@@ -2,11 +2,16 @@ export const getParsedCliOptions = () => {
   const { program } = require('commander');
 
   program
+    .option(
+      '-i, --index-json',
+      'Run in index json mode. Automatically detected (requires a compatible Storybook)'
+    )
     .option(
       '-s, --stories-json',
-      'Run in stories json mode. Automatically detected (requires a compatible Storybook)'
+      'Run in index json mode. Automatically detected (requires a compatible Storybook) [deprecated, use --index-json]'
     )
-    .option('--no-stories-json', 'Disable stories json mode')
+    .option('--no-index-json', 'Disable index json mode')
+    .option('--no-stories-json', 'Disable index json mode [deprecated, use --no-index-json]')
     .option(
       '-c, --config-dir <directory>',
       'Directory where to load Storybook configurations from',
@@ -67,8 +72,12 @@ export const getParsedCliOptions = () => {
     }
   }
 
+  const { storiesJson, ...options } = program.opts();
   return {
-    options: program.opts(),
+    options: {
+      indexJson: storiesJson,
+      ...options,
+    },
     extraArgs: program.args,
   };
 };