try again

packages/seed/src/cli.ts

@@ -73,7 +73,7 @@ function addTestCommand(cli: Argv) {
             const tests: Promise<boolean>[] = [];
 
             for (const generator of generators) {
-                if (argv.workspace != null && !argv.generator?.includes(generator.workspaceName)) {
+                if (argv.generator != null && !argv.generator.includes(generator.workspaceName)) {
                     continue;
                 }
                 const testRunner = new DockerTestRunner({

packages/seed/src/commands/rewrite-inputs/rewriteInputsForWorkspace.ts

@@ -31,7 +31,7 @@ export async function rewriteInputsForWorkspace({
     for (const fixture of fixtures) {
         const fixtureConfig = generator.workspaceConfig.fixtures?.[fixture];
         const docker = parseDockerOrThrow(generator.workspaceConfig.docker);
-        const absolutePathToFernDefinition = AbsoluteFilePath.of(
+        const absolutePathToAPIDefinition = AbsoluteFilePath.of(
             path.join(__dirname, FERN_DIRECTORY, APIS_DIRECTORY, fixture)
         );
         const absolutePathToOutput = join(generator.absolutePathToWorkspace, RelativeFilePath.of(fixture));
@@ -41,7 +41,7 @@ export async function rewriteInputsForWorkspace({
                     `${generator.workspaceName}:${fixture} - ${fixtureConfigInstance.outputFolder}`
                 );
                 const fernWorkspace = await convertGeneratorWorkspaceToFernWorkspace({
-                    absolutePathToWorkspace: absolutePathToFernDefinition,
+                    absolutePathToAPIDefinition,
                     taskContext,
                     fixture
                 });
@@ -71,7 +71,7 @@ export async function rewriteInputsForWorkspace({
         } else {
             const taskContext = taskContextFactory.create(`${generator.workspaceName}:${fixture}`);
             const fernWorkspace = await convertGeneratorWorkspaceToFernWorkspace({
-                absolutePathToWorkspace: absolutePathToFernDefinition,
+                absolutePathToAPIDefinition,
                 taskContext,
                 fixture
             });

packages/seed/src/commands/run/runWithCustomFixture.ts

@@ -37,7 +37,7 @@ export async function runWithCustomFixture({
     });
 
     const fernWorkspace = await convertGeneratorWorkspaceToFernWorkspace({
-        absolutePathToWorkspace: pathToFixture,
+        absolutePathToAPIDefinition: pathToFixture,
         taskContext,
         fixture: "custom"
     });

packages/seed/src/commands/test/printTestCases.ts

@@ -8,7 +8,8 @@ export function printTestCases(result: TestRunner.TestResult[]): void {
             Result: r.type,
             "Generation Time": r.metrics.generationTime,
             "Compile Time": r.metrics.compileTime,
-            "Failure Type": r.type === "failure" ? r.cause : ""
+            "Failure Type": r.type === "failure" ? r.cause : "",
+            "Failure Message": r.type === "failure" ? r.message : ""
         };
     });
     printTable(items);

packages/seed/src/commands/test/test-runner/TestRunner.ts

@@ -72,6 +72,7 @@ export declare namespace TestRunner {
 }
 
 export abstract class TestRunner {
+    private built: boolean = false;
     protected readonly generator: GeneratorWorkspace;
     protected readonly lock: Semaphore;
     protected readonly taskContextFactory: TaskContextFactory;
@@ -97,6 +98,11 @@ export abstract class TestRunner {
      * Runs the generator.
      */
     public async run({ fixture, configuration }: TestRunner.RunArgs): Promise<TestRunner.TestResult> {
+        if (!this.built) {
+            await this.build();
+            this.built = true;
+        }
+
         const metrics: TestRunner.TestCaseMetrics = {};
 
         const fixtureConfig = this.generator.workspaceConfig.fixtures?.[fixture];
@@ -110,13 +116,14 @@ export abstract class TestRunner {
             };
         }
 
-        const absolutePathToWorkspace = AbsoluteFilePath.of(
+        const id = configuration != null ? `${fixture}:${configuration.outputFolder}` : `${fixture}`;
+        const absolutePathToAPIDefinition = AbsoluteFilePath.of(
             path.join(__dirname, FERN_DIRECTORY, APIS_DIRECTORY, fixture)
         );
-        const taskContext = this.taskContextFactory.create(`${this.generator.workspaceName}:${fixture}`);
+        const taskContext = this.taskContextFactory.create(`${this.generator.workspaceName}:${id}`);
         const outputFolder = configuration?.outputFolder ?? fixture;
         const fernWorkspace = await convertGeneratorWorkspaceToFernWorkspace({
-            absolutePathToWorkspace,
+            absolutePathToAPIDefinition,
             taskContext,
             fixture
         });
@@ -137,9 +144,9 @@ export abstract class TestRunner {
             const generationStopwatch = new Stopwatch();
             generationStopwatch.start();
             await this.runGenerator({
-                id: configuration != null ? `${fixture}:${configuration.outputFolder}` : `${fixture}`,
+                id,
                 fernWorkspace,
-                absolutePathToWorkspace,
+                absolutePathToWorkspace: this.generator.absolutePathToWorkspace,
                 irVersion: this.generator.workspaceConfig.irVersion,
                 outputVersion: configuration?.outputVersion,
                 language: this.generator.workspaceConfig.language,

packages/seed/src/utils/convertSeedWorkspaceToFernWorkspace.ts

@@ -4,15 +4,15 @@ import { convertOpenApiWorkspaceToFernWorkspace, FernWorkspace, loadAPIWorkspace
 
 export async function convertGeneratorWorkspaceToFernWorkspace({
     fixture,
-    absolutePathToWorkspace,
+    absolutePathToAPIDefinition,
     taskContext
 }: {
     fixture: string;
-    absolutePathToWorkspace: AbsoluteFilePath;
+    absolutePathToAPIDefinition: AbsoluteFilePath;
     taskContext: TaskContext;
 }): Promise<FernWorkspace | undefined> {
     const workspace = await loadAPIWorkspace({
-        absolutePathToWorkspace,
+        absolutePathToWorkspace: absolutePathToAPIDefinition,
         context: taskContext,
         cliVersion: "DUMMY",
         workspaceName: fixture