diff --git a/servers/fern-bot/src/functions/github-webhook-listener/actions/githubWebhookListener.ts b/servers/fern-bot/src/functions/github-webhook-listener/actions/githubWebhookListener.ts index a40408b3d3..e6d59bc440 100644 --- a/servers/fern-bot/src/functions/github-webhook-listener/actions/githubWebhookListener.ts +++ b/servers/fern-bot/src/functions/github-webhook-listener/actions/githubWebhookListener.ts @@ -1,44 +1,12 @@ -import { FernRegistryClient } from "@fern-fern/paged-generators-sdk"; import { Env } from "@libs/env"; -import { execFernCli } from "@libs/fern"; -import { cloneRepo, configureGit, setupGithubApp } from "@libs/github"; -import { EmitterWebhookEvent } from "@octokit/webhooks"; -import execa from "execa"; +import { setupGithubApp } from "@libs/github"; import { App } from "octokit"; -import { markCheckInProgress, updateCheck } from "./utilities"; -import { Octokit } from "octokit"; -import { type Repository } from "@libs/github/utilities"; -import { join } from "path"; - -interface OctokitMetadata { - octokit: Octokit; - repository: Repository; -} - -// Try to persist this in memory for the lambda -const repoNameToInstallationId: Map = new Map(); - -async function setRepoInstallationMap(app: App) { - await app.eachRepository((installation) => { - repoNameToInstallationId.set(installation.repository.full_name, { - octokit: installation.octokit, - repository: installation.repository, - }); - }); -} - -async function getInstallationFromRepo(repoFullName: string, app: App): Promise { - // Fetch from map, unless map is empty, indicating it's not been initialized, then hydrate it and return the value - if (repoNameToInstallationId.size == 0) { - await setRepoInstallationMap(app); - } - return repoNameToInstallationId.get(repoFullName); -} +import { deserializeRunId } from "./utilities"; +import { previewSdk, runDefaultAction } from "./previewSdk"; +import { initiatePreviewRuns } from "./initiatePreviewRuns"; export async function handleIncomingRequest(request: Request, env: Env): Promise { const application: App = setupGithubApp(env); - // Setup your in memory map of the repos - await setRepoInstallationMap(application); // Process the incoming events await actionWebhook(application, env); @@ -71,27 +39,6 @@ const verifySignature = async (app: App, request: Request): Promise => { }); }; -interface RunId { - // TODO: This should become some union of strings/enums of possible actions we can switch on in the check_run function - action: "sdk_preview"; - organizationId: string; - githubRepositoryFullName: string | undefined; - - // Also the docker image for the generator - generatorDockerImage: string; - groupName: string; - apiName: string | undefined; -} - -function stringifyRunId(runId: RunId): string { - return JSON.stringify(runId); -} - -function deserializeRunId(stringifiedRunId: string): RunId { - // TODO: we should throw here if it cannot deserialize correctly - return JSON.parse(stringifiedRunId) as RunId; -} - const actionWebhook = async (app: App, env: Env): Promise => { app.log.info("Listening for webhooks"); @@ -105,23 +52,12 @@ const actionWebhook = async (app: App, env: Env): Promise => { if (action === "requested" || action === "rerequested") { app.log.info(`A check run was requested: ${action}`); - // TODO: Check that it's a fern config repo - // Then actually create a check run SDK per-repo - await app.octokit.rest.checks.create({ - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - owner: context.payload.repository.owner.name!, - repo: context.payload.repository.name, - name: "🌿 SDK Preview", - head_sha: context.payload.check_suite.head_sha, - // TODO: fill this out with the repo data - external_id: stringifyRunId({ - action: "sdk_preview", - organizationId: "", - githubRepositoryFullName: "", - generatorDockerImage: "", - groupName: "", - apiName: "", - }), + // Kick off SDK previews + await initiatePreviewRuns({ + context, + app, + fernBotLoginName: env.GITHUB_APP_LOGIN_NAME, + fernBotLoginId: env.GITHUB_APP_LOGIN_ID, }); } }); @@ -133,7 +69,7 @@ const actionWebhook = async (app: App, env: Env): Promise => { // link to an external resource, as Chromatic does // // TODO: `actions` would be a good vector for kicking off a release after viewing the preview, etc. - app.webhooks.on("check_run", async (context) => { + app.webhooks.on("check_run", async (context): Promise => { // Now you know you've been given a check to run const action = context.payload.action; @@ -143,7 +79,7 @@ const actionWebhook = async (app: App, env: Env): Promise => { // Add new run actions here: switch (runId.action) { case "sdk_preview": - await runSdkPreview({ + await previewSdk({ context, app, fernBotLoginName: env.GITHUB_APP_LOGIN_NAME, @@ -158,171 +94,3 @@ const actionWebhook = async (app: App, env: Env): Promise => { } }); }; - -// We want to: -// 1. Update the status to in_progress -// 2. Pull the config repo -// 3. Run `fern generate --preview --group ${} --generator ${} --api ${}` -// 4. Kick off the checks (from generator CRUD API) -// 5. If there's a repo, push a branch to that repo -// 6. Complete the action with the checks status -// 6a. Include detailed logs in the `output` block -async function runSdkPreview({ - context, - app, - fernBotLoginName, - fernBotLoginId, - runId, - fdrUrl, -}: { - context: EmitterWebhookEvent<"check_run">; - app: App; - fernBotLoginName: string; - fernBotLoginId: string; - runId: RunId; - fdrUrl: string; -}) { - // Tell github we're working on this now - await markCheckInProgress({ context, app }); - if (context.payload.installation == null) { - await updateCheck({ - context, - app, - status: "completed", - conclusion: "failure", - output: { - title: "No installation found", - summary: "🌱 The Fern bot app was unable to determine the installation, and could not run checks. 😵", - text: undefined, - }, - }); - return; - } - - const fdrClient = new FernRegistryClient({ environment: fdrUrl }); - const { generatorDockerImage, groupName, apiName, githubRepositoryFullName } = runId; - // ==== DO THE ACTUAL ACTION ==== - const octokit = await app.getInstallationOctokit(context.payload.installation.id); - const repository = context.payload.repository; - const [git, fullRepoPath] = await configureGit(repository); - // Get the config repo - await cloneRepo(git, repository, octokit, fernBotLoginName, fernBotLoginId); - // Generate preview - let previewCommand = `generate --group ${groupName}`; - if (apiName != null) { - previewCommand += ` --api ${apiName}`; - } - await execFernCli(previewCommand, fullRepoPath, true); - // Kick off the checks + compile a summary - const generatorEntity = await fdrClient.generators.getGeneratorByImage({ dockerImage: generatorDockerImage }); - if (generatorEntity == null || generatorEntity.scripts == null) { - await runDefaultAction({ context, app }); - return; - } - const { preInstallScript, installScript, compileScript, testScript } = generatorEntity.scripts; - - let details: string | undefined; - let totalTasks = 0; - let failedTasks = 0; - if (preInstallScript != null) { - totalTasks++; - const [log, didFail] = await runScriptAndCollectOutput(preInstallScript.steps, "Setup"); - if (didFail) { - failedTasks++; - } - details += log + "\n\n"; - } - if (installScript != null) { - totalTasks++; - const [log, didFail] = await runScriptAndCollectOutput(installScript.steps, "Install"); - if (didFail) { - failedTasks++; - } - details += log + "\n\n"; - } - if (compileScript != null) { - const [log, didFail] = await runScriptAndCollectOutput(compileScript.steps, "Compile"); - if (didFail) { - failedTasks++; - } - details += log + "\n\n"; - } - if (testScript != null) { - const [log, didFail] = await runScriptAndCollectOutput(testScript.steps, "Test"); - if (didFail) { - failedTasks++; - } - details += log + "\n\n"; - } - - // Push the preview to a new branch - // HACK HACK: we should likely add a command to the CLI that spits out the preview path, since it's the one - // downloading the preview repo to disk - let previewUrl: string | undefined; - if (githubRepositoryFullName != null) { - let relativePathToPreview = `./.preview/${generatorDockerImage.replace("fernapi/", "")}`; - if (apiName != null) { - relativePathToPreview = join(`./apis/${apiName}`, relativePathToPreview); - } - relativePathToPreview = join(fullRepoPath, "fern", relativePathToPreview); - previewUrl = await setRemoteAndPush(githubRepositoryFullName, relativePathToPreview, app); - } - // ====== ACTION COMPLETE ====== - - // Tell github we're done and deliver the deets - let summary = `### 🌱 ${generatorEntity.generatorLanguage ?? "SDK"} Preview Checks - ${failedTasks}/${totalTasks} ${failedTasks > 0 ? "❌" : "✅"}\n\n`; - if (previewUrl != null) { - summary += `**[🔗 Generated Preview Link 🔗](${previewUrl})**`; - } - await updateCheck({ - context, - app, - status: "completed", - conclusion: "success", - output: { - title: `Preview ${generatorEntity.displayName} Generator: (\`${groupName}\`)`, - summary, - text: details, - }, - }); -} - -async function setRemoteAndPush(repoFullName: string, repositoryFullPath: string, app: App): Promise { - const repoMetadata = await getInstallationFromRepo(repoFullName, app); -} - -async function runScriptAndCollectOutput(commands: string[], sectionTitle: string): Promise<[string, boolean]> { - let outputs: string | undefined; - let didFail = false; - - for (const command in commands) { - // Write the command - outputs += `> $ ${command}\n\n`; - const out = await execa(command, { reject: false, all: true }); - if (out.exitCode != 0) { - didFail = true; - } - - if (out.all != null) { - // Write the logs - outputs += out.all + "\n\n"; - } - } - - const log = `**${sectionTitle}** - ${didFail ? "❌" : "✅"}\n\n\`\`\`\n${outputs}\n\`\`\``; - return [log, didFail]; -} - -async function runDefaultAction({ context, app }: { context: EmitterWebhookEvent<"check_run">; app: App }) { - await updateCheck({ - context, - app, - status: "completed", - conclusion: "neutral", - output: { - title: "No checks run", - summary: "🌱 No checks were run as a result of this commit 🚫", - text: undefined, - }, - }); -} diff --git a/servers/fern-bot/src/functions/github-webhook-listener/actions/initiatePreviewRuns.ts b/servers/fern-bot/src/functions/github-webhook-listener/actions/initiatePreviewRuns.ts new file mode 100644 index 0000000000..656e41b356 --- /dev/null +++ b/servers/fern-bot/src/functions/github-webhook-listener/actions/initiatePreviewRuns.ts @@ -0,0 +1,74 @@ +import { EmitterWebhookEvent } from "@octokit/webhooks"; +import { execFernCli, getGenerators, NO_API_FALLBACK_KEY } from "@libs/fern"; +import { App } from "octokit"; +import tmp from "tmp-promise"; +import { stringifyRunId } from "./utilities"; +import { cloneRepo, configureGit } from "@libs/github"; +import { readFile } from "fs/promises"; + +export async function initiatePreviewRuns({ + context, + app, + fernBotLoginName, + fernBotLoginId, +}: { + context: EmitterWebhookEvent<"check_suite">; + app: App; + fernBotLoginName: string; + fernBotLoginId: string; +}): Promise { + if (context.payload.installation == null) { + // If there's no installation ID, do not kick off any checks + console.log(); + return; + } + // Get the repo, and make sure it's a fern config repo + const octokit = await app.getInstallationOctokit(context.payload.installation.id); + const repository = context.payload.repository; + const [git, fullRepoPath] = await configureGit(repository); + await cloneRepo(git, repository, octokit, fernBotLoginName, fernBotLoginId); + + const generatorsList = await getGenerators(fullRepoPath); + for (const [apiName, api] of Object.entries(generatorsList)) { + for (const [groupName, group] of Object.entries(api)) { + for (const generator of group) { + let checkApiName: string | undefined; + if (apiName !== NO_API_FALLBACK_KEY) { + checkApiName = apiName; + } + + const tmpDir = await tmp.dir(); + const repoJsonFileName = `${tmpDir.path}/${repository.id.toString()}.json`; + let getRepoCommand = `generator get --repository --generator ${generator} --group ${groupName} -o ${repoJsonFileName}`; + if (apiName !== NO_API_FALLBACK_KEY) { + getRepoCommand += ` --api ${apiName}`; + } + await execFernCli(getRepoCommand, fullRepoPath); + const maybeRepo = await readFile(repoJsonFileName, "utf8"); + let maybeGithubRepositoryFullName: string | undefined; + if (maybeRepo?.length > 0) { + // Of the form { repository: string, language: string } + const generatorsYmlRepo = JSON.parse(maybeRepo); + if ("repository" in generatorsYmlRepo && generatorsYmlRepo.repository.length > 0) { + maybeGithubRepositoryFullName = generatorsYmlRepo.repository; + } + } + + await app.octokit.rest.checks.create({ + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + owner: context.payload.repository.owner.name!, + repo: context.payload.repository.name, + name: "🌿 SDK Preview", + head_sha: context.payload.check_suite.head_sha, + external_id: stringifyRunId({ + action: "sdk_preview", + githubRepositoryFullName: maybeGithubRepositoryFullName, + generatorDockerImage: generator, + groupName, + apiName: checkApiName, + }), + }); + } + } + } +} diff --git a/servers/fern-bot/src/functions/github-webhook-listener/actions/previewSdk.ts b/servers/fern-bot/src/functions/github-webhook-listener/actions/previewSdk.ts new file mode 100644 index 0000000000..1ceb9d99a8 --- /dev/null +++ b/servers/fern-bot/src/functions/github-webhook-listener/actions/previewSdk.ts @@ -0,0 +1,161 @@ +import { FernRegistryClient } from "@fern-fern/paged-generators-sdk"; +import { execFernCli } from "@libs/fern"; +import { cloneRepo, configureGit } from "@libs/github"; +import { EmitterWebhookEvent } from "@octokit/webhooks"; +import execa from "execa"; +import { App } from "octokit"; +import { markCheckInProgress, updateCheck, RunId } from "./utilities"; + +// We want to: +// 1. Update the status to in_progress +// 2. Pull the config repo +// 3. Run `fern generate --preview --group ${} --generator ${} --api ${}` +// 4. Kick off the checks (from generator CRUD API) +// 5. If there's a repo, push a branch to that repo +// 6. Complete the action with the checks status +// 6a. Include detailed logs in the `output` block +export async function previewSdk({ + context, + app, + fernBotLoginName, + fernBotLoginId, + runId, + fdrUrl, +}: { + context: EmitterWebhookEvent<"check_run">; + app: App; + fernBotLoginName: string; + fernBotLoginId: string; + runId: RunId; + fdrUrl: string; +}): Promise { + // Tell github we're working on this now + await markCheckInProgress({ context, app }); + if (context.payload.installation == null) { + await updateCheck({ + context, + app, + status: "completed", + conclusion: "failure", + output: { + title: "No installation found", + summary: "🌱 The Fern bot app was unable to determine the installation, and could not run checks. 😵", + text: undefined, + }, + }); + return; + } + + const fdrClient = new FernRegistryClient({ environment: fdrUrl }); + const { generatorDockerImage, groupName, apiName } = runId; + // ==== DO THE ACTUAL ACTION ==== + const octokit = await app.getInstallationOctokit(context.payload.installation.id); + const repository = context.payload.repository; + const [git, fullRepoPath] = await configureGit(repository); + // Get the config repo + await cloneRepo(git, repository, octokit, fernBotLoginName, fernBotLoginId); + // Generate preview + let previewCommand = `generate --group ${groupName}`; + if (apiName != null) { + previewCommand += ` --api ${apiName}`; + } + await execFernCli(previewCommand, fullRepoPath, true); + // Kick off the checks + compile a summary + const generatorEntity = await fdrClient.generators.getGeneratorByImage({ dockerImage: generatorDockerImage }); + if (generatorEntity == null || generatorEntity.scripts == null) { + await runDefaultAction({ context, app }); + return; + } + const { preInstallScript, installScript, compileScript, testScript } = generatorEntity.scripts; + + let details: string | undefined; + let totalTasks = 0; + let failedTasks = 0; + if (preInstallScript != null) { + totalTasks++; + const [log, didFail] = await runScriptAndCollectOutput(preInstallScript.steps, "Setup"); + if (didFail) { + failedTasks++; + } + details += log + "\n\n"; + } + if (installScript != null) { + totalTasks++; + const [log, didFail] = await runScriptAndCollectOutput(installScript.steps, "Install"); + if (didFail) { + failedTasks++; + } + details += log + "\n\n"; + } + if (compileScript != null) { + const [log, didFail] = await runScriptAndCollectOutput(compileScript.steps, "Compile"); + if (didFail) { + failedTasks++; + } + details += log + "\n\n"; + } + if (testScript != null) { + const [log, didFail] = await runScriptAndCollectOutput(testScript.steps, "Test"); + if (didFail) { + failedTasks++; + } + details += log + "\n\n"; + } + // ====== ACTION COMPLETE ====== + + // Tell github we're done and deliver the deets + const summary = `### 🌱 ${generatorEntity.generatorLanguage ?? "SDK"} Preview Checks - ${failedTasks}/${totalTasks} ${failedTasks > 0 ? "❌" : "✅"}\n\n`; + await updateCheck({ + context, + app, + status: "completed", + conclusion: "success", + output: { + title: `Preview ${generatorEntity.displayName} Generator: (\`${groupName}\`)`, + summary, + text: details, + }, + }); +} + +async function runScriptAndCollectOutput(commands: string[], sectionTitle: string): Promise<[string, boolean]> { + let outputs: string | undefined; + let didFail = false; + + for (const command in commands) { + // Write the command + outputs += `> $ ${command}\n\n`; + const out = await execa(command, { reject: false, all: true }); + if (out.exitCode != 0) { + didFail = true; + } + + if (out.all != null) { + // Write the logs + outputs += out.all + "\n\n"; + } + } + + const log = `**${sectionTitle}** - ${didFail ? "❌" : "✅"}\n\n\`\`\`\n${outputs}\n\`\`\``; + return [log, didFail]; +} + +export async function runDefaultAction({ + context, + app, +}: { + context: EmitterWebhookEvent<"check_run">; + app: App; +}): Promise { + await updateCheck({ + context, + app, + status: "completed", + conclusion: "neutral", + output: { + title: "No checks run", + summary: "🌱 No checks were run as a result of this commit 🚫", + text: undefined, + }, + }); +} diff --git a/servers/fern-bot/src/functions/github-webhook-listener/actions/utilities.ts b/servers/fern-bot/src/functions/github-webhook-listener/actions/utilities.ts index 2dc5727491..cc2136f614 100644 --- a/servers/fern-bot/src/functions/github-webhook-listener/actions/utilities.ts +++ b/servers/fern-bot/src/functions/github-webhook-listener/actions/utilities.ts @@ -1,6 +1,26 @@ import { App } from "octokit"; import { EmitterWebhookEvent } from "@octokit/webhooks"; +export interface RunId { + // TODO: This should become some union of strings/enums of possible actions we can switch on in the check_run function + action: "sdk_preview"; + githubRepositoryFullName: string | undefined; + + // Also the docker image for the generator + generatorDockerImage: string; + groupName: string; + apiName: string | undefined; +} + +export function stringifyRunId(runId: RunId): string { + return JSON.stringify(runId); +} + +export function deserializeRunId(stringifiedRunId: string): RunId { + // TODO: we should throw here if it cannot deserialize correctly + return JSON.parse(stringifiedRunId) as RunId; +} + // Octokit isn't exporting these types export type CheckStatus = "queued" | "in_progress" | "completed";