Skip to content

Commit

Permalink
feat(fern-bot): add a weekly summary slack message for stale PRs (#1522)
Browse files Browse the repository at this point in the history
  • Loading branch information
armandobelardo authored Sep 22, 2024
1 parent 1550b90 commit 924c746
Show file tree
Hide file tree
Showing 15 changed files with 662 additions and 259 deletions.
1 change: 1 addition & 0 deletions .github/workflows/deploy-fern-bot-dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ env:
DEFAULT_FDR_ORIGIN: "https://registry-dev2.buildwithfern.com"
FERNIE_SLACK_APP_TOKEN: ${{ secrets.FERNIE_SLACK_APP_TOKEN }}
CUSTOMER_ALERTS_SLACK_CHANNEL: "customer-upgrades-dev"
CUSTOMER_PULLS_SLACK_CHANNEL: "customer-pulls-dev"
CO_API_KEY: ${{ secrets.DEV_CO_API_KEY }}
FERN_TOKEN: ${{ secrets.FERN_TOKEN }}

Expand Down
1 change: 1 addition & 0 deletions .github/workflows/deploy-fern-bot-prod.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ env:
GITHUB_APP_WEBHOOK_SECRET: ${{ secrets.FERN_BOT_PROD_GITHUB_APP_WEBHOOK_SECRET }}
FERNIE_SLACK_APP_TOKEN: ${{ secrets.FERNIE_SLACK_APP_TOKEN }}
CUSTOMER_ALERTS_SLACK_CHANNEL: "customer-upgrades"
CUSTOMER_PULLS_SLACK_CHANNEL: "customer-pulls"
DEFAULT_VENUS_ORIGIN: "https://venus.buildwithfern.com"
DEFAULT_FDR_ORIGIN: "https://registry.buildwithfern.com"
CO_API_KEY: ${{ secrets.PROD_CO_API_KEY }}
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/invoke-fern-bot-upgrade-generators.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ jobs:
GITHUB_APP_WEBHOOK_SECRET: ${{ secrets.FERN_BOT_DEV_GITHUB_APP_WEBHOOK_SECRET }}
FERNIE_SLACK_APP_TOKEN: ${{ secrets.FERNIE_SLACK_APP_TOKEN }}
CUSTOMER_ALERTS_SLACK_CHANNEL: "customer-upgrades-dev"
CUSTOMER_PULLS_SLACK_CHANNEL: "customer-pulls-dev"
DEFAULT_VENUS_ORIGIN: "https://venus-dev2.buildwithfern.com"
DEFAULT_FDR_ORIGIN: "https://registry-dev2.buildwithfern.com"
CO_API_KEY: ${{ secrets.DEV_CO_API_KEY }}
Expand Down Expand Up @@ -64,6 +65,7 @@ jobs:
CO_API_KEY: ${{ secrets.PROD_CO_API_KEY }}
FERNIE_SLACK_APP_TOKEN: ${{ secrets.FERNIE_SLACK_APP_TOKEN }}
CUSTOMER_ALERTS_SLACK_CHANNEL: "customer-upgrades"
CUSTOMER_PULLS_SLACK_CHANNEL: "customer-pulls"
REPO_TO_RUN_ON: ${{ github.event.inputs.repo }}
CI: false
steps:
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/publish-fdr-sdk.yml
Original file line number Diff line number Diff line change
Expand Up @@ -71,3 +71,4 @@ jobs:
git_version="$(scripts/fdr-version.sh)"
fern generate --api fdr --group fdr-cjs-sdk --version ${git_version} --log-level debug
fern generate --api fdr --group generators-sdk --version ${git_version} --log-level debug
fern generate --api fdr --group paged-generators-sdk --version ${git_version} --log-level debug
1 change: 1 addition & 0 deletions .github/workflows/test-fern-bot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ env:
DEFAULT_FDR_ORIGIN: "https://registry-dev2.buildwithfern.com"
FERNIE_SLACK_APP_TOKEN: ${{ secrets.FERNIE_SLACK_APP_TOKEN }}
CUSTOMER_ALERTS_SLACK_CHANNEL: "customer-upgrades-dev"
CUSTOMER_PULLS_SLACK_CHANNEL: "customer-pulls-dev"
CO_API_KEY: ${{ secrets.DEV_CO_API_KEY }}

jobs:
Expand Down
18 changes: 17 additions & 1 deletion fern/apis/fdr/generators.yml
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ groups:
- generators
generators:
- name: fernapi/fern-typescript-node-sdk
version: 0.38.6
version: 0.40.5
output:
location: npm
url: npm.buildwithfern.com
Expand All @@ -87,6 +87,22 @@ groups:
neverThrowErrors: true
timeoutInSeconds: infinity

paged-generators-sdk:
audiences:
- generators
generators:
- name: fernapi/fern-typescript-node-sdk
version: 0.40.5
output:
location: npm
url: npm.buildwithfern.com
package-name: "@fern-fern/paged-generators-sdk"
config:
noSerdeLayer: false
skipResponseValidation: true
outputSourceFiles: true
timeoutInSeconds: infinity

internal-snippets-sdks:
audiences:
- snippets
Expand Down
526 changes: 277 additions & 249 deletions pnpm-lock.yaml

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion servers/fern-bot/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@
"@fern-api/core-utils": "0.15.0-rc63",
"@fern-api/github": "workspace:*",
"@fern-api/venus-api-sdk": "0.8.1-1-gd6d1a5b",
"@fern-fern/generators-sdk": "0.110.0-a6864cd4c",
"@fern-fern/generators-sdk": "0.110.0-8f5b3cd3c",
"@fern-fern/paged-generators-sdk": "0.0.5706",
"@octokit/openapi-types": "^22.1.0",
"@slack/web-api": "^6.9.0",
"cohere-ai": "^7.9.5",
Expand Down
17 changes: 17 additions & 0 deletions servers/fern-bot/serverless.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ provider:
DEFAULT_FDR_ORIGIN: ${env:DEFAULT_FDR_ORIGIN, 'placeholder'}
FERNIE_SLACK_APP_TOKEN: ${env:FERNIE_SLACK_APP_TOKEN, 'placeholder'}
CUSTOMER_ALERTS_SLACK_CHANNEL: ${env:CUSTOMER_ALERTS_SLACK_CHANNEL, 'placeholder'}
CUSTOMER_PULLS_SLACK_CHANNEL: ${env:CUSTOMER_PULLS_SLACK_CHANNEL, 'placeholder'}
REPO_TO_RUN_ON: ${env:REPO_TO_RUN_ON, 'OMIT'}
FERN_TOKEN: ${env:FERN_TOKEN, 'OMIT'}

Expand Down Expand Up @@ -110,6 +111,22 @@ functions:
layers:
- arn:aws:lambda:us-east-1:553035198032:layer:git-lambda2:8

# =============================================

# Non-step function event-driven functions
sendStaleNotifications:
timeout: 900
memorySize: 5120
ephemeralStorageSize: 10240
handler: "src/functions/stale-notifs/sendStaleNotifications.handler"
layers:
- arn:aws:lambda:us-east-1:553035198032:layer:git-lambda2:8
events:
- schedule:
# Every Monday at 6a EST (10a UTC), note we run the below updates daily at midnight
rate: cron(0 10 * * ? 1)
enabled: true

stepFunctions:
stateMachines:
updateSpecs:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
import { Env } from "@libs/env";
import { FernRegistryClient } from "@fern-fern/paged-generators-sdk";
import { PullRequest, PullRequestState } from "@fern-fern/paged-generators-sdk/api";
import { SlackService } from "@libs/slack/SlackService";

const STALE_IN_DAYS = 7;
const STALE_IN_MS = STALE_IN_DAYS * 24 * 60 * 60 * 1000;
const VERBOSE_MESSAGE_THRESHOLD = 5;
const FERN_TEAM = new Set<string>([
"abvthecity",
"amckinney",
"armandobelardo",
"rohinbhargava",
"dsinghvi",
"dcb6",
"chdeskur",
"dannysheridan",
"fern-bot",
]);
// We don't care about notifying for our own orgs
const EXCLUDE_ORGS = new Set<string>(["fern-api", "fern-demo"]);

export async function sendStaleNotificationsInternal(env: Env): Promise<void> {
const client = new FernRegistryClient({ environment: env.DEFAULT_FDR_ORIGIN, token: env.FERN_TOKEN });
const botPulls = await client.git.listPullRequests({
// Is the author any fern member, or the github app?
author: [env.GITHUB_APP_LOGIN_NAME],
state: [PullRequestState.Open],
});

const orgPullMap = new Map<string, PullRequest[]>();
let staleBotPRsFound = false;
for await (const pull of botPulls) {
console.log(`about to process pull ${pull.repositoryOwner}`);
if (EXCLUDE_ORGS.has(pull.repositoryOwner)) {
continue;
}

if (pull.createdAt < new Date(Date.now() - STALE_IN_MS)) {
console.log("processing pull");
orgPullMap.set(pull.repositoryOwner, [...(orgPullMap.get(pull.repositoryOwner) || []), pull]);
staleBotPRsFound = true;
}
}

// Notify stale upgrade PRs to CUSTOMER_ALERTS_SLACK_CHANNEL
const upgradesSlackClient = new SlackService(env.FERNIE_SLACK_APP_TOKEN, env.CUSTOMER_ALERTS_SLACK_CHANNEL);
for (const [org, pulls] of orgPullMap) {
let maybeApiSpecPull: PullRequest | undefined;
const versionUpdatePulls: PullRequest[] = [];
for (const pull of pulls) {
console.log("found pull");
if (pull.title.includes("Update API Spec")) {
maybeApiSpecPull = pull;
} else {
versionUpdatePulls.push(pull);
}
}
if (maybeApiSpecPull != null || versionUpdatePulls.length > 0) {
console.log("we're here");
const allPulls = versionUpdatePulls.concat(maybeApiSpecPull ? [maybeApiSpecPull] : []);
const aPull = allPulls[0];
upgradesSlackClient.notifyStaleUpgradePRs({
organization: org,
apiSpecPull: maybeApiSpecPull,
versionUpdatePulls,
// We only call this function if there's at least one PR so this should be safe
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
repoName: `${aPull!.repositoryOwner}/${aPull!.repositoryName}`,
retoolLink: "https://buildwithfern.retool.com/apps/703271ca-7777-11ef-aecd-ab097775918e/Stale%20Pulls",
// Truncate the messages if we're going to be writing a lot of them
shouldBeVerbose: versionUpdatePulls.length <= VERBOSE_MESSAGE_THRESHOLD,
});
}
}
if (!staleBotPRsFound) {
console.log("No stale fern-bot PRs found");
}

const teamPulls = await client.git.listPullRequests({
// Is the author any fern member, or the github app?
author: Array.from(FERN_TEAM.values()),
state: [PullRequestState.Open],
});
const teamPullsSlackClient = new SlackService(env.FERNIE_SLACK_APP_TOKEN, env.CUSTOMER_PULLS_SLACK_CHANNEL);
// Notify on any PRs opened by us to CUSTOMER_PULLS_SLACK_CHANNEL
let numStaleTeamPulls = 0;
for await (const pull of teamPulls) {
if (EXCLUDE_ORGS.has(pull.repositoryOwner)) {
continue;
}
if (pull.createdAt < new Date(Date.now() - STALE_IN_MS)) {
numStaleTeamPulls++;
}
}
if (numStaleTeamPulls > 0) {
teamPullsSlackClient.notifyStaleTeamPulls({
numStaleTeamPulls,
retoolLink: "https://buildwithfern.retool.com/apps/703271ca-7777-11ef-aecd-ab097775918e/Stale%20Pulls",
});
} else {
console.log("No stale team PRs found");
}
}

// The below seems to not be possible with installation auth/a github app, so we'd need to use a personal access token
// which feels a bit jank right now, so just hardcoding the list above instead.
//
// Logins for fern team members, this is effectively a cache so we're not making requests to Github for every author
// const FERN_TEAM = new Set<string>();
// async function isFernOrgMember(login: string, app: App) {
// const membershipCheck = await app.octokit.rest.orgs.checkMembershipForUser({
// org: "fern-api",
// username: login,
// });

// if (membershipCheck.data) {
// FERN_TEAM.add(login);
// }

// return membershipCheck.data;
// }
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
import { evaluateEnv } from "@libs/env";
import { handlerWrapper } from "@libs/handler-wrapper";
import { sendStaleNotificationsInternal } from "./actions/sendStaleNotifications";

const sendStaleNotifications = async (event: unknown) => {
console.debug("Beginning scheduled run of `sendStaleNotifications`, received event:", event);
const env = evaluateEnv();
console.debug("Environment evaluated, continuing to actual action execution.");
return await sendStaleNotificationsInternal(env);
};

export const handler = handlerWrapper(sendStaleNotifications);
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,7 @@ export async function updateFDRRepoDataInternal(env: Env, repoData: RepoData | u

// Get repo data for the given repo
await app.eachRepository(async (installation) => {
if (
(repoData && installation.repository.full_name !== repoData.full_name) ||
installation.repository.full_name === "fern-api/fern-platform"
) {
if (repoData && installation.repository.full_name !== repoData.full_name) {
return;
}
await updateRepoDb(
Expand Down Expand Up @@ -128,13 +125,13 @@ async function updateRepoDb(

async function getAndUpsertPulls(client: FernRegistryClient, octokit: Octokit, repository: Repository) {
// Get all PRs on repo, update PRs in FDR
const pulls = await octokit.rest.pulls.list({
const pulls = await octokit.paginate(octokit.rest.pulls.list, {
state: "all",
owner: repository.owner.login,
repo: repository.name,
});

for (const pull of pulls.data) {
for (const pull of pulls) {
try {
await client.git.upsertPullRequest({
pullRequestNumber: pull.number,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { updateFDRRepoDataInternal } from "./actions/updateFDRRepoData";
import { RepoData } from "@libs/schemas";

const updateFDRRepoData = async (event: unknown) => {
console.debug("Beginning scheduled run of `updateRepoData`, received event:", event);
console.debug("Beginning scheduled run of `updateFDRRepoData`, received event:", event);
const env = evaluateEnv();
console.debug("Environment evaluated, continuing to actual action execution.");
return await updateFDRRepoDataInternal(env, event as RepoData | undefined);
Expand Down
2 changes: 2 additions & 0 deletions servers/fern-bot/src/libs/env.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ export interface Env {
DEFAULT_VENUS_ORIGIN: string;
DEFAULT_FDR_ORIGIN: string;
CUSTOMER_ALERTS_SLACK_CHANNEL: string;
CUSTOMER_PULLS_SLACK_CHANNEL: string;
FERNIE_SLACK_APP_TOKEN: string;
FERN_TOKEN: string;
}
Expand All @@ -41,6 +42,7 @@ export function evaluateEnv(): Env {
DEFAULT_FDR_ORIGIN: process?.env.DEFAULT_FDR_ORIGIN!,
FERNIE_SLACK_APP_TOKEN: process?.env.FERNIE_SLACK_APP_TOKEN!,
CUSTOMER_ALERTS_SLACK_CHANNEL: process?.env.CUSTOMER_ALERTS_SLACK_CHANNEL!,
CUSTOMER_PULLS_SLACK_CHANNEL: process?.env.CUSTOMER_PULLS_SLACK_CHANNEL!,
FERN_TOKEN: process?.env.FERN_TOKEN!,
};
}
Loading

0 comments on commit 924c746

Please sign in to comment.