From cbfba14ac10b9e8444ba873469fdfd3ceee39576 Mon Sep 17 00:00:00 2001 From: Robert Sese Date: Fri, 3 Sep 2021 11:34:48 -0500 Subject: [PATCH] Docker image deploy to Heroku (#21248) * Start parallel Docker image deploy workflows Co-authored-by: Mike Surowiec Co-authored-by: James M. Greene * Add early access content build stage Co-authored-by: Mike Surowiec * Create Heroku App script and workflow steps * Tag the image for Heroku * Push the image and grab the image ID * Set app name and image id outputs * Add parallel deploy script for Docker * Scope workflow run to 'docker-' and release image to Heroku * Update .github/workflows/staging-build-pr-docker.yml Co-authored-by: James M. Greene * Exclude Docker workflow * Cleanup Docker deploys * Use action sha Co-authored-by: Mike Surowiec Co-authored-by: James M. Greene Co-authored-by: James M. Greene --- .github/workflows/staging-build-pr-docker.yml | 94 +++ .../workflows/staging-deploy-pr-docker.yml | 312 +++++++++ .github/workflows/workflow-lint.yml | 2 +- Dockerfile | 11 + script/deployment/create-app.js | 44 ++ script/deployment/create-staging-app-name.js | 4 +- script/deployment/deploy-to-staging-docker.js | 609 ++++++++++++++++++ script/remove-stale-staging-apps.js | 2 +- 8 files changed, 1074 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/staging-build-pr-docker.yml create mode 100644 .github/workflows/staging-deploy-pr-docker.yml create mode 100644 script/deployment/create-app.js create mode 100644 script/deployment/deploy-to-staging-docker.js diff --git a/.github/workflows/staging-build-pr-docker.yml b/.github/workflows/staging-build-pr-docker.yml new file mode 100644 index 000000000000..710eb38f3791 --- /dev/null +++ b/.github/workflows/staging-build-pr-docker.yml @@ -0,0 +1,94 @@ +name: Staging - Build PR Docker + +# **What it does**: Builds PRs before deploying them. +# **Why we have it**: Because it's not safe to share our deploy secrets with forked repos: https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ +# **Who does it impact**: All contributors. + +on: + pull_request: + types: + - opened + - reopened + - synchronize + - unlocked + branches: + - 'docker-*' + +jobs: + build: + if: ${{ github.repository == 'github/docs-internal' || github.repository == 'github/docs' }} + name: Build + runs-on: ubuntu-latest + timeout-minutes: 5 + concurrency: + group: staging_${{ github.head_ref }} + cancel-in-progress: true + steps: + - name: Check out repo + uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f + + # Make sure only approved files are changed if it's in github/docs + - name: Check changed files + if: github.repository == 'github/docs' && github.event.pull_request.user.login != 'Octomerger' + uses: dorny/paths-filter@eb75a1edc117d3756a18ef89958ee59f9500ba58 + id: filter + with: + # Base branch used to get changed files + base: 'main' + + # Enables setting an output in the format in `${FILTER_NAME}_files + # with the names of the matching files formatted as JSON array + list-files: json + + # Returns list of changed files matching each filter + filters: | + notAllowed: + - '*.mjs' + - '*.ts' + - '*.tsx' + - '*.json' + - 'Dockerfile*' + + # When there are changes to files we can't accept + - name: 'Fail when not allowed files are changed' + if: ${{ steps.filter.outputs.notAllowed }} + run: exit 1 + + - name: Create an archive + run: | + tar -cf app.tar \ + assets/ \ + content/ \ + stylesheets/ \ + pages/ \ + data/ \ + includes/ \ + lib/ \ + middleware/ \ + translations/ \ + server.mjs \ + package*.json \ + .npmrc \ + feature-flags.json \ + next.config.js \ + tsconfig.json \ + next-env.d.ts \ + Dockerfile + + # Upload only the files needed to run + build this application. + # We are not willing to trust the rest (e.g. script/) for the remainder + # of the deployment process. + - name: Upload build artifact + uses: actions/upload-artifact@27121b0bdffd731efa15d66772be8dc71245d074 + with: + name: pr_build_docker + path: app.tar + + - name: Send Slack notification if workflow fails + uses: someimportantcompany/github-actions-slack-message@0b470c14b39da4260ed9e3f9a4f1298a74ccdefd + if: ${{ failure() }} + with: + channel: ${{ secrets.DOCS_STAGING_DEPLOYMENT_FAILURES_SLACK_CHANNEL_ID }} + bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }} + color: failure + text: Staging build (docker) failed for PR ${{ github.event.pull_request.html_url }} at commit ${{ github.sha }}. See https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/staging-deploy-pr-docker.yml b/.github/workflows/staging-deploy-pr-docker.yml new file mode 100644 index 000000000000..69a5671944ab --- /dev/null +++ b/.github/workflows/staging-deploy-pr-docker.yml @@ -0,0 +1,312 @@ +name: Staging - Deploy PR Docker + +# **What it does**: To deploy PRs to a Heroku staging environment. +# **Why we have it**: To deploy with high visibility in case of failures. +# **Who does it impact**: All contributors. + +on: + workflow_run: + workflows: + - 'Staging - Build PR Docker' + types: + - completed + +env: + EARLY_ACCESS_SCRIPT_PATH: script/early-access/clone-for-build.js + EARLY_ACCESS_SUPPORT_FILES: script/package.json + # In this specific workflow relationship, the `github.event.workflow_run.pull_requests` + # array will always contain only 1 item! Specifically, it will contain the PR associated + # with the `github.event.workflow_run.head_branch` that triggered the preceding + # `pull_request` event that triggered the "Staging - Build PR" workflow. + PR_URL: ${{ github.event.workflow_run.repository.html_url }}/pull/${{ github.event.workflow_run.pull_requests[0].number }} + +jobs: + prepare: + if: >- + ${{ + github.event.workflow_run.conclusion == 'success' && + (github.repository == 'github/docs-internal' || github.repository == 'github/docs') && + startsWith(github.event.workflow_run.head_branch, 'docker-') + }} + runs-on: ubuntu-latest + timeout-minutes: 5 + concurrency: + group: staging_${{ github.event.workflow_run.head_branch }} + cancel-in-progress: true + outputs: + source_blob_url: ${{ steps.build-source.outputs.download_url }} + app_name: ${{ steps.create-app.outputs.app_name}} + docker_image_id: ${{ steps.image-id.outputs.image_id}} + steps: + - name: Dump event context + env: + GITHUB_EVENT_CONTEXT: ${{ toJSON(github.event) }} + run: echo "$GITHUB_EVENT_CONTEXT" + + - name: Download build artifact + uses: dawidd6/action-download-artifact@b9571484721e8187f1fd08147b497129f8972c74 + with: + workflow: ${{ github.event.workflow_run.workflow_id }} + run_id: ${{ github.event.workflow_run.id }} + name: pr_build_docker + path: ./ + + - name: Show contents + run: ls -l + + - name: Extract the archive + run: | + tar -xf app.tar -C ./ + rm app.tar + + - name: Show contents again + run: ls -l + + - if: ${{ github.repository == 'github/docs-internal' }} + name: Setup node to clone early access + uses: actions/setup-node@38d90ce44d5275ad62cc48384b3d8a58c500bb5f + with: + node-version: 16.x + cache: npm + + - if: ${{ github.repository == 'github/docs-internal' }} + name: Download the script to clone early access + uses: Bhacaz/checkout-files@c8f01756bfd894ba746d5bf48205e19000b0742b + with: + files: ${{ env.EARLY_ACCESS_SCRIPT_PATH }} ${{ env.EARLY_ACCESS_SUPPORT_FILES }} + token: ${{ secrets.GITHUB_TOKEN }} + + # Add any dependencies that are needed for this workflow below + - if: ${{ github.repository == 'github/docs-internal' }} + name: Install temporary development-only dependencies + run: npm install --no-save rimraf dotenv + + - if: ${{ github.repository == 'github/docs-internal' }} + name: Clone early access + run: node ${{ env.EARLY_ACCESS_SCRIPT_PATH }} + env: + DOCUBOT_REPO_PAT: ${{ secrets.DOCUBOT_REPO_PAT }} + GIT_BRANCH: ${{ github.event.workflow_run.head_branch }} + + # Remove any dependencies installed for this workflow below + # - if: ${{ github.repository == 'github/docs-internal' }} + # name: Remove development-only dependencies + # run: npm prune --production + + - if: ${{ github.repository == 'github/docs-internal' }} + name: Delete the script directory after cloning early access + run: rm -rf script/ + + # - name: Create a gzipped archive + # run: | + # touch app.tar.gz + # tar --exclude=app.tar.gz -czf app.tar.gz ./ + + # - name: Install Heroku client development-only dependency + # run: npm install --no-save heroku-client + + # - name: Create a Heroku build source + # id: build-source + # uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d + # env: + # HEROKU_API_TOKEN: ${{ secrets.HEROKU_API_TOKEN }} + # with: + # script: | + # const { owner, repo } = context.repo + + # if (owner !== 'github') { + # throw new Error(`Repository owner must be 'github' but was: ${owner}`) + # } + # if (repo !== 'docs-internal' && repo !== 'docs') { + # throw new Error(`Repository name must be either 'docs-internal' or 'docs' but was: ${repo}`) + # } + + # const Heroku = require('heroku-client') + # const heroku = new Heroku({ token: process.env.HEROKU_API_TOKEN }) + + # const { source_blob: sourceBlob } = await heroku.post('/sources') + # const { put_url: uploadUrl, get_url: downloadUrl } = sourceBlob + + # core.setOutput('upload_url', uploadUrl) + # core.setOutput('download_url', downloadUrl) + + # # See: https://devcenter.heroku.com/articles/build-and-release-using-the-api#sources-endpoint + # - name: Upload to the Heroku build source + # run: | + # curl '${{ steps.build-source.outputs.upload_url }}' \ + # -X PUT \ + # -H 'Content-Type:' \ + # --data-binary @app.tar.gz + + - name: Install one-off development-only dependencies + run: npm install --no-save --include=optional esm + + - name: Create app + id: create-app + uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d + with: + script: | + const esm = require('esm') + require = esm({}) + + const { default: createApp } = require('./scripts/create-app.js') + const { default: parsePrUrl } = require('./scripts/parse-pr-url.js') + + // This helper uses the `GITHUB_TOKEN` implicitly! + // We're using our usual version of Octokit vs. the provided `github` + // instance to avoid versioning discrepancies. + const octokit = getOctokit() + + try { + const { owner, repo, pullNumber } = parsePrUrl(process.env.PR_URL) + if (!owner || !repo || !pullNumber) { + throw new Error(`'pullRequestUrl' input must match URL format 'https://github.com/github/(docs|docs-internal)/pull/123' but was '${PR_URL}'`) + } + + const { data: pullRequest } = await octokit.pulls.get({ + owner, + repo, + pull_number: pullNumber + }) + + const appName = await createApp(pullRequest) + core.setOutput('app_name', appName) + } catch(err) { + console.log(`Failed to create app: ${err}`) + throw(err) + } + + - name: Build, tag, push, and release the Docker image + env: + HEROKU_API_KEY: ${{ secrets.HEROKU_TOKEN }} + run: | + docker image build --target production_early_access -t registry.heroku.com/${{ steps.create-app.outputs.app_name}}/web . + heroku container:login + docker push registry.heroku.com/${{ steps.create-app.outputs.app_name }}/web + heroku container:release web --app=${{ steps.create-app.outputs.app_name }} + + # https://devcenter.heroku.com/articles/container-registry-and-runtime#getting-a-docker-image-id + - name: Get Docker Image ID + id: image-id + run: | + echo "::set-output name=image_id::$(docker image inspect registry.heroku.com/${{ steps.create-app.outputs.app_name }}/web --format={{.Id}})" + exit 1 # Stop at this point, don't move on to prepare job + + # TODO - heroku stuff + # - create a release based on the image + + - name: Send Slack notification if workflow fails + uses: someimportantcompany/github-actions-slack-message@0b470c14b39da4260ed9e3f9a4f1298a74ccdefd + if: ${{ failure() }} + with: + channel: ${{ secrets.DOCS_STAGING_DEPLOYMENT_FAILURES_SLACK_CHANNEL_ID }} + bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }} + color: failure + text: Staging preparation (docker) failed for PR ${{ env.PR_URL }} at commit ${{ github.sha }}. See https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + + deploy: + needs: prepare + runs-on: ubuntu-latest + timeout-minutes: 10 + concurrency: + group: staging_${{ github.event.workflow_run.head_branch }} + cancel-in-progress: true + steps: + - name: Check out repo's default branch + uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f + + - name: Setup node + uses: actions/setup-node@38d90ce44d5275ad62cc48384b3d8a58c500bb5f + with: + node-version: 16.x + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Install one-off development-only dependencies + run: npm install --no-save --include=optional esm + + - name: Deploy + id: deploy + uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HEROKU_API_TOKEN: ${{ secrets.HEROKU_API_TOKEN }} + HYDRO_ENDPOINT: ${{ secrets.HYDRO_ENDPOINT }} + HYDRO_SECRET: ${{ secrets.HYDRO_SECRET }} + PR_URL: ${{ env.PR_URL }} + SOURCE_BLOB_URL: ${{ needs.prepare.outputs.source_blob_url }} + APP_NAME: ${{ needs.prepare.outputs.app_name }} + DOCKER_IMAGE_ID: ${{ needs.prepare.outputs.docker_image_id }} + with: + script: | + const { GITHUB_TOKEN, HEROKU_API_TOKEN } = process.env + + // Exit if GitHub Actions PAT is not found + if (!GITHUB_TOKEN) { + throw new Error('You must supply a GITHUB_TOKEN environment variable!') + } + + // Exit if Heroku API token is not found + if (!HEROKU_API_TOKEN) { + throw new Error('You must supply a HEROKU_API_TOKEN environment variable!') + } + + // Workaround to allow us to load ESM files with `require(...)` + const esm = require('esm') + require = esm({}) + + const { default: parsePrUrl } = require('./script/deployment/parse-pr-url') + const { default: getOctokit } = require('./script/helpers/github') + const { default: deployToStaging } = require('./script/deployment/deploy-to-staging-docker') + + // This helper uses the `GITHUB_TOKEN` implicitly! + // We're using our usual version of Octokit vs. the provided `github` + // instance to avoid versioning discrepancies. + const octokit = getOctokit() + + try { + const { PR_URL, SOURCE_BLOB_URL } = process.env + const { owner, repo, pullNumber } = parsePrUrl(PR_URL) + if (!owner || !repo || !pullNumber) { + throw new Error(`'pullRequestUrl' input must match URL format 'https://github.com/github/(docs|docs-internal)/pull/123' but was '${PR_URL}'`) + } + + const { data: pullRequest } = await octokit.pulls.get({ + owner, + repo, + pull_number: pullNumber + }) + + await deployToStaging({ + octokit, + pullRequest, + forceRebuild: false, + // These parameters will ONLY be set by Actions + sourceBlobUrl: SOURCE_BLOB_URL, + runId: context.runId + }) + } catch (error) { + console.error(`Failed to deploy to staging: ${error.message}`) + console.error(error) + throw error + } + + - name: Mark the deployment as inactive if timed out + if: ${{ steps.deploy.outcome == 'cancelled' }} + uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d + with: + script: | + // TODO: Find the relevant deployment + // TODO: Create a new deployment status for it as "inactive" + return 'TODO' + + - name: Send Slack notification if workflow fails + uses: someimportantcompany/github-actions-slack-message@0b470c14b39da4260ed9e3f9a4f1298a74ccdefd + if: ${{ failure() }} + with: + channel: ${{ secrets.DOCS_STAGING_DEPLOYMENT_FAILURES_SLACK_CHANNEL_ID }} + bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }} + color: failure + text: Staging deployment failed for PR ${{ env.PR_URL }} at commit ${{ github.sha }}. See https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/workflow-lint.yml b/.github/workflows/workflow-lint.yml index c4b0657aaa6b..2153bb944ad0 100644 --- a/.github/workflows/workflow-lint.yml +++ b/.github/workflows/workflow-lint.yml @@ -28,4 +28,4 @@ jobs: - name: Run linter uses: cschleiden/actions-linter@caffd707beda4fc6083926a3dff48444bc7c24aa with: - workflows: '[".github/workflows/*.yml", ".github/workflows/*.yaml", "!.github/workflows/remove-from-fr-board.yaml", "!.github/workflows/staging-deploy-pr.yml"]' + workflows: '[".github/workflows/*.yml", ".github/workflows/*.yaml", "!.github/workflows/remove-from-fr-board.yaml", "!.github/workflows/staging-deploy-pr.yml", "!.github/workflows/staging-deploy-pr-docker.yml"]' diff --git a/Dockerfile b/Dockerfile index e2517e1f3429..3f1294560946 100644 --- a/Dockerfile +++ b/Dockerfile @@ -98,3 +98,14 @@ EXPOSE 80 EXPOSE 443 EXPOSE 4000 CMD ["node", "server.mjs"] + + +# -------------------------------------------------------------------------------- +# MAIN IMAGE WITH EARLY ACCESS +# -------------------------------------------------------------------------------- + +FROM production as production_early_access + +COPY --chown=node:node content/early-access ./content/early-access + +CMD ["node", "server.mjs"] diff --git a/script/deployment/create-app.js b/script/deployment/create-app.js new file mode 100644 index 000000000000..9392a6867af7 --- /dev/null +++ b/script/deployment/create-app.js @@ -0,0 +1,44 @@ +#!/usr/bin/env node +import Heroku from 'heroku-client' +import createAppName from './create-staging-app-name.js' + +export default async function createApp(pullRequest) { + // Extract some important properties from the PR + const { + number: pullNumber, + base: { + repo: { name: repo }, + }, + head: { ref: branch }, + } = pullRequest + + const appName = createAppName({ prefix: 'ghd', repo, pullNumber, branch }) + + // Check if there's already a Heroku App for this PR, if not create one + let appExists = true + const heroku = new Heroku({ token: process.env.HEROKU_API_TOKEN }) + + try { + await heroku.get(`/apps/${appName}`) + } catch (e) { + appExists = false + } + + if (!appExists) { + try { + const newApp = await heroku.post('/apps', { + body: { + name: appName, + }, + }) + + console.log('Heroku App created', newApp) + } catch (error) { + throw new Error(`Failed to create Heroku App ${appName}. Error: ${error}`) + } + } else { + console.log(`Heroku App ${appName} already exists.`) + } + + return appName +} diff --git a/script/deployment/create-staging-app-name.js b/script/deployment/create-staging-app-name.js index e49a31193f44..36bef07b8a86 100644 --- a/script/deployment/create-staging-app-name.js +++ b/script/deployment/create-staging-app-name.js @@ -4,9 +4,9 @@ const slugify = GithubSlugger.slug const APP_NAME_MAX_LENGTH = 30 -export default function ({ repo, pullNumber, branch }) { +export default function ({ prefix = 'gha', repo, pullNumber, branch }) { return ( - `gha-${repo}-${pullNumber}--${slugify(branch)}` + `${prefix}-${repo}-${pullNumber}--${slugify(branch)}` // Shorten the string to the max allowed length .slice(0, APP_NAME_MAX_LENGTH) // Convert underscores to dashes diff --git a/script/deployment/deploy-to-staging-docker.js b/script/deployment/deploy-to-staging-docker.js new file mode 100644 index 000000000000..507ee3f69b8e --- /dev/null +++ b/script/deployment/deploy-to-staging-docker.js @@ -0,0 +1,609 @@ +#!/usr/bin/env node +import sleep from 'await-sleep' +import got from 'got' +import Heroku from 'heroku-client' +import createStagingAppName from './create-staging-app-name.js' + +const SLEEP_INTERVAL = 5000 +const HEROKU_LOG_LINES_TO_SHOW = 25 + +// Allow for a few 404 (Not Found) or 429 (Too Many Requests) responses from the +// semi-unreliable Heroku API when we're polling for status updates +const ALLOWED_MISSING_RESPONSE_COUNT = 5 + +export default async function deployToStaging({ + octokit, + pullRequest, + forceRebuild = false, + // These parameters will only be set by Actions + sourceBlobUrl = null, + runId = null, +}) { + // Start a timer so we can report how long the deployment takes + const startTime = Date.now() + + // Extract some important properties from the PR + const { + number: pullNumber, + base: { + repo: { + name: repo, + owner: { login: owner }, + }, + }, + state, + head: { ref: branch, sha }, + user: author, + } = pullRequest + + // Verify the PR is still open + if (state !== 'open') { + throw new Error(`This pull request is not open. State is: '${state}'`) + } + + // Put together application configuration variables + const isPrivateRepo = owner === 'github' && repo === 'docs-internal' + const isPrebuilt = !!sourceBlobUrl + const { DOCUBOT_REPO_PAT, HYDRO_ENDPOINT, HYDRO_SECRET } = process.env + const appConfigVars = { + // Track the git branch + GIT_BRANCH: branch, + // If prebuilt: prevent the Heroku Node.js buildpack from installing devDependencies + NPM_CONFIG_PRODUCTION: isPrebuilt.toString(), + // If prebuilt: prevent the Heroku Node.js buildpack from using `npm ci` as it would + // delete all of the vendored "node_modules/" directory. + USE_NPM_INSTALL: isPrebuilt.toString(), + // IMPORTANT: These secrets should only be set in the private repo! + // This is only required for cloning the `docs-early-access` repo + ...(isPrivateRepo && !isPrebuilt && DOCUBOT_REPO_PAT && { DOCUBOT_REPO_PAT }), + // These are required for Hydro event tracking + ...(isPrivateRepo && HYDRO_ENDPOINT && HYDRO_SECRET && { HYDRO_ENDPOINT, HYDRO_SECRET }), + } + + const workflowRunLog = runId ? `https://github.com/${owner}/${repo}/actions/runs/${runId}` : null + let deploymentId = null + let logUrl = workflowRunLog + let appIsNewlyCreated = false + + const appName = createStagingAppName({ repo, pullNumber, branch }) + const homepageUrl = `https://${appName}.herokuapp.com/` + + try { + const title = `branch '${branch}' at commit '${sha}' in the 'staging' environment as '${appName}'` + + console.log(`About to deploy ${title}...`) + + // Kick off a pending GitHub Deployment right away, so the PR author + // will have instant feedback that their work is being deployed. + const { data: deployment } = await octokit.repos.createDeployment({ + owner, + repo, + + description: `Deploying ${title}`, + + // Use a commit SHA instead of a branch name as the ref for more precise + // feedback, and also because the branch may have already been deleted. + ref: sha, + + // In the GitHub API, there can only be one active deployment per environment. + // For our many staging apps, we must use the unique appName as the environment. + environment: appName, + + // Indicate this environment will no longer exist at some point in the future. + transient_environment: true, + + // The status contexts to verify against commit status checks. If you omit + // this parameter, GitHub verifies all unique contexts before creating a + // deployment. To bypass checking entirely, pass an empty array. Defaults + // to all unique contexts. + required_contexts: [], + + // Do not try to merge the base branch into the feature branch + auto_merge: false, + }) + console.log('GitHub Deployment created', deployment) + + // Store this ID for later updating + deploymentId = deployment.id + + await octokit.repos.createDeploymentStatus({ + owner, + repo, + deployment_id: deploymentId, + state: 'in_progress', + description: 'Deploying the app...', + // The 'ant-man' preview is required for `state` values of 'inactive', as well as + // the use of the `log_url`, `environment_url`, and `auto_inactive` parameters. + // The 'flash' preview is required for `state` values of 'in_progress' and 'queued'. + mediaType: { + previews: ['ant-man', 'flash'], + }, + }) + console.log('🚀 Deployment status: in_progress - Preparing to deploy the app...') + + // Time to talk to Heroku... + const heroku = new Heroku({ token: process.env.HEROKU_API_TOKEN }) + let appSetup = null + let build = null + + // Is there already a Heroku App for this PR? + let appExists = true + try { + await heroku.get(`/apps/${appName}`) + } catch (error) { + appExists = false + } + + // If there is an existing app but we want to forcibly rebuild, delete the app first + if (appExists && forceRebuild) { + console.log('🚀 Deployment status: in_progress - Destroying existing Heroku app...') + + try { + await heroku.delete(`/apps/${appName}`) + appExists = false + + console.log(`Heroku app '${appName}' deleted for forced rebuild`) + } catch (error) { + throw new Error( + `Failed to delete Heroku app '${appName}' for forced rebuild. Error: ${error}` + ) + } + } + + if (!sourceBlobUrl) { + try { + sourceBlobUrl = await getTarballUrl({ + octokit, + owner, + repo, + sha, + }) + } catch (error) { + throw new Error(`Failed to generate source blob URL. Error: ${error}`) + } + } + + // If an app does not exist, create one! + // This action will also trigger a build as a by-product. + if (!appExists) { + appIsNewlyCreated = true + + console.log(`Heroku app '${appName}' does not exist. Creating a new AppSetup...`) + + console.log('🚀 Deployment status: in_progress - Creating a new Heroku app...') + + const appSetupStartTime = Date.now() + try { + appSetup = await heroku.post('/app-setups', { + body: { + app: { + name: appName, + }, + source_blob: { + url: sourceBlobUrl, + }, + + // Pass some environment variables to staging apps via Heroku + // config variables. + overrides: { + env: appConfigVars, + }, + }, + }) + console.log('Heroku AppSetup created', appSetup) + + // This probably will not be available yet + build = appSetup.build + } catch (error) { + throw new Error(`Failed to create Heroku app '${appName}'. Error: ${error}`) + } + + // Add PR author (if staff) as a collaborator on the new staging app + try { + if (author.site_admin === true) { + await heroku.post(`/apps/${appName}/collaborators`, { + body: { + user: `${author.login}@github.com`, + // We don't want an email invitation for every new staging app + silent: true, + }, + }) + console.log(`Added PR author @${author.login} as a Heroku app collaborator`) + } + } catch (error) { + // It's fine if this fails, it shouldn't block the app from deploying! + console.warn( + `Warning: failed to add PR author as a Heroku app collaborator. Error: ${error}` + ) + } + + // A new Build is created as a by-product of creating an AppSetup. + // Poll until there is a Build object attached to the AppSetup. + let setupAcceptableErrorCount = 0 + while (!build || !build.id) { + await sleep(SLEEP_INTERVAL) + try { + appSetup = await heroku.get(`/app-setups/${appSetup.id}`) + build = appSetup.build + } catch (error) { + // Allow for a few bad responses from the Heroku API + if (error.statusCode === 404 || error.statusCode === 429) { + setupAcceptableErrorCount += 1 + if (setupAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) { + continue + } + } + throw new Error(`Failed to get AppSetup status. Error: ${error}`) + } + + console.log( + `AppSetup status: ${appSetup.status} (after ${Math.round( + (Date.now() - appSetupStartTime) / 1000 + )} seconds)` + ) + } + + console.log('Heroku build detected', build) + } else { + // If the app does exist, just manually trigger a new build + console.log(`Heroku app '${appName}' already exists.`) + + console.log('Updating Heroku app configuration variables...') + + // Reconfigure environment variables + // https://devcenter.heroku.com/articles/platform-api-reference#config-vars-update + try { + await heroku.patch(`/apps/${appName}/config-vars`, { + body: appConfigVars, + }) + } catch (error) { + throw new Error(`Failed to update Heroku app configuration variables. Error: ${error}`) + } + + console.log('Reconfigured') + console.log('Building Heroku app...') + + try { + build = await heroku.post(`/apps/${appName}/builds`, { + body: { + source_blob: { + url: sourceBlobUrl, + }, + }, + }) + } catch (error) { + throw new Error(`Failed to create Heroku build. Error: ${error}`) + } + + console.log('Heroku build created', build) + } + + const buildStartTime = Date.now() // Close enough... + const buildId = build.id + logUrl = build.output_stream_url + + console.log('🚀 Deployment status: in_progress - Building a new Heroku slug...') + + // Poll until the Build's status changes from "pending" to "succeeded" or "failed". + let buildAcceptableErrorCount = 0 + while (!build || build.status === 'pending' || !build.release || !build.release.id) { + await sleep(SLEEP_INTERVAL) + try { + build = await heroku.get(`/apps/${appName}/builds/${buildId}`) + } catch (error) { + // Allow for a few bad responses from the Heroku API + if (error.statusCode === 404 || error.statusCode === 429) { + buildAcceptableErrorCount += 1 + if (buildAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) { + continue + } + } + throw new Error(`Failed to get build status. Error: ${error}`) + } + console.log( + `Heroku build status: ${(build || {}).status} (after ${Math.round( + (Date.now() - buildStartTime) / 1000 + )} seconds)` + ) + } + + if (build.status !== 'succeeded') { + throw new Error( + `Failed to build after ${Math.round( + (Date.now() - buildStartTime) / 1000 + )} seconds. See Heroku logs for more information:\n${logUrl}` + ) + } + + console.log( + `Finished Heroku build after ${Math.round((Date.now() - buildStartTime) / 1000)} seconds.`, + build + ) + + const releaseStartTime = Date.now() // Close enough... + let releaseId = build.release.id + let release = null + + // Poll until the associated Release's status changes from "pending" to "succeeded" or "failed". + let releaseAcceptableErrorCount = 0 + while (!release || release.status === 'pending') { + await sleep(SLEEP_INTERVAL) + try { + const result = await heroku.get(`/apps/${appName}/releases/${releaseId}`) + + // Update the deployment status but only on the first retrieval + if (!release) { + logUrl = result.output_stream_url + + console.log('Heroku Release created', result) + + console.log('🚀 Deployment status: in_progress - Releasing the built Heroku slug...') + } + + release = result + } catch (error) { + // Allow for a few bad responses from the Heroku API + if (error.statusCode === 404 || error.statusCode === 429) { + releaseAcceptableErrorCount += 1 + if (releaseAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) { + continue + } + } + throw new Error(`Failed to get release status. Error: ${error}`) + } + + console.log( + `Release status: ${(release || {}).status} (after ${Math.round( + (Date.now() - releaseStartTime) / 1000 + )} seconds)` + ) + } + + if (release.status !== 'succeeded') { + throw new Error( + `Failed to release after ${Math.round( + (Date.now() - releaseStartTime) / 1000 + )} seconds. See Heroku logs for more information:\n${logUrl}` + ) + } + + console.log( + `Finished Heroku release after ${Math.round( + (Date.now() - releaseStartTime) / 1000 + )} seconds.`, + release + ) + + // Monitor dyno state for this release to ensure it reaches "up" rather than crashing. + // This will help us catch issues with faulty startup code and/or the package manifest. + const dynoBootStartTime = Date.now() + console.log('Checking Heroku dynos...') + logUrl = workflowRunLog + + console.log('🚀 Deployment status: in_progress - Monitoring the Heroku dyno start-up...') + + // Keep checking while there are still dynos in non-terminal states + let newDynos = [] + let dynoAcceptableErrorCount = 0 + while (newDynos.length === 0 || newDynos.some((dyno) => dyno.state === 'starting')) { + await sleep(SLEEP_INTERVAL) + try { + const dynoList = await heroku.get(`/apps/${appName}/dynos`) + const dynosForThisRelease = dynoList.filter((dyno) => dyno.release.id === releaseId) + + // To track them afterward + newDynos = dynosForThisRelease + + // Dynos for this release OR a newer release + const relevantDynos = dynoList.filter((dyno) => dyno.release.version >= release.version) + + // If this Heroku app was just newly created, often a secondary release + // is requested to enable automatically managed SSL certificates. The + // release description will read: + // "Enable allow-multiple-sni-endpoints feature" + // + // If that is the case, we need to update to monitor that secondary + // release instead. + if (relevantDynos.length > 0 && dynosForThisRelease.length === 0) { + // If the app is NOT newly created, fail fast! + if (!appIsNewlyCreated) { + throw new Error('The dynos for this release disappeared unexpectedly') + } + + // Check for the secondary release + let nextRelease = null + try { + nextRelease = await heroku.get(`/apps/${appName}/releases/${release.version + 1}`) + } catch (error) { + throw new Error( + `Could not find a secondary release to explain the disappearing dynos. Error: ${error}` + ) + } + + if (nextRelease) { + if (nextRelease.description === 'Enable allow-multiple-sni-endpoints feature') { + // Track dynos for the next release instead + release = nextRelease + releaseId = nextRelease.id + + console.warn('Switching to monitor secondary release...') + + // Allow the loop to repeat to fetch the dynos for the secondary release + } else { + // Otherwise, assume another release replaced this one but it + // PROBABLY would've succeeded...? + newDynos.forEach((dyno) => { + dyno.state = 'up' + }) + } + } + // else just keep monitoring and hope for the best + } + + console.log( + `Dyno states: ${JSON.stringify(newDynos.map((dyno) => dyno.state))} (after ${Math.round( + (Date.now() - dynoBootStartTime) / 1000 + )} seconds)` + ) + } catch (error) { + // Allow for a few bad responses from the Heroku API + if (error.statusCode === 404 || error.statusCode === 429) { + dynoAcceptableErrorCount += 1 + if (dynoAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) { + continue + } + } + throw new Error(`Failed to find dynos for this release. Error: ${error}`) + } + } + + const crashedDynos = newDynos.filter((dyno) => ['crashed', 'restarting'].includes(dyno.state)) + const runningDynos = newDynos.filter((dyno) => dyno.state === 'up') + + // If any dynos crashed on start-up, fail the deployment + if (crashedDynos.length > 0) { + const errorMessage = `At least ${crashedDynos.length} Heroku dyno(s) crashed on start-up!` + + console.error(errorMessage) + + // Attempt to dump some of the Heroku log here for debugging + try { + const logSession = await heroku.post(`/apps/${appName}/log-sessions`, { + body: { + dyno: crashedDynos[0].name, + lines: HEROKU_LOG_LINES_TO_SHOW, + tail: false, + }, + }) + + logUrl = logSession.logplex_url + + const logText = await got(logUrl).text() + console.error( + `Here are the last ${HEROKU_LOG_LINES_TO_SHOW} lines of the Heroku log:\n\n${logText}` + ) + } catch (error) { + // Don't fail because of this error + console.error(`Failed to retrieve the Heroku logs for the crashed dynos. Error: ${error}`) + } + + throw new Error(errorMessage) + } + + console.log( + `At least ${runningDynos.length} Heroku dyno(s) are ready after ${Math.round( + (Date.now() - dynoBootStartTime) / 1000 + )} seconds.` + ) + + // Send a series of requests to trigger the server warmup routines + console.log('🚀 Deployment status: in_progress - Triggering server warmup routines...') + + const warmupStartTime = Date.now() + console.log(`Making warmup requests to: ${homepageUrl}`) + try { + await got(homepageUrl, { + timeout: 10000, // Maximum 10 second timeout per request + retry: 7, // About 2 minutes 7 seconds of delay, plus active request time for 8 requests + hooks: { + beforeRetry: [ + (options, error = {}, retryCount = '?') => { + const statusCode = error.statusCode || (error.response || {}).statusCode || -1 + console.log( + `Retrying after warmup request attempt #${retryCount} (${statusCode}) after ${Math.round( + (Date.now() - warmupStartTime) / 1000 + )} seconds...` + ) + }, + ], + }, + }) + console.log( + `Warmup requests passed after ${Math.round((Date.now() - warmupStartTime) / 1000)} seconds` + ) + } catch (error) { + throw new Error( + `Warmup requests failed after ${Math.round( + (Date.now() - warmupStartTime) / 1000 + )} seconds. Error: ${error}` + ) + } + + // Report success! + const successMessage = `Deployment succeeded after ${Math.round( + (Date.now() - startTime) / 1000 + )} seconds.` + console.log(successMessage) + + await octokit.repos.createDeploymentStatus({ + owner, + repo, + deployment_id: deploymentId, + state: 'success', + description: successMessage, + ...(logUrl && { log_url: logUrl }), + environment_url: homepageUrl, + // The 'ant-man' preview is required for `state` values of 'inactive', as well as + // the use of the `log_url`, `environment_url`, and `auto_inactive` parameters. + // The 'flash' preview is required for `state` values of 'in_progress' and 'queued'. + mediaType: { + previews: ['ant-man', 'flash'], + }, + }) + + console.log(`🚀 Deployment status: success - ${successMessage}`) + console.log(`Visit the newly deployed app at: ${homepageUrl}`) + } catch (error) { + // Report failure! + const failureMessage = `Deployment failed after ${Math.round( + (Date.now() - startTime) / 1000 + )} seconds. See logs for more information.` + console.error(failureMessage) + + try { + if (deploymentId) { + await octokit.repos.createDeploymentStatus({ + owner, + repo, + deployment_id: deploymentId, + state: 'error', + description: failureMessage, + ...(logUrl && { log_url: logUrl }), + environment_url: homepageUrl, + // The 'ant-man' preview is required for `state` values of 'inactive', as well as + // the use of the `log_url`, `environment_url`, and `auto_inactive` parameters. + // The 'flash' preview is required for `state` values of 'in_progress' and 'queued'. + mediaType: { + previews: ['ant-man', 'flash'], + }, + }) + + console.log( + `🚀 Deployment status: error - ${failureMessage}` + (logUrl ? ` Logs: ${logUrl}` : '') + ) + } + } catch (error) { + console.error(`Failed to finalize GitHub DeploymentStatus as a failure. Error: ${error}`) + } + + // Re-throw the error to bubble up + throw error + } +} + +async function getTarballUrl({ octokit, owner, repo, sha }) { + // Get a URL for the tarballed source code bundle + const { + headers: { location: tarballUrl }, + } = await octokit.repos.downloadTarballArchive({ + owner, + repo, + ref: sha, + // Override the underlying `node-fetch` module's `redirect` option + // configuration to prevent automatically following redirects. + request: { + redirect: 'manual', + }, + }) + return tarballUrl +} diff --git a/script/remove-stale-staging-apps.js b/script/remove-stale-staging-apps.js index 44cd11d2cecb..a70431f112f9 100755 --- a/script/remove-stale-staging-apps.js +++ b/script/remove-stale-staging-apps.js @@ -41,7 +41,7 @@ async function main() { .orderBy('name') .value() - const prInfoMatch = /^(?:gha-)?(?docs(?:-internal)?)-(?\d+)--.*$/ + const prInfoMatch = /^(?:gha-|ghd-)?(?docs(?:-internal)?)-(?\d+)--.*$/ const appsPlusPullIds = apps.map((app) => { const match = prInfoMatch.exec(app.name)