From bb0455962e1a25cc28a7b8d84f318a5b05831edc Mon Sep 17 00:00:00 2001 From: "James M. Greene" Date: Wed, 6 Oct 2021 12:57:30 -0500 Subject: [PATCH] Do not use `build.status` as a looping condition for Heroku deployment (#21909) * Do not use `build.status` of 'pending' as a looping condition for Heroku deployment * Don't wait for `appSetup.status` either * Fix incorrect Octokit method usage in local deploy script * Bump the number of allowable errors from 5 to 10 * More logging! * Add an environment variable for easily increasing the number of allowed Heroku failures per phase of polling --- .github/workflows/prod-build-deploy.yml | 1 + .github/workflows/staging-deploy-pr.yml | 1 + script/deploy.js | 6 +++--- script/deployment/deploy-to-production.js | 15 +++++++++++++-- script/deployment/deploy-to-staging.js | 21 ++++++++++++++++++--- 5 files changed, 36 insertions(+), 8 deletions(-) diff --git a/.github/workflows/prod-build-deploy.yml b/.github/workflows/prod-build-deploy.yml index e79204d34efc..c4230e96d47e 100644 --- a/.github/workflows/prod-build-deploy.yml +++ b/.github/workflows/prod-build-deploy.yml @@ -143,6 +143,7 @@ jobs: HYDRO_SECRET: ${{ secrets.HYDRO_SECRET }} SOURCE_BLOB_URL: ${{ steps.build-source.outputs.download_url }} DELAY_FOR_PREBOOT: 'true' + ALLOWED_POLLING_FAILURES_PER_PHASE: '15' with: script: | const { diff --git a/.github/workflows/staging-deploy-pr.yml b/.github/workflows/staging-deploy-pr.yml index ca3a18bbd1a8..099059c61c21 100644 --- a/.github/workflows/staging-deploy-pr.yml +++ b/.github/workflows/staging-deploy-pr.yml @@ -472,6 +472,7 @@ jobs: CONTEXT_NAME: ${{ env.CONTEXT_NAME }} ACTIONS_RUN_LOG: ${{ env.ACTIONS_RUN_LOG }} HEAD_SHA: ${{ needs.pr-metadata.outputs.head_sha }} + ALLOWED_POLLING_FAILURES_PER_PHASE: '15' with: script: | const { GITHUB_TOKEN, HEROKU_API_TOKEN } = process.env diff --git a/script/deploy.js b/script/deploy.js index 98eea665d93c..184785da6509 100755 --- a/script/deploy.js +++ b/script/deploy.js @@ -234,7 +234,7 @@ async function deployStaging({ owner, repo, pullNumber, forceRebuild = false, de pullRequest, }) } else { - await octokit.repos.createStatus({ + await octokit.repos.createCommitStatus({ owner, repo, sha: pullRequest.head.sha, @@ -249,7 +249,7 @@ async function deployStaging({ owner, repo, pullNumber, forceRebuild = false, de forceRebuild, }) - await octokit.repos.createStatus({ + await octokit.repos.createCommitStatus({ owner, repo, sha: pullRequest.head.sha, @@ -264,7 +264,7 @@ async function deployStaging({ owner, repo, pullNumber, forceRebuild = false, de console.error(error) if (!destroy) { - await octokit.repos.createStatus({ + await octokit.repos.createCommitStatus({ owner, repo, sha: pullRequest.head.sha, diff --git a/script/deployment/deploy-to-production.js b/script/deployment/deploy-to-production.js index 4014ed42262a..14d8990bd002 100755 --- a/script/deployment/deploy-to-production.js +++ b/script/deployment/deploy-to-production.js @@ -10,7 +10,8 @@ const DELAY_FOR_PREBOOT_SWAP = 135000 // 2:15 // Allow for a few 404 (Not Found), 429 (Too Many Requests), etc. responses from // the semi-unreliable Heroku API when we're polling for status updates -const ALLOWED_MISSING_RESPONSE_COUNT = 5 +const ALLOWED_MISSING_RESPONSE_COUNT = + parseInt(process.env.ALLOWED_POLLING_FAILURES_PER_PHASE, 10) || 10 const ALLOWABLE_ERROR_CODES = [404, 429, 500, 503] export default async function deployToProduction({ @@ -175,7 +176,7 @@ export default async function deployToProduction({ // Poll until the Build's status changes from "pending" to "succeeded" or "failed". let buildAcceptableErrorCount = 0 - while (!build || build.status === 'pending' || !build.release || !build.release.id) { + while (!build || !build.release || !build.release.id) { await sleep(SLEEP_INTERVAL) try { build = await heroku.get(`/apps/${appName}/builds/${buildId}`) @@ -184,6 +185,9 @@ export default async function deployToProduction({ if (isAllowableHerokuError(error)) { buildAcceptableErrorCount += 1 if (buildAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) { + console.warn( + `Ignoring allowable Heroku error #${buildAcceptableErrorCount}: ${error.statusCode}` + ) continue } } @@ -210,6 +214,7 @@ export default async function deployToProduction({ `Finished Heroku build after ${Math.round((Date.now() - buildStartTime) / 1000)} seconds.`, build ) + console.log('Heroku release detected', build.release) const releaseStartTime = Date.now() // Close enough... const releaseId = build.release.id @@ -237,6 +242,9 @@ export default async function deployToProduction({ if (isAllowableHerokuError(error)) { releaseAcceptableErrorCount += 1 if (releaseAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) { + console.warn( + `Ignoring allowable Heroku error #${releaseAcceptableErrorCount}: ${error.statusCode}` + ) continue } } @@ -296,6 +304,9 @@ export default async function deployToProduction({ if (isAllowableHerokuError(error)) { dynoAcceptableErrorCount += 1 if (dynoAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) { + console.warn( + `Ignoring allowable Heroku error #${dynoAcceptableErrorCount}: ${error.statusCode}` + ) continue } } diff --git a/script/deployment/deploy-to-staging.js b/script/deployment/deploy-to-staging.js index c1f2b79b8661..f2d200beece5 100644 --- a/script/deployment/deploy-to-staging.js +++ b/script/deployment/deploy-to-staging.js @@ -10,7 +10,8 @@ const HEROKU_LOG_LINES_TO_SHOW = 25 // Allow for a few 404 (Not Found), 429 (Too Many Requests), etc. responses from // the semi-unreliable Heroku API when we're polling for status updates -const ALLOWED_MISSING_RESPONSE_COUNT = 5 +const ALLOWED_MISSING_RESPONSE_COUNT = + parseInt(process.env.ALLOWED_POLLING_FAILURES_PER_PHASE, 10) || 10 const ALLOWABLE_ERROR_CODES = [404, 429, 500, 503] export default async function deployToStaging({ @@ -233,7 +234,7 @@ export default async function deployToStaging({ // A new Build is created as a by-product of creating an AppSetup. // Poll until there is a Build object attached to the AppSetup. let setupAcceptableErrorCount = 0 - while (!appSetup || appSetup.status === 'pending' || !build || !build.id) { + while (!appSetup || !build || !build.id) { await sleep(SLEEP_INTERVAL) try { appSetup = await heroku.get(`/app-setups/${appSetup.id}`) @@ -243,6 +244,9 @@ export default async function deployToStaging({ if (isAllowableHerokuError(error)) { setupAcceptableErrorCount += 1 if (setupAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) { + console.warn( + `Ignoring allowable Heroku error #${setupAcceptableErrorCount}: ${error.statusCode}` + ) continue } } @@ -272,6 +276,7 @@ See Heroku logs for more information:\n${logUrl}` ) } + console.log('Heroku AppSetup finished', appSetup) console.log('Heroku build detected', build) } else { // If the app does exist, just manually trigger a new build @@ -317,7 +322,7 @@ See Heroku logs for more information:\n${logUrl}` // Poll until the Build's status changes from "pending" to "succeeded" or "failed". let buildAcceptableErrorCount = 0 - while (!build || build.status === 'pending' || !build.release || !build.release.id) { + while (!build || !build.release || !build.release.id) { await sleep(SLEEP_INTERVAL) try { build = await heroku.get(`/apps/${appName}/builds/${buildId}`) @@ -326,6 +331,9 @@ See Heroku logs for more information:\n${logUrl}` if (isAllowableHerokuError(error)) { buildAcceptableErrorCount += 1 if (buildAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) { + console.warn( + `Ignoring allowable Heroku error #${buildAcceptableErrorCount}: ${error.statusCode}` + ) continue } } @@ -352,6 +360,7 @@ See Heroku logs for more information:\n${logUrl}` `Finished Heroku build after ${Math.round((Date.now() - buildStartTime) / 1000)} seconds.`, build ) + console.log('Heroku release detected', build.release) const releaseStartTime = Date.now() // Close enough... let releaseId = build.release.id @@ -379,6 +388,9 @@ See Heroku logs for more information:\n${logUrl}` if (isAllowableHerokuError(error)) { releaseAcceptableErrorCount += 1 if (releaseAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) { + console.warn( + `Ignoring allowable Heroku error #${releaseAcceptableErrorCount}: ${error.statusCode}` + ) continue } } @@ -485,6 +497,9 @@ See Heroku logs for more information:\n${logUrl}` if (isAllowableHerokuError(error)) { dynoAcceptableErrorCount += 1 if (dynoAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) { + console.warn( + `Ignoring allowable Heroku error #${dynoAcceptableErrorCount}: ${error.statusCode}` + ) continue } }