From d419bba2090c27a748dae4e4f888adf350d285c2 Mon Sep 17 00:00:00 2001 From: Tyler Chong Date: Thu, 14 Nov 2024 06:59:22 -1000 Subject: [PATCH] known issue: ghe-repl-promote when the primary is down (#53098) Co-authored-by: Vanessa --- data/release-notes/enterprise-server/3-13/0-rc1.yml | 4 ++++ data/release-notes/enterprise-server/3-13/0.yml | 6 +++++- data/release-notes/enterprise-server/3-13/2.yml | 4 ++++ data/release-notes/enterprise-server/3-13/3.yml | 5 +++++ data/release-notes/enterprise-server/3-13/4.yml | 5 +++++ data/release-notes/enterprise-server/3-13/5.yml | 4 ++++ data/release-notes/enterprise-server/3-13/6.yml | 4 ++++ data/release-notes/enterprise-server/3-14/0-rc1.yml | 4 ++++ data/release-notes/enterprise-server/3-14/0.yml | 4 ++++ data/release-notes/enterprise-server/3-14/1.yml | 4 ++++ data/release-notes/enterprise-server/3-14/2.yml | 4 ++++ data/release-notes/enterprise-server/3-14/3.yml | 6 +++++- data/release-notes/enterprise-server/3-15/0-rc1.yml | 3 +++ .../2024-11-ghe-repl-promote-primary-down.md | 8 ++++++++ 14 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 data/reusables/release-notes/2024-11-ghe-repl-promote-primary-down.md diff --git a/data/release-notes/enterprise-server/3-13/0-rc1.yml b/data/release-notes/enterprise-server/3-13/0-rc1.yml index 334fea83a8bb..b6e1a422a478 100644 --- a/data/release-notes/enterprise-server/3-13/0-rc1.yml +++ b/data/release-notes/enterprise-server/3-13/0-rc1.yml @@ -174,6 +174,10 @@ sections: When enabling log forwarding, specific service logs, including babeld, are duplicated. For more information, see "[AUTOTITLE](/admin/monitoring-activity-in-your-enterprise/exploring-user-activity-in-your-enterprise/log-forwarding#enabling-log-forwarding)." - | {% data reusables.release-notes.2024-06-possible-frontend-5-minute-outage-during-hotpatch-upgrade %} [Updated: 2024-06-17] + - | + {% data reusables.release-notes.2024-11-ghe-repl-promote-primary-down %} + + [Updated: 2024-11-13] deprecations: # https://github.com/github/releases/issues/2732 diff --git a/data/release-notes/enterprise-server/3-13/0.yml b/data/release-notes/enterprise-server/3-13/0.yml index a2ffba974f15..0192372af1c8 100644 --- a/data/release-notes/enterprise-server/3-13/0.yml +++ b/data/release-notes/enterprise-server/3-13/0.yml @@ -187,7 +187,11 @@ sections: Following an upgrade, Elasticsearch search migrations are sometimes incorrectly reported as failing in the audit log, even though the migrations completed successfully. [Updated: 2024-08-02] - | Images embedded in wiki pages may stop rendering shortly after being published. [Updated: 2024-10-16] - + - | + {% data reusables.release-notes.2024-11-ghe-repl-promote-primary-down %} + + [Updated: 2024-11-13] + deprecations: # https://github.com/github/releases/issues/2732 - | diff --git a/data/release-notes/enterprise-server/3-13/2.yml b/data/release-notes/enterprise-server/3-13/2.yml index 1e91efe3f296..ba60d65a9df2 100644 --- a/data/release-notes/enterprise-server/3-13/2.yml +++ b/data/release-notes/enterprise-server/3-13/2.yml @@ -171,3 +171,7 @@ sections: Following an upgrade, Elasticsearch search migrations are sometimes incorrectly reported as failing in the audit log, even though the migrations completed successfully. [Updated: 2024-08-02] - | Images embedded in wiki pages may stop rendering shortly after being published. [Updated: 2024-10-16] + - | + {% data reusables.release-notes.2024-11-ghe-repl-promote-primary-down %} + + [Updated: 2024-11-13] diff --git a/data/release-notes/enterprise-server/3-13/3.yml b/data/release-notes/enterprise-server/3-13/3.yml index 13b84b216d20..8b6039626f07 100644 --- a/data/release-notes/enterprise-server/3-13/3.yml +++ b/data/release-notes/enterprise-server/3-13/3.yml @@ -127,6 +127,11 @@ sections: {% data reusables.release-notes.2024-08-resolvconf-wont-start %} [Updated: 2024-08-26] + - | + {% data reusables.release-notes.2024-11-ghe-repl-promote-primary-down %} + + [Updated: 2024-11-13] + errata: - | These release notes previously indicated as a known issue that on GitHub Enterprise Server 3.13.3 when log forwarding is enabled, some forwarded log entries may be duplicated. diff --git a/data/release-notes/enterprise-server/3-13/4.yml b/data/release-notes/enterprise-server/3-13/4.yml index de07283f7eb1..1b1f6918d64c 100644 --- a/data/release-notes/enterprise-server/3-13/4.yml +++ b/data/release-notes/enterprise-server/3-13/4.yml @@ -76,5 +76,10 @@ sections: Services may respond with a `503` status due to an out of date `haproxy` configuration. This can usually be resolved with a `ghe-config-apply` run. - | For customers using Secret Scanning, internal jobs were created and not worked that could contribute to performance issues. + - | + {% data reusables.release-notes.2024-11-ghe-repl-promote-primary-down %} + + [Updated: 2024-11-13] + errata: - 'The "[Known issues](/admin/release-notes#3.13.4-known-issues)" section previously indicated that `Instance setup in AWS with IMDSv2 enforced fails if no public IP is present` is still an issue. The issue is resolved and is documented in the "[Bug fixes](/admin/release-notes#3.13.4-bugs)" section. [Updated: 2024-09-30]' diff --git a/data/release-notes/enterprise-server/3-13/5.yml b/data/release-notes/enterprise-server/3-13/5.yml index bc11898f705f..43b769f6ef40 100644 --- a/data/release-notes/enterprise-server/3-13/5.yml +++ b/data/release-notes/enterprise-server/3-13/5.yml @@ -56,3 +56,7 @@ sections: When restoring from a backup snapshot, a large number of `mapper_parsing_exception` errors may be displayed. - | Services may respond with a `503` status due to an out of date `haproxy` configuration. This can usually be resolved with a `ghe-config-apply` run. + - | + {% data reusables.release-notes.2024-11-ghe-repl-promote-primary-down %} + + [Updated: 2024-11-13] diff --git a/data/release-notes/enterprise-server/3-13/6.yml b/data/release-notes/enterprise-server/3-13/6.yml index e4b29dbdea16..fea517a4ce16 100644 --- a/data/release-notes/enterprise-server/3-13/6.yml +++ b/data/release-notes/enterprise-server/3-13/6.yml @@ -62,3 +62,7 @@ sections: Services may respond with a `503` status due to an out of date `haproxy` configuration. This can usually be resolved with a `ghe-config-apply` run. - | Customers doing feature version upgrade to 3.13.6 may experience issues with database migrations due to data issues during database conversions. [Added: 2024-11-08] + - | + {% data reusables.release-notes.2024-11-ghe-repl-promote-primary-down %} + + [Updated: 2024-11-13] diff --git a/data/release-notes/enterprise-server/3-14/0-rc1.yml b/data/release-notes/enterprise-server/3-14/0-rc1.yml index b9b246f563c4..ab55b972bb66 100644 --- a/data/release-notes/enterprise-server/3-14/0-rc1.yml +++ b/data/release-notes/enterprise-server/3-14/0-rc1.yml @@ -216,6 +216,10 @@ sections: In the header bar displayed to site administrators, some icons are not available. - | When restoring from a backup snapshot, a large number of `mapper_parsing_exception` errors may be displayed. + - | + {% data reusables.release-notes.2024-11-ghe-repl-promote-primary-down %} + + [Updated: 2024-11-13] deprecations: - | diff --git a/data/release-notes/enterprise-server/3-14/0.yml b/data/release-notes/enterprise-server/3-14/0.yml index f7e1af5654c2..d2b4433e2900 100644 --- a/data/release-notes/enterprise-server/3-14/0.yml +++ b/data/release-notes/enterprise-server/3-14/0.yml @@ -217,6 +217,10 @@ sections: Following an upgrade, Elasticsearch search migrations are sometimes incorrectly reported as failing in the audit log, even though the migrations completed successfully. [Updated: 2024-09-27] - | Images embedded in wiki pages may stop rendering shortly after being published. [Updated: 2024-10-16] + - | + {% data reusables.release-notes.2024-11-ghe-repl-promote-primary-down %} + + [Updated: 2024-11-13] deprecations: - | diff --git a/data/release-notes/enterprise-server/3-14/1.yml b/data/release-notes/enterprise-server/3-14/1.yml index 37c23240f47b..ed9461e56fb7 100644 --- a/data/release-notes/enterprise-server/3-14/1.yml +++ b/data/release-notes/enterprise-server/3-14/1.yml @@ -76,3 +76,7 @@ sections: Services may respond with a `503` status due to an out of date `haproxy` configuration. This can usually be resolved with a `ghe-config-apply` run. - | Images embedded in wiki pages may stop rendering shortly after being published. [Updated: 2024-10-16] + - | + {% data reusables.release-notes.2024-11-ghe-repl-promote-primary-down %} + + [Updated: 2024-11-13] diff --git a/data/release-notes/enterprise-server/3-14/2.yml b/data/release-notes/enterprise-server/3-14/2.yml index cd75c747023d..8d211f405120 100644 --- a/data/release-notes/enterprise-server/3-14/2.yml +++ b/data/release-notes/enterprise-server/3-14/2.yml @@ -78,3 +78,7 @@ sections: Services may respond with a `503` status due to an out of date `haproxy` configuration. This can usually be resolved with a `ghe-config-apply` run. - | Images embedded in wiki pages may stop rendering shortly after being published. [Updated: 2024-10-16] + - | + {% data reusables.release-notes.2024-11-ghe-repl-promote-primary-down %} + + [Updated: 2024-11-13] diff --git a/data/release-notes/enterprise-server/3-14/3.yml b/data/release-notes/enterprise-server/3-14/3.yml index cbdcf5a787d8..169b86946d4f 100644 --- a/data/release-notes/enterprise-server/3-14/3.yml +++ b/data/release-notes/enterprise-server/3-14/3.yml @@ -7,7 +7,7 @@ sections: Packages have been updated to the latest security version. - | **HIGH**: An attacker could bypass SAML single sign-on (SSO) authentication with the optional encrypted assertions feature, allowing unauthorized provisioning of users and access to the instance, by exploiting an improper verification of cryptographic signatures vulnerability in GitHub Enterprise Server. This is a follow up fix for [CVE-2024-9487](https://www.cve.org/cverecord?id=CVE-2024-9487) to further harden the encrypted assertions feature against this type of attack. Please note that encrypted assertions are not enabled by default. Instances not utilizing SAML SSO, or utilizing SAML SSO authentication without encrypted assertions, are not impacted. Additionally, an attacker would require direct network access as well as a signed SAML response or metadata document to exploit this vulnerability. - - | + - | **HIGH**: An attacker with Enterprise Administrator access to the GitHub Enterprise Server instance could escalate privileges to SSH root access. This is achieved by exploiting the pre-receive hook environment to bypass symlink checks in the `ghe-firejail` path and execute malicious scripts. GitHub has requested CVE ID [CVE-2024-10007](https://www.cve.org/cverecord?id=CVE-2024-10007) for this vulnerability, which was reported via the [GitHub Bug Bounty program](https://bounty.github.com/). [Updated: 2024-11-07] bugs: - | @@ -76,3 +76,7 @@ sections: Services may respond with a `503` status due to an out of date `haproxy` configuration. This can usually be resolved with a `ghe-config-apply` run. - | Customers doing feature version upgrade to 3.14.3 may experience issues with database migrations due to data issues during database conversions. [Added: 2024-11-08] + - | + {% data reusables.release-notes.2024-11-ghe-repl-promote-primary-down %} + + [Updated: 2024-11-13] diff --git a/data/release-notes/enterprise-server/3-15/0-rc1.yml b/data/release-notes/enterprise-server/3-15/0-rc1.yml index bfd4c4129222..3e5f33629004 100644 --- a/data/release-notes/enterprise-server/3-15/0-rc1.yml +++ b/data/release-notes/enterprise-server/3-15/0-rc1.yml @@ -206,6 +206,9 @@ sections: Services may respond with a `503` status due to an out of date `haproxy` configuration. This can usually be resolved with a `ghe-config-apply` run. - | Customers doing feature version upgrade to 3.14.3 may experience issues with database migrations due to data issues during database conversions. + - | + {% data reusables.release-notes.2024-11-ghe-repl-promote-primary-down %} + [Updated: 2024-11-13] closing_down: diff --git a/data/reusables/release-notes/2024-11-ghe-repl-promote-primary-down.md b/data/reusables/release-notes/2024-11-ghe-repl-promote-primary-down.md new file mode 100644 index 000000000000..4952ec3a3529 --- /dev/null +++ b/data/reusables/release-notes/2024-11-ghe-repl-promote-primary-down.md @@ -0,0 +1,8 @@ +When operating in a high availability configuration, running `ghe-repl-promote` on a replica node may fail if the original primary cannot be reached by the replica node. This is because the `ghe-repl-promote` script attempts to decommission all Elasticsearch nodes other than the promoted node, however these requests are made to the original primary node which is no longer reachable. + The error message will be similar to: + +```shell +Maintenance mode has been enabled for active replica +{"message": "No server is currently available to service your request. Sorry about that. Please try resubmitting your request and contact your local GitHub Enterprise site administrator if the problem persists."} +jq: error (at :3): Cannot index string with string "node" +```