From ded1b3509e6b81fb2266130058ebccd4d3787893 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Mon, 1 May 2023 10:25:25 -0700 Subject: [PATCH] Added section on embedded HA to Architecture page (#122) * Add diagram for etcd HA * Add tabs, remove old version gates Signed-off-by: Derek Nola * Clean up language around fixed registration address Signed-off-by: Derek Nola --- docs/architecture/architecture.md | 45 +- docs/datastore/ha-embedded.md | 12 +- docs/datastore/ha.md | 6 +- docs/installation/network-options.md | 2 +- .../img/k3s-architecture-ha-embedded-dark.svg | 1444 +++++++++++++++++ static/img/k3s-architecture-ha-embedded.svg | 1434 ++++++++++++++++ ... => k3s-architecture-ha-external-dark.svg} | 151 +- ...r.svg => k3s-architecture-ha-external.svg} | 207 +-- .../k3s-architecture-single-server-dark.svg | 190 +-- 9 files changed, 3157 insertions(+), 334 deletions(-) create mode 100644 static/img/k3s-architecture-ha-embedded-dark.svg create mode 100644 static/img/k3s-architecture-ha-embedded.svg rename static/img/{k3s-architecture-ha-server-dark.svg => k3s-architecture-ha-external-dark.svg} (93%) rename static/img/{k3s-architecture-ha-server.svg => k3s-architecture-ha-external.svg} (88%) diff --git a/docs/architecture/architecture.md b/docs/architecture/architecture.md index 7ac375829..a812aa05d 100644 --- a/docs/architecture/architecture.md +++ b/docs/architecture/architecture.md @@ -5,6 +5,9 @@ weight: 1 import ThemedImage from '@theme/ThemedImage'; import useBaseUrl from '@docusaurus/useBaseUrl'; +import TabItem from '@theme/TabItem'; +import Tabs from '@theme/Tabs'; + This page describes the architecture of a high-availability K3s server cluster and how it differs from a single-node server cluster. @@ -30,25 +33,43 @@ In this configuration, each agent node is registered to the same server node. A }} /> +### High-Availability K3s -### High-Availability K3s Server with an External DB +Single server clusters can meet a variety of use cases, but for environments where uptime of the Kubernetes control plane is critical, you can run K3s in an HA configuration. An HA K3s cluster comprises: -Single server clusters can meet a variety of use cases, but for environments where uptime of the Kubernetes control plane is critical, you can run K3s in an HA configuration. An HA K3s cluster is comprised of: + + + +* Three or more **server nodes** that will serve the Kubernetes API and run other control plane services +* An **embedded etcd datastore** (as opposed to the embedded SQLite datastore used in single-server setups) -* Two or more **server nodes** that will serve the Kubernetes API and run other control plane services -* An **external datastore** (as opposed to the embedded SQLite datastore used in single-server setups) + light: useBaseUrl('/img/k3s-architecture-ha-embedded.svg'), + dark: useBaseUrl('/img/k3s-architecture-ha-embedded-dark.svg'), +}} /> + + + + +* Two or more **server nodes** that will serve the Kubernetes API and run other control plane services +* An **external datastore** (such as MySQL, PostgreSQL, or etcd) + + + + + ### Fixed Registration Address for Agent Nodes -In the high-availability server configuration, each node must also register with the Kubernetes API by using a fixed registration address, as shown in the diagram below. +In the high-availability server configuration, each node can also register with the Kubernetes API by using a fixed registration address, as shown in the diagram below. After registration, the agent nodes establish a connection directly to one of the server nodes. @@ -62,14 +83,10 @@ After registration, the agent nodes establish a connection directly to one of th ### How Agent Node Registration Works -Agent nodes are registered with a websocket connection initiated by the `k3s agent` process, and the connection is maintained by a client-side load balancer running as part of the agent process. This load-balancer maintains stable connections to all servers in the cluster, providing a connection to the apiserver that tolerates outages of individual servers. +Agent nodes are registered with a websocket connection initiated by the `k3s agent` process, and the connection is maintained by a client-side load balancer running as part of the agent process. Initially, the agent connects to the supervisor (and kube-apiserver) via the local load-balancer on port 6443. The load-balancer maintains a list of available endpoints to connect to. The default (and initially only) endpoint is seeded by the hostname from the `--server` address. Once it connects to the cluster, the agent retrieves a list of kube-apiserver addresses from the Kubernetes service endpoint list in the default namespace. Those endpoints are added to the load balancer, which then maintains stable connections to all servers in the cluster, providing a connection to the kube-apiserver that tolerates outages of individual servers. Agents will register with the server using the node cluster secret along with a randomly generated password for the node, stored at `/etc/rancher/node/password`. The server will store the passwords for individual nodes as Kubernetes secrets, and any subsequent attempts must use the same password. Node password secrets are stored in the `kube-system` namespace with names using the template `.node-password.k3s`. This is done to protect the integrity of node IDs. If the `/etc/rancher/node` directory of an agent is removed, or you wish to rejoin a node using an existing name, the node should be deleted from the cluster. This will clean up both the old node entry, and the node password secret, and allow the node to (re)join the cluster. -:::note -Prior to K3s v1.20.2 servers stored passwords on disk at `/var/lib/rancher/k3s/server/cred/node-passwd`. -::: - If you frequently reuse hostnames, but are unable to remove the node password secrets, a unique node ID can be automatically appended to the hostname by launching K3s servers or agents using the `--with-node-id` flag. When enabled, the node ID is also stored in `/etc/rancher/node/`. diff --git a/docs/datastore/ha-embedded.md b/docs/datastore/ha-embedded.md index 39eff7d92..1f7e88a51 100644 --- a/docs/datastore/ha-embedded.md +++ b/docs/datastore/ha-embedded.md @@ -3,15 +3,6 @@ title: "High Availability Embedded etcd" weight: 40 --- -:::info Version Gate -Full support as of [v1.19.5+k3s1](https://github.com/k3s-io/k3s/releases/tag/v1.19.5%2Bk3s1) -Experimental support as of [v1.19.1+k3s1](https://github.com/k3s-io/k3s/releases/tag/v1.19.1%2Bk3s1) -::: - -:::note Notice: Deprecated Dqlite -Embedded etcd replaced experimental Dqlite in the K3s v1.19.1 release. This is a breaking change. Please note that upgrades from experimental Dqlite to embedded etcd are not supported. If you attempt an upgrade it will not succeed and data will be lost. -::: - :::caution Embedded etcd (HA) may have performance issues on slower disks such as Raspberry Pis running with SD cards. ::: @@ -36,9 +27,10 @@ $ kubectl get nodes NAME STATUS ROLES AGE VERSION server1 Ready control-plane,etcd,master 28m vX.Y.Z server2 Ready control-plane,etcd,master 13m vX.Y.Z +server3 Ready control-plane,etcd,master 10m vX.Y.Z ``` -Now you have a highly available control plane. Any successfully clustered servers can be used in the `--server` argument to join additional server and worker nodes. Joining additional worker nodes to the cluster follows the same procedure as a single server cluster. +Now you have a highly available control plane. Any successfully clustered servers can be used in the `--server` argument to join additional server and agent nodes. Joining additional agent nodes to the cluster follows the same procedure as a single server cluster. There are a few config flags that must be the same in all server nodes: diff --git a/docs/datastore/ha.md b/docs/datastore/ha.md index 9a0ad2983..621d33d43 100644 --- a/docs/datastore/ha.md +++ b/docs/datastore/ha.md @@ -3,8 +3,6 @@ title: High Availability External DB weight: 30 --- -> **Note:** Official support for installing Rancher on a Kubernetes cluster was introduced in our v1.0.0 release. - This section describes how to install a high-availability K3s cluster with an external database. Single server clusters can meet a variety of use cases, but for environments where uptime of the Kubernetes control plane is critical, you can run K3s in an HA configuration. An HA K3s cluster is comprised of: @@ -14,7 +12,7 @@ Single server clusters can meet a variety of use cases, but for environments whe * An **external datastore** (as opposed to the embedded SQLite datastore used in single-server setups) * A **fixed registration address** that is placed in front of the server nodes to allow agent nodes to register with the cluster -For more details on how these components work together, refer to the [architecture section.](../architecture/architecture.md#high-availability-k3s-server-with-an-external-db) +For more details on how these components work together, refer to the [architecture section.](../architecture/architecture.md#high-availability-k3s) Agents register through the fixed registration address, but after registration they establish a connection directly to one of the server nodes. This is a websocket connection initiated by the `k3s agent` process, it is maintained by a client-side load balancer running as part of the agent process. @@ -50,7 +48,7 @@ By default, server nodes will be schedulable and thus your workloads can get lau Once you've launched the `k3s server` process on all server nodes, ensure that the cluster has come up properly with `k3s kubectl get nodes`. You should see your server nodes in the Ready state. -### 3. Configure the Fixed Registration Address +### 3. Optional: Configure a Fixed Registration Address Agent nodes need a URL to register against. This can be the IP or hostname of any of the server nodes, but in many cases those may change over time. For example, if you are running your cluster in a cloud that supports scaling groups, you may scale the server node group up and down over time, causing nodes to be created and destroyed and thus having different IPs from the initial set of server nodes. Therefore, you should have a stable endpoint in front of the server nodes that will not change over time. This endpoint can be set up using any number approaches, such as: diff --git a/docs/installation/network-options.md b/docs/installation/network-options.md index 674694ca4..c50516ad7 100644 --- a/docs/installation/network-options.md +++ b/docs/installation/network-options.md @@ -178,7 +178,7 @@ and on agents: --node-external-ip= ``` -where `SERVER_EXTERNAL_IP` is the IP through which we can reach the server node and `AGENT_EXTERNAL_IP` is the IP through which we can reach the agent/worker node. Note that the `K3S_URL` config parameter in the agent/worker should use the `SERVER_EXTERNAL_IP` to be able to connect to it. Remember to check the [Networking Requirements](../installation/requirements.md#networking) and allow access to the listed ports on both internal and external addresses. +where `SERVER_EXTERNAL_IP` is the IP through which we can reach the server node and `AGENT_EXTERNAL_IP` is the IP through which we can reach the agent node. Note that the `K3S_URL` config parameter in the agent should use the `SERVER_EXTERNAL_IP` to be able to connect to it. Remember to check the [Networking Requirements](../installation/requirements.md#networking) and allow access to the listed ports on both internal and external addresses. Both `SERVER_EXTERNAL_IP` and `AGENT_EXTERNAL_IP` must have connectivity between them and are normally public IPs. diff --git a/static/img/k3s-architecture-ha-embedded-dark.svg b/static/img/k3s-architecture-ha-embedded-dark.svg new file mode 100644 index 000000000..56fdad099 --- /dev/null +++ b/static/img/k3s-architecture-ha-embedded-dark.svg @@ -0,0 +1,1444 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/static/img/k3s-architecture-ha-embedded.svg b/static/img/k3s-architecture-ha-embedded.svg new file mode 100644 index 000000000..8f87e89e4 --- /dev/null +++ b/static/img/k3s-architecture-ha-embedded.svg @@ -0,0 +1,1434 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/static/img/k3s-architecture-ha-server-dark.svg b/static/img/k3s-architecture-ha-external-dark.svg similarity index 93% rename from static/img/k3s-architecture-ha-server-dark.svg rename to static/img/k3s-architecture-ha-external-dark.svg index 3d50cdb9d..bb9746d4d 100644 --- a/static/img/k3s-architecture-ha-server-dark.svg +++ b/static/img/k3s-architecture-ha-external-dark.svg @@ -6,8 +6,8 @@ viewBox="0 0 6756.000000 3183.000000" preserveAspectRatio="xMidYMid meet" id="svg494" - sodipodi:docname="k3s-architecture-ha-server-dark.svg" - inkscape:version="1.2.2 (b0a8486541, 2022-12-01, custom)" + sodipodi:docname="k3s-architecture-ha-external-dark.svg" + inkscape:version="1.2.2 (732a01da63, 2022-12-09, custom)" xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" xmlns="http://www.w3.org/2000/svg" @@ -34,9 +34,9 @@ inkscape:deskcolor="#d1d1d1" inkscape:document-units="pt" showgrid="false" - inkscape:zoom="0.25" - inkscape:cx="4742" - inkscape:cy="2306" + inkscape:zoom="0.70710678" + inkscape:cx="4096.2696" + inkscape:cy="709.2281" inkscape:window-width="2247" inkscape:window-height="1376" inkscape:window-x="1193" @@ -196,54 +196,59 @@ d="m 26181,25388 c 0,-7 37,-116 83,-243 l 82,-230 102,-3 c 56,-1 102,-1 102,1 0,2 38,111 85,242 47,131 85,240 85,242 0,1 -38,3 -83,3 h -84 l -12,-42 c -7,-24 -31,-101 -53,-173 l -40,-130 -53,170 -52,170 -81,3 c -62,2 -82,0 -81,-10 z" id="path138" style="fill-opacity:1" /> - - - - - - - - - - - - + + + + + + + + + + + + + + + style="fill:#ffe8a4;fill-opacity:1" /> + style="fill:#ffe8a4;fill-opacity:1" /> + style="fill:#ffe8a4;fill-opacity:1" /> + + style="fill:#ffe8a4;fill-opacity:1" /> + style="fill:#ffe8a4;fill-opacity:1" /> + style="fill:#ffe8a4;fill-opacity:1" /> + style="fill:#ffe8a4;fill-opacity:1" /> + style="fill:#ffe8a4;fill-opacity:1" /> + style="fill:#ffe8a4;fill-opacity:1" /> + inkscape:window-maximized="1" + inkscape:current-layer="g492" /> + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + inkscape:current-layer="g438" />