From 0e6b6606d1e29c5449157c0bfe82d5c0681f36e6 Mon Sep 17 00:00:00 2001 From: Vlad Vitan <23100181+vlasebian@users.noreply.github.com> Date: Tue, 21 Jan 2025 14:54:18 +0200 Subject: [PATCH] doc: Update helm chart ingress controller setup info (#1386) * doc: Update helm chart ingress controller setup info * doc: Add sample ingress controllers page * doc: Update helm chart docs * doc: Add redis version requirement detail for the Helm chart * doc: Update location for sample ingress controllers config --- .../generic/configuration/_index.md | 90 +++++++- .../generic/install-charts/_index.md | 2 +- .../generic/prerequisites/_index.md | 203 +++++++++--------- .../sample-ingress-controllers.md | 183 ++++++++++++++++ .../generic/troubleshooting/_index.md | 4 - .../layouts/shortcodes/ttigpro.html | 1 + 6 files changed, 368 insertions(+), 115 deletions(-) create mode 100644 doc/content/enterprise/kubernetes/generic/prerequisites/sample-ingress-controllers.md create mode 100644 doc/themes/the-things-stack/layouts/shortcodes/ttigpro.html diff --git a/doc/content/enterprise/kubernetes/generic/configuration/_index.md b/doc/content/enterprise/kubernetes/generic/configuration/_index.md index 2eed8e66f2..760947c492 100644 --- a/doc/content/enterprise/kubernetes/generic/configuration/_index.md +++ b/doc/content/enterprise/kubernetes/generic/configuration/_index.md @@ -13,7 +13,7 @@ aliases: -The following is a list of mandatory minimum fields. For a full list of possible values, check the `values.yaml` file at the root of the {{% tts %}} Helm chart. +The following is a list of mandatory minimum fields. For a full list of possible values, check the `values.yaml` file at the root of the {{% tts %}} Helm chart. You can find it on [artifacthub.io](https://artifacthub.io/packages/helm/thethingsindustries/lorawan-stack-helm-chart?modal=values) as well. {{< warning >}} Some values in this file are secrets. Make sure to check this file into a secure repository.{{}} @@ -63,9 +63,9 @@ global: oauth: clientSecret: # See preparation section. ingress: - traefik: - tls: - secretName: # Secret Name containing the TLS Certificates for the Domain. + controller: # Ingress controller class name. + tls: + secretName: # Secret Name containing the TLS Certificates for the Domain. tenancy: adminKey: # See preparation section. interop: @@ -94,3 +94,85 @@ dcs: blob: bucket: # End Device Claiming Server bucket from "Section 4. Blob Storage" ``` + +## {{% ttigpro %}} configuration + +The Helm chart does not support {{% ttigpro %}} by default. To enable it in the Helm chart, The Things Gateway Controller must be enabled and the ingress controller of the Kubernetes cluster must have mTLS configured. + +To set up the gateway controller, the following fields must be filled in `values.yaml`: + +**Field name** |**Description** +----------------------------|---------------------------------------------------------------- +`global.ttgc.enabled` | The Things Gateway Controller enable flag. Set this to `true`. +`global.ttgc.domain` | Domain name of the cluster. Should be the same as `global.domain`. +`global.ttgc.tls.secretName`| Kubernetes Secret name containing the TLS. Should be the same as `global.ingress.tls.secretName`. +`global.ttgc.address` | (Optional) The URL of the gateway controller. It not specified, it defaults to `gc.thethings.industries:443`. + +{{% tts %}} verifies the identity of each connected TTIGPro gateway for security reasons, therefore mutual TLS is necessary to be configured in the ingress controller. mTLS configuration depends on the chosen ingress controller and is left to the operator of the Kubernetes cluster. {{% tts %}} recognizes the following client certificate header names: +- `X-Forwarded-Client-Cert` +- `X-Forwarded-Tls-Client-Cert` + +As an example, the steps to configure Traefik to support mTLS are the following: + +1. Create the Traefik middleware that passes the TLS client certificates in the request headers. + + - Paste the k8s middleware manifest into a yaml file (make sure to set the correct namespace): + ```yaml + # traefik-passtlsclientcert.yaml + apiVersion: traefik.io/v1alpha1 + kind: Middleware + metadata: + name: traefik-passtlsclientcert + namespace: + spec: + passTLSClientCert: + pem: true + ``` + + - Apply the k8s manifest to the cluster: + + ```bash + kubectl apply -f traefik-passtlsclientcert.yaml + ``` + +For more info check the [Traefik docs on the PassTLSClientCert middleware](https://doc.traefik.io/traefik/middlewares/http/passtlsclientcert/). + +2. Configure the default TLS options for Traefik. + + - Paste the k8s middleware manifest into a yaml file (make sure to set the correct namespace): + ```yaml + # traefik-tlsoption.yaml + apiVersion: traefik.io/v1alpha1 + kind: TLSOption + metadata: + name: default + namespace: + spec: + clientAuth: + clientAuthType: RequestClientCert + ``` + + - Apply the k8s manifest to the cluster: + + ```bash + kubectl apply -f traefik-passtlsclientcert.yaml + ``` + +{{< note "{{% tts %}} Helm chart uses wildcard domains in ingress routes. This is necessary for multi-tenant deployments as we don't know the names of the tenants in advance. Traefik does not support TLS options for wildcard domains, because it maps the TLS options based solely on the host name (the `Host` part of the ingress rule) and it needs a concrete domain to match ([check out more in the Traefik docs](https://doc.traefik.io/traefik/v2.3/routing/routers/#options)). To go around this limitation, a default TLS option can be used which is the fallback for any unmatched host." />}} + +For more info check the [Traefik docs on TLS options](https://doc.traefik.io/traefik/https/tls/#tls-options). + +3. Set the protocol annotations for {{% ttigpro %}}, middleware annotations and serviceAnnotations in `values.yaml` +(in addition to the existing annotations): + +```yaml +annotations: + ttigw: + traefik.ingress.kubernetes.io/router.entrypoints: ttigw,ttigwsecure + traefik.ingress.kubernetes.io/router.middlewares: traefik-passtlsclientcert@kubernetescrd + traefik.ingress.kubernetes.io/router.tls: "true" +serviceAnnotations: + traefik.ingress.kubernetes.io/service.serversscheme: h2c +``` + +4. Install (or upgrade) the helm chart. diff --git a/doc/content/enterprise/kubernetes/generic/install-charts/_index.md b/doc/content/enterprise/kubernetes/generic/install-charts/_index.md index a247d7e10d..1997434c04 100644 --- a/doc/content/enterprise/kubernetes/generic/install-charts/_index.md +++ b/doc/content/enterprise/kubernetes/generic/install-charts/_index.md @@ -9,7 +9,7 @@ aliases: ] --- -{{% tts %}} Helm charts are packaged and distributed as [OCI packages](https://helm.sh/docs/topics/registries/) and are published to [Docker Hub](https://hub.docker.com/r/thethingsindustries/lorawan-stack-helm-chart). +{{% tts %}} Helm charts are packaged and distributed as [OCI packages](https://helm.sh/docs/topics/registries/) and are published to [Docker Hub](https://hub.docker.com/r/thethingsindustries/lorawan-stack-helm-chart). You can also find information about the Helm chart on [artifacthub.io](https://artifacthub.io/packages/helm/thethingsindustries/lorawan-stack-helm-chart). Use the following steps to install the chart. diff --git a/doc/content/enterprise/kubernetes/generic/prerequisites/_index.md b/doc/content/enterprise/kubernetes/generic/prerequisites/_index.md index baf1c1012c..ab493a57bb 100644 --- a/doc/content/enterprise/kubernetes/generic/prerequisites/_index.md +++ b/doc/content/enterprise/kubernetes/generic/prerequisites/_index.md @@ -37,12 +37,12 @@ Please [contact our sales team](mailto:sales@thethingsindustries.com) for access {{% tts %}} on Kubernetes requires the following infrastructural services to run. -1. A Kubernetes cluster. -2. PostgreSQL compatible database. -3. Redis compatible database. +1. A Kubernetes cluster +2. PostgreSQL compatible database +3. Redis compatible database (Redis 6.2 or above is required) 4. Blob Storage -5. Traefik Proxy to handle the ingress routes. -6. TLS Certificates. +5. An ingress controller to handle the ingress routes +6. TLS Certificates 7. (Optional) TimescaleDB 8. (Optional) Metrics Server @@ -111,113 +111,102 @@ $ sudo chown -R 886:886 ##### Disabling Blob Storage -{{% tts %}} Helm Charts by default expects a blob storage configured but it is possible to use {{% tts %}} without it. You can disable the usage of blob by setting `global.interop.configSource` and `global.blob.provider` values to an empty string `""`. +{{% tts %}} Helm Chart by default expects a blob storage configured but it is possible to use {{% tts %}} without it. You can disable the usage of blob by setting `global.interop.configSource` and `global.blob.provider` values to an empty string `""`. -#### 5. Traefik Proxy - -The Things Stack Helm Charts currently only supports the [Traefik](https://traefik.io/traefik/) proxy out of the box to load balance incoming traffic. - -To use a custom proxy/load balancer, set `global.ingress.traefik.enabled` to `false`. - -When Traefik is disabled, the ports and routes necessary for The Things Stack should be mapped manually by the operator. -Check the `ingress-routes.yaml` files for each component and adapt it to your proxy. Setting up and maintaining a custom proxy is out of the scope of this document. - -If Traefik is not used, skip ahead to [setup TLS certificates]({{< ref "enterprise/kubernetes/generic/prerequisites#6-tls-certificates" >}}). - -The simplest way to install Traefik in the kubernetes cluster is use the [official Helm charts](https://artifacthub.io/packages/helm/traefik/traefik). - -The Things Stack ingress routes need to be mapped to the Traefik entry points. This can be done during installation. - -Save the following as a YAML file (example `traefik.values.yaml`) and use that as the values file for Helm. +#### 5. An ingress controller +An ingress controller is needed to route the incoming traffic. Specify the ingress controller by setting the `global.ingress.controller` to the class name of the ingress controller deployed in the cluster. For TLS, make sure to set the `global.ingress.controller.tls.secretName`. The secret has to be accessible from the namespace where the {{% tts %}} Helm Chart is deployed. These ports are needed by {{% tts %}} and must be exposed: + ```yaml -deployment: - replicas: 2 -ports: - web: # NOTE: This name is predefined in traefik. - protocol: TCP - port: 1885 - expose: true - exposedPort: 80 - redirectTo: websecure - websecure: # NOTE: This name is predefined in traefik. - protocol: TCP - port: 8885 - expose: true - exposedPort: 443 - grpc: - protocol: TCP - port: 1884 - expose: true - exposedPort: 1884 - grpcsecure: - protocol: TCP - port: 8884 - expose: true - exposedPort: 8884 - # Gateway Connectivity - gtwmqttv2: - protocol: TCP - port: 1881 - expose: true - exposedPort: 1881 - gtwmqttv2secure: - protocol: TCP - port: 8881 - expose: true - exposedPort: 8881 - gtwmqttv3: - protocol: TCP - port: 1882 - expose: true - exposedPort: 1882 - gtwmqttv3secure: - protocol: TCP - port: 8882 - expose: true - exposedPort: 8882 - lbs: - protocol: TCP - port: 1887 - expose: true - exposedPort: 1887 - lbssecure: - protocol: TCP - port: 8887 - expose: true - exposedPort: 8887 - # Application MQTT - appmqtt: - protocol: TCP - port: 1883 - expose: true - exposedPort: 1883 - appmqttsecure: - protocol: TCP - port: 8883 - expose: true - exposedPort: 8883 - udp: - protocol: UDP - port: 1700 - expose: true - exposedPort: 1700 - # Interoperability. This part is optional. Only enable it if interoperability is needed. - interop: - protocol: TCP - # Note: Change this to 1886 if using `server-only` mode. - port: 8886 - expose: true - exposedPort: 8886 +web: + protocol: TCP + port: 1885 + exposedPort: 80 +websecure: + protocol: TCP + port: 8885 + exposedPort: 443 +grpc: + protocol: TCP + port: 1884 + exposedPort: 1884 +grpcsecure: + protocol: TCP + port: 8884 + exposedPort: 8884 +# Gateway Connectivity +gtwmqttv2: + protocol: TCP + port: 1881 + exposedPort: 1881 +gtwmqttv2secure: + protocol: TCP + port: 8881 + exposedPort: 8881 +gtwmqttv3: + protocol: TCP + port: 1882 + exposedPort: 1882 +gtwmqttv3secure: + protocol: TCP + port: 8882 + exposedPort: 8882 +lbs: + protocol: TCP + port: 1887 + exposedPort: 1887 +lbssecure: + protocol: TCP + port: 8887 + exposedPort: 8887 +# Application MQTT +appmqtt: + protocol: TCP + port: 1883 + exposedPort: 1883 +appmqttsecure: + protocol: TCP + port: 8883 + exposedPort: 8883 +# The Things Indoor Gateway Pro +ttigw: + protocol: "TCP" + port: 1889 + exposedPort: 1889 +ttigwsecure: + protocol: "TCP" + port: 8889 + exposedPort: 8889 +# Interoperability. This part is optional. Only enable it if interoperability is needed. +interop: + protocol: TCP + # Note: Change this to 1886 if using `server-only` mode. + port: 8886 + exposedPort: 8886 ``` -##### Custom Resource Definitions (CRDs) - -Traefik requires the installation of multiple CRDs (Custom Resource Definitions) to run. +In case annotations are needed for certain protocols or for the {{% tts %}} services, these can be specified under `global.ingress.annotations` and `global.ingress.serviceAnnotations`. E.g. Traefik annotations can be specified as: +```yaml +ingress: + controller: "traefik" + tls: + secretName: "ingress-tls-cert" + annotations: + grpc: + traefik.ingress.kubernetes.io/router.entrypoints: grpcsecure + traefik.ingress.kubernetes.io/router.tls: "true" + http: + traefik.ingress.kubernetes.io/router.entrypoints: websecure + semtechws: + traefik.ingress.kubernetes.io/router.entrypoints: semtechwssecure, semtechws + traefik.ingress.kubernetes.io/router.tls: "true" + serviceAnnotations: + traefik.ingress.kubernetes.io/service.serversscheme: h2c +``` -This can be done using `kubectl`. Choose the appropriate CRD file for the version of Traefik that you are using. +Examples of ingress controllers configurations can be found [here](https://www.thethingsindustries.com/docs/the-things-stack/host/kubernetes/generic/prerequisites/sample-ingress-controllers/). -If you are using the official [Traefik Helm Chart](https://github.com/traefik/traefik-helm-chart), the CRDs are installed automatically for you. +{{< note "{{% tts %}} Helm chart uses Kubernetes ingress rules for routing requests to the components of {{% tts %}}. This allows the users of {{% tts %}} Helm chart to configure an ingress controller of their choice. However, Kubernetes ingress routes support only L7 traffic (HTTP/gRPC). For this reason, UDP Packet Forwarder for gateways is not supported in the Helm chart for now." />}} #### 6. TLS Certificates @@ -233,10 +222,12 @@ Consequently, the TLS certificates used should cover `domain` and one of the fol - `*.domain` - `.domain` -The Things Stack expects the name of this secret to be set in the value `global.ingress.tls.secretName`. +The Things Stack expects the name of this secret to be set in the value `global.ingress.tls.secretName`. In case the gateway controller is enabled in the Helm chart, the name of the secret must be set in the value of `global.ttgc.tls.secretName` as well. The process of provisioning and maintenance of the certificate secret is left to the operator. +If the cluster has a custom CA, it must be specified in `global.tls.rootCA`. The certificate must be specified as a base64 encoded x509 certificate. Multiple certificates must be separated by a new line. + #### 7. (Optional) TimescaleDB Both {{% tts %}} [Storage Integration](https://www.thethingsindustries.com/docs/integrations/storage/) and {{% tts %}} [Network Operations Center](https://www.thethingsindustries.com/docs/concepts/architecture/components/network-operations-center/#accessing-network-operations-center) require a TimescaleDB instance. diff --git a/doc/content/enterprise/kubernetes/generic/prerequisites/sample-ingress-controllers.md b/doc/content/enterprise/kubernetes/generic/prerequisites/sample-ingress-controllers.md new file mode 100644 index 0000000000..36880eec98 --- /dev/null +++ b/doc/content/enterprise/kubernetes/generic/prerequisites/sample-ingress-controllers.md @@ -0,0 +1,183 @@ +--- +title: "Sample ingress controllers" +description: "" +weight: 1 +aliases: [ + /getting-started/kubernetes/self-managed/prerequisites/sample-ingress-controllers, + /the-things-stack/host/kubernetes/generic/prerequisites/sample-ingress-controllers, + ] +--- + +The following are examples of ingress controllers for {{% tts %}} deployment on Kubernetes. + + + +## Traefik + +Example of a Traefik configuration provided through the values of an Traefik Helm chart. More info about the Helm chart +can be found [here](https://github.com/traefik/traefik-helm-chart). + +```yaml +deployment: + kind: "Deployment" + replicas: 1 +ingressRoute: + dashboard: + enabled: false +additionalArguments: +- "--entrypoints.udp.udp.timeout=90s" +ports: + web: + protocol: "TCP" + port: 1885 + expose: + default: true + exposedPort: 80 + redirectTo: + port: "websecure" + websecure: + protocol: "TCP" + port: 8885 + expose: + default: true + exposedPort: 443 + traefik: + protocol: "TCP" + port: 9000 + expose: + default: false + grpc: + protocol: "TCP" + port: 1884 + expose: + default: true + exposedPort: 1884 + grpcsecure: + protocol: "TCP" + port: 8884 + expose: + default: true + exposedPort: 8884 + gtwmqttv2: + protocol: "TCP" + port: 1881 + expose: + default: true + exposedPort: 1881 + gtwmqttv2secure: + protocol: "TCP" + port: 8881 + expose: + default: true + exposedPort: 8881 + gtwmqttv3: + protocol: "TCP" + port: 1882 + expose: + default: true + exposedPort: 1882 + gtwmqttv3secure: + protocol: "TCP" + port: 8882 + expose: + default: true + exposedPort: 8882 + semtechws: + protocol: "TCP" + port: 1887 + expose: + default: true + exposedPort: 1887 + semtechwssecure: + protocol: "TCP" + port: 8887 + expose: + default: true + exposedPort: 8887 + appmqtt: + protocol: "TCP" + port: 1883 + expose: + default: true + exposedPort: 1883 + appmqttsecure: + protocol: "TCP" + port: 8883 + expose: + default: true + exposedPort: 8883 + ttigw: + protocol: "TCP" + port: 1889 + expose: + default: true + exposedPort: 1889 + ttigwsecure: + protocol: "TCP" + port: 8889 + expose: + default: true + exposedPort: 8889 + interop: + protocol: "TCP" + port: 8886 + expose: + default: true + exposedPort: 8886 +``` + +## Ingress NGINX + +Example of an Ingress NGINX configuration provided through the values of an Ingress NGINX Helm chart. More info about the +Helm chart can be found [here](https://artifacthub.io/packages/helm/bitnami/nginx). + +```yaml +fullnameOverride: "nginx" +namespaceOverride: "ingress-nginx" +kind: Deployment +replicaCount: '1' +config: + # redirect port 80 to 443 for HTTP to HTTPS. + ssl-redirect: "true" + upstream-keepalive-timeout: '90s' +service: + ports: + http: 80 + https: 443 + extraPorts: + - name: semtechws + port: 1887 + targetPort: 1887 + nodePorts: + tcp: + # http + "80": "1885" + # https + "443": "8885" + # grpc + "1884": "1884" + # grpcsecure + "8844": "8884" + # gtwmqttv2 + "1881": "1881" + # gtwmqttv2secure + "8881": "8881" + # gtwmqttv3 + "1882": "1882" + # gtwmqttv3secure + "8882": "8882" + # semtechws + "1887": "1887" + # semtechwssecure + "8887": "8887" + # appmqtt + "1883": "1883" + # appmqttsecure + "8883": "8883" + # interop + "8886": "8886" + # ttigw + "1889": "1889" + # ttigwsecure + "8889": "8889" +``` diff --git a/doc/content/enterprise/kubernetes/generic/troubleshooting/_index.md b/doc/content/enterprise/kubernetes/generic/troubleshooting/_index.md index 9a8075e7c6..bf349628d7 100644 --- a/doc/content/enterprise/kubernetes/generic/troubleshooting/_index.md +++ b/doc/content/enterprise/kubernetes/generic/troubleshooting/_index.md @@ -34,7 +34,3 @@ For this error, make sure that the value set in `ingress.traefik.tls.secretName` ## pkg/util/store:driver (driver error) {{% tts %}} runs Kubernetes jobs to initialize and migrate Postgres. This error can occur if the {{% tts %}} is accessed either before these jobs are run or if the jobs failed to execute. Check the status of the jobs for more details on what went wrong. - -## Gateways don't work when targeting the UDP port 1700 - -Ensure that UDP port (default 1700) is exposed outside the Kubernetes cluster via your chosen proxy or load balancer. diff --git a/doc/themes/the-things-stack/layouts/shortcodes/ttigpro.html b/doc/themes/the-things-stack/layouts/shortcodes/ttigpro.html new file mode 100644 index 0000000000..7e55ce4f84 --- /dev/null +++ b/doc/themes/the-things-stack/layouts/shortcodes/ttigpro.html @@ -0,0 +1 @@ +{{- printf "The Things Indoor Gateway Pro" -}}