From a8e1c2f7ef5c110c08bef14d1357848ce69a713f Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Mon, 7 Oct 2024 17:39:01 +0100 Subject: [PATCH 1/2] Fixes https://github.com/redpanda-data/documentation-private/issues/2757 --- .../manage/partials/remote-read-replicas.adoc | 137 +++++++--- modules/manage/partials/tiered-storage.adoc | 247 ++++++++---------- 2 files changed, 220 insertions(+), 164 deletions(-) diff --git a/modules/manage/partials/remote-read-replicas.adoc b/modules/manage/partials/remote-read-replicas.adoc index c6292a9cb..9ebbae114 100644 --- a/modules/manage/partials/remote-read-replicas.adoc +++ b/modules/manage/partials/remote-read-replicas.adoc @@ -3,6 +3,8 @@ include::shared:partial$enterprise-license.adoc[] ==== +:env-kubernetes: true + ifdef::env-kubernetes[] :tiered-storage-link: manage:kubernetes/storage/tiered-storage/k-tiered-storage.adoc :data-archiving-link: manage:kubernetes/storage/tiered-storage/k-tiered-storage.adoc#data-archiving @@ -111,6 +113,9 @@ Amazon S3:: You can configure access to Amazon S3 with either an IAM role attached to the instance or with access keys. +[discrete] +=== Use IAM roles + To configure access to an S3 bucket with an IAM role: . Configure an xref:manage:security/iam-roles.adoc#configuring-iam-roles[IAM role] with read permissions for the S3 bucket. @@ -152,6 +157,9 @@ Replace the following placeholders: + - ``: The region of your S3 bucket. +[discrete] +=== Use access keys + To configure access to an S3 bucket with access keys instead of an IAM role: . Grant a user the following permissions to read objects on the bucket to be used with the cluster (or on all buckets): @@ -159,7 +167,23 @@ To configure access to an S3 bucket with access keys instead of an IAM role: - `GetObject` - `ListBucket` -. Copy the access key and secret key for the `cloud_storage_access_key` and `cloud_storage_secret_key` cluster properties. +. Create a Secret in which to store the access key and secret key. ++ +[source,yaml] +---- +apiVersion: v1 +kind: Secret +metadata: + name: storage-secrets + namespace: +type: Opaque +data: + access-key: + secret-key: +---- ++ +- Replace `` with your base64-encoded access key. +- Replace `` with your base64-encoded secret key. . Override the following required cluster properties in the Helm chart: + [tabs] @@ -171,11 +195,16 @@ To configure access to an S3 bucket with access keys instead of an IAM role: ---- storage: tiered: + credentialsSecretRef: + accessKey: + name: storage-secrets + key: access-key + secretKey: + name: storage-secrets + key: secret-key config: cloud_storage_enabled: true cloud_storage_credentials_source: config_file - cloud_storage_access_key: - cloud_storage_secret_key: cloud_storage_region: cloud_storage_bucket: "none" ---- @@ -189,19 +218,17 @@ helm upgrade --install redpanda redpanda/redpanda --namespace --crea ```bash helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ --set storage.tiered.config.cloud_storage_enabled=true \ + --set storage.tiered.credentialsSecretRef.accessKey.name=storage-secrets \ + --set storage.tiered.credentialsSecretRef.accessKey.key=access-key \ + --set storage.tiered.credentialsSecretRef.secretKey.name=storage-secrets \ + --set storage.tiered.credentialsSecretRef.secretKey.key=secret-key \ --set storage.tiered.config.cloud_storage_credentials_source=config_file \ - --set storage.tiered.config.cloud_storage_access_key= \ - --set storage.tiered.config.cloud_storage_secret_key= \ --set storage.tiered.config.cloud_storage_region= \ --set storage.tiered.config.cloud_storage_bucket="none" ``` ==== + -Replace the following placeholders: -+ -- ``: The access key for your S3 bucket. -- ``: The secret key for your S3 bucket. -- ``: The region of your S3 bucket. +Replace `` with the region of your S3 bucket. -- Google Cloud Storage:: @@ -210,8 +237,11 @@ Google Cloud Storage:: You can configure access to Google Cloud Storage with either an IAM role attached to the instance or with access keys. -- To configure access to Google Cloud Storage with an IAM role, override the following required cluster properties in the Helm chart: -+ +[discrete] +=== Use IAM roles + +To configure access to Google Cloud Storage with an IAM role, override the following required cluster properties in the Helm chart: + [tabs] ==== --values:: @@ -242,10 +272,33 @@ helm upgrade --install redpanda redpanda/redpanda --namespace --crea --set storage.tiered.config.cloud_storage_bucket="none" ``` ==== -+ + Replace `` with the region of your bucket. -- To configure access to Google Cloud Storage with access keys instead of an IAM role, override the following required cluster properties in the Helm chart: +[discrete] +=== Use access keys + +To configure access to Google Cloud Storage with access keys instead of an IAM role: + +. Create a Secret in which to store the access key and secret key. ++ +[source,yaml] +---- +apiVersion: v1 +kind: Secret +metadata: + name: storage-secrets + namespace: +type: Opaque +data: + access-key: + secret-key: +---- ++ +- Replace `` with your base64-encoded access key. +- Replace `` with your base64-encoded secret key. + +. Override the following required cluster properties in the Helm chart: + [tabs] ==== @@ -256,12 +309,17 @@ Replace `` with the region of your bucket. ---- storage: tiered: + credentialsSecretRef: + accessKey: + name: storage-secrets + key: access-key + secretKey: + name: storage-secrets + key: secret-key config: cloud_storage_enabled: true cloud_storage_credentials_source: config_file cloud_storage_api_endpoint: storage.googleapis.com - cloud_storage_access_key: - cloud_storage_secret_key: cloud_storage_region: cloud_storage_bucket: "none" ---- @@ -275,30 +333,44 @@ helm upgrade --install redpanda redpanda/redpanda --namespace --crea ```bash helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ --set storage.tiered.config.cloud_storage_enabled=true \ + --set storage.tiered.credentialsSecretRef.accessKey.name=storage-secrets \ + --set storage.tiered.credentialsSecretRef.accessKey.key=access-key \ + --set storage.tiered.credentialsSecretRef.secretKey.name=storage-secrets \ + --set storage.tiered.credentialsSecretRef.secretKey.key=secret-key \ --set storage.tiered.config.cloud_storage_credentials_source=config_file \ --set storage.tiered.config.cloud_storage_api_endpoint=storage.googleapis.com \ - --set storage.tiered.config.cloud_storage_access_key= \ - --set storage.tiered.config.cloud_storage_secret_key= \ --set storage.tiered.config.cloud_storage_region= \ --set storage.tiered.config.cloud_storage_bucket="none" ``` ==== + -Replace the following placeholders: -+ -- ``: The access key for your bucket. -- ``: The secret key for your bucket. -- ``: The region of your bucket. +Replace `` with the region of your bucket. -- Azure Blob Storage:: + -- -To configure access to Azure Blob Storage, override the following required cluster properties in the Helm chart: +To configure access to Azure Blob Storage: -Replace the following placeholders: +. Create a Secret in which to store the access key. ++ +[source,yaml] +---- +apiVersion: v1 +kind: Secret +metadata: + name: storage-secrets + namespace: +type: Opaque +data: + access-key: +---- ++ +- Replace `` with your base64-encoded access key. +. Override the following required cluster properties in the Helm chart: ++ [tabs] ==== --values:: @@ -308,9 +380,13 @@ Replace the following placeholders: ---- storage: tiered: + credentialsSecretRef: + secretKey: + configurationKey: cloud_storage_azure_shared_key + name: storage-secrets + key: access-key config: cloud_storage_enabled: true - cloud_storage_azure_shared_key: cloud_storage_azure_storage_account: cloud_storage_azure_container: "none" ---- @@ -324,16 +400,15 @@ helm upgrade --install redpanda redpanda/redpanda --namespace --crea ```bash helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ --set storage.tiered.config.cloud_storage_enabled=true \ - --set storage.tiered.config.cloud_storage_azure_shared_key= \ + --set storage.tiered.credentialsSecretRef.secretKey.configurationKey=cloud_storage_azure_shared_key \ + --set storage.tiered.credentialsSecretRef.secretKey.name=storage-secrets \ + --set storage.tiered.credentialsSecretRef.secretKey.key=access-key \ --set storage.tiered.config.cloud_storage_azure_storage_account= \ --set storage.tiered.config.cloud_storage_azure_container="none" ``` ==== -Replace the following placeholders: - -- ``: The access key for your Azure account. -- ``: The name of your Azure account. +Replace `` with the name of your Azure account. -- ====== endif::[] diff --git a/modules/manage/partials/tiered-storage.adoc b/modules/manage/partials/tiered-storage.adoc index 8213a656b..8cb037749 100644 --- a/modules/manage/partials/tiered-storage.adoc +++ b/modules/manage/partials/tiered-storage.adoc @@ -58,6 +58,8 @@ TIP: If deploying Redpanda on an AWS Auto-Scaling group (ASG), keep in mind that You can configure access to Amazon S3 with either an IAM role attached to the instance or with access keys. +===== Use IAM roles + To configure access to an S3 bucket with an IAM role: . Configure an xref:manage:security/iam-roles.adoc#configuring-iam-roles[IAM role]. @@ -136,6 +138,8 @@ Replace the following placeholders: + CAUTION: Do not set an object storage property to an empty string `""` or to `null` as a way to reset it to its default value. +===== Use access keys + To configure access to an S3 bucket with access keys instead of an IAM role: . Grant an IAM user the following permissions to read and create objects in your buckets: @@ -145,8 +149,25 @@ To configure access to an S3 bucket with access keys instead of an IAM role: - `PutObjectTagging` - `ListBucket` -. Copy the access key and secret key for the `storage.tiered.config.cloud_storage_access_key` and `storage.tiered.config.cloud_storage_secret_key` cluster properties. -. Override the following required cluster properties in the Helm chart: +. Make a note of the access key and secret key. +. Create a Secret in which to store the access key and secret key. ++ +[source,yaml] +---- +apiVersion: v1 +kind: Secret +metadata: + name: storage-secrets + namespace: +type: Opaque +data: + access-key: + secret-key: +---- ++ +- Replace `` with your base64-encoded access key. +- Replace `` with your base64-encoded secret key. +. Override the following required cluster properties: + [tabs] ====== @@ -165,11 +186,16 @@ spec: clusterSpec: storage: tiered: + credentialsSecretRef: + accessKey: + name: storage-secrets + key: access-key + secretKey: + name: storage-secrets + key: secret-key config: cloud_storage_enabled: "true" cloud_storage_credentials_source: config_file - cloud_storage_access_key: - cloud_storage_secret_key: cloud_storage_region: cloud_storage_bucket: ---- @@ -190,11 +216,16 @@ Helm:: ---- storage: tiered: + credentialsSecretRef: + accessKey: + name: storage-secrets + key: access-key + secretKey: + name: storage-secrets + key: secret-key config: cloud_storage_enabled: true cloud_storage_credentials_source: config_file - cloud_storage_access_key: - cloud_storage_secret_key: cloud_storage_region: cloud_storage_bucket: ---- @@ -208,9 +239,11 @@ helm upgrade --install redpanda redpanda/redpanda --namespace --crea ```bash helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ --set storage.tiered.config.cloud_storage_enabled=true \ + --set storage.tiered.credentialsSecretRef.accessKey.name=storage-secrets \ + --set storage.tiered.credentialsSecretRef.accessKey.key=access-key \ + --set storage.tiered.credentialsSecretRef.secretKey.name=storage-secrets \ + --set storage.tiered.credentialsSecretRef.secretKey.key=secret-key \ --set storage.tiered.config.cloud_storage_credentials_source=config_file \ - --set storage.tiered.config.cloud_storage_access_key= \ - --set storage.tiered.config.cloud_storage_secret_key= \ --set storage.tiered.config.cloud_storage_region= \ --set storage.tiered.config.cloud_storage_bucket= ``` @@ -220,46 +253,6 @@ helm upgrade --install redpanda redpanda/redpanda --namespace --crea + Replace the following placeholders: + -- ``: The access key for your S3 bucket. -+ -.Manage sensitive data with Secrets -[%collapsible] -==== -To avoid exposing sensitive data in your configuration files, you can also use a Kubernetes Secret. For example: - -. Create the Secret: -+ -[source,yaml] ----- -apiVersion: v1 -kind: Secret -metadata: - name: storage-secrets -type: Opaque -data: - access-key: - secret-key: ----- -+ -- Replace `` with your base64-encoded access key. -- Replace `` with your base64-encoded secret key. - -. Update your Redpanda configuration to include the secret reference for the shared key: -+ -[source,yaml] ----- -storage: - tiered: - credentialsSecretRef: - accessKey: - name: storage-secrets - key: access-key - secretKey: - name: storage-secrets - key: secret-key ----- -==== -- ``: The secret key for your S3 bucket. - ``: The region of your S3 bucket. - ``: The name of your S3 bucket. + @@ -269,6 +262,8 @@ CAUTION: Do not set an object storage property to an empty string `""` or to `nu You can configure access to Google Cloud Storage with either an IAM role attached to the instance or with access keys. +===== Use IAM roles + To configure access to Google Cloud Storage with an IAM role: . Configure an xref:manage:security/iam-roles.adoc#configuring-iam-roles[IAM role]. @@ -348,12 +343,32 @@ Replace the following placeholders: CAUTION: Do not set an object storage property to an empty string `""` or to `null` as a way to reset it to its default value. +===== Use access keys + To configure access to Google Cloud Storage with access keys instead of an IAM role: . Choose a uniform access control when you create the bucket. . Use a Google-managed encryption key. . Set a https://cloud.google.com/storage/docs/migrating#defaultproj[default project^]. -. Create a service user with https://cloud.google.com/storage/docs/authentication/managing-hmackeys[HMAC keys^] and copy the access key and secret key for the `cloud_storage_access_key` and `cloud_storage_secret_key` properties. +. Create a service user with https://cloud.google.com/storage/docs/authentication/managing-hmackeys[HMAC keys^] and make a note of the access key and secret key. +. Make a note of the access key and secret key. +. Create a Secret in which to store the access key and secret key. ++ +[source,yaml] +---- +apiVersion: v1 +kind: Secret +metadata: + name: storage-secrets + namespace: +type: Opaque +data: + access-key: + secret-key: +---- ++ +- Replace `` with your base64-encoded access key. +- Replace `` with your base64-encoded secret key. . Override the following required cluster properties in the Helm chart: + [tabs] @@ -373,12 +388,17 @@ spec: clusterSpec: storage: tiered: + credentialsSecretRef: + accessKey: + name: storage-secrets + key: access-key + secretKey: + name: storage-secrets + key: secret-key config: cloud_storage_enabled: "true" cloud_storage_credentials_source: config_file cloud_storage_api_endpoint: storage.googleapis.com - cloud_storage_access_key: - cloud_storage_secret_key: cloud_storage_region: cloud_storage_bucket: ---- @@ -399,12 +419,17 @@ Helm:: ---- storage: tiered: + credentialsSecretRef: + accessKey: + name: storage-secrets + key: access-key + secretKey: + name: storage-secrets + key: secret-key config: cloud_storage_enabled: true cloud_storage_credentials_source: config_file cloud_storage_api_endpoint: storage.googleapis.com - cloud_storage_access_key: - cloud_storage_secret_key: cloud_storage_region: cloud_storage_bucket: ---- @@ -418,10 +443,12 @@ helm upgrade --install redpanda redpanda/redpanda --namespace --crea ```bash helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ --set storage.tiered.config.cloud_storage_enabled=true \ + --set storage.tiered.credentialsSecretRef.accessKey.name=storage-secrets \ + --set storage.tiered.credentialsSecretRef.accessKey.key=access-key \ + --set storage.tiered.credentialsSecretRef.secretKey.name=storage-secrets \ + --set storage.tiered.credentialsSecretRef.secretKey.key=secret-key \ --set storage.tiered.config.cloud_storage_credentials_source=config_file \ --set storage.tiered.config.cloud_storage_api_endpoint=storage.googleapis.com \ - --set storage.tiered.config.cloud_storage_access_key= \ - --set storage.tiered.config.cloud_storage_secret_key= \ --set storage.tiered.config.cloud_storage_region= \ --set storage.tiered.config.cloud_storage_bucket= ``` @@ -431,46 +458,6 @@ helm upgrade --install redpanda redpanda/redpanda --namespace --crea + Replace the following placeholders: + -- ``: The access key for your bucket. -+ -.Manage sensitive data with Secrets -[%collapsible] -==== -To avoid exposing sensitive data in your configuration files, you can also use a Kubernetes Secret. For example: - -. Create the Secret: -+ -[source,yaml] ----- -apiVersion: v1 -kind: Secret -metadata: - name: storage-secrets -type: Opaque -data: - access-key: - secret-key: ----- -+ -- Replace `` with your base64-encoded access key. -- Replace `` with your base64-encoded secret key. - -. Update your Redpanda configuration to include the secret reference for the shared key: -+ -[source,yaml] ----- -storage: - tiered: - credentialsSecretRef: - accessKey: - name: storage-secrets - key: access-key - secretKey: - name: storage-secrets - key: secret-key ----- -==== -- ``: The secret key for your bucket. - ``: The region of your bucket. - ``: The name of your bucket. + @@ -482,6 +469,8 @@ You can configure access to Azure Blob Storage with either account access keys o include::manage:partial$azure-blob-limitations.adoc[] +===== Use managed identities + To configure access to an Azure container with a managed identity in AKS: . Configure an xref:manage:security/iam-roles.adoc#configuring-iam-roles[Azure managed identity]. @@ -582,9 +571,26 @@ NOTE: The `serviceAccount` annotations and the `statefulset` Pod labels are esse CAUTION: Do not set an object storage property to an empty string `""` or to `null` as a way to reset it to its default value. +===== Use access keys + To configure access to ABS/ADLS with account access keys: . Get an account access key for the Azure container that Redpanda will run on. For information on how to view your account access keys, see the https://learn.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?toc=%2Fazure%2Fstorage%2Fblobs%2Ftoc.json&bc=%2Fazure%2Fstorage%2Fblobs%2Fbreadcrumb%2Ftoc.json&tabs=azure-portal#view-account-access-keys[Azure documentation^]. +. Create a Secret in which to store the access key. ++ +[source,yaml] +---- +apiVersion: v1 +kind: Secret +metadata: + name: storage-secrets + namespace: +type: Opaque +data: + access-key: +---- ++ +- Replace `` with your base64-encoded access key. . Override the following required cluster properties in the Helm chart: + [tabs] @@ -604,9 +610,13 @@ spec: clusterSpec: storage: tiered: + credentialsSecretRef: + secretKey: + configurationKey: cloud_storage_azure_shared_key + name: storage-secrets + key: access-key config: cloud_storage_enabled: "true" - cloud_storage_azure_shared_key: cloud_storage_azure_storage_account: cloud_storage_azure_container: ---- @@ -627,9 +637,13 @@ Helm:: ---- storage: tiered: + credentialsSecretRef: + secretKey: + configurationKey: cloud_storage_azure_shared_key + name: storage-secrets + key: access-key config: cloud_storage_enabled: true - cloud_storage_azure_shared_key: cloud_storage_azure_storage_account: cloud_storage_azure_container: ---- @@ -643,7 +657,9 @@ helm upgrade --install redpanda redpanda/redpanda --namespace --crea ```bash helm upgrade --install redpanda redpanda/redpanda --namespace --create-namespace \ --set storage.tiered.config.cloud_storage_enabled=true \ - --set storage.tiered.config.cloud_storage_azure_shared_key= \ + --set storage.tiered.credentialsSecretRef.secretKey.configurationKey=cloud_storage_azure_shared_key \ + --set storage.tiered.credentialsSecretRef.secretKey.name=storage-secrets \ + --set storage.tiered.credentialsSecretRef.secretKey.key=access-key \ --set storage.tiered.config.cloud_storage_azure_storage_account= \ --set storage.tiered.config.cloud_storage_azure_container= ``` @@ -653,41 +669,6 @@ helm upgrade --install redpanda redpanda/redpanda --namespace --crea + Replace the following placeholders: + -- ``: The access key for your Azure account. -+ -.Manage sensitive data with Secrets -[%collapsible] -==== -To avoid exposing sensitive data in your configuration files, you can also use a Kubernetes Secret. For example: - -. Create the Secret: -+ -[source,yaml] ----- -apiVersion: v1 -kind: Secret -metadata: - name: storage-secrets -type: Opaque -data: - secret-key: ----- -+ -Replace `` with your base64-encoded secret key. - -. Update your Redpanda configuration to include the secret reference for the secret key: -+ -[source,yaml] ----- -storage: - tiered: - credentialsSecretRef: - secretKey: - configurationKey: cloud_storage_azure_shared_key - name: storage-secrets - key: secret-key ----- -==== - ``: The name of your Azure account. - ``: The name of the Azure container in your Azure account. + From 32a6592c696cdc86ef1270bc9455c8f27009182d Mon Sep 17 00:00:00 2001 From: Jake Cahill <45230295+JakeSCahill@users.noreply.github.com> Date: Tue, 8 Oct 2024 12:35:49 +0100 Subject: [PATCH 2/2] Apply suggestions from code review --- modules/manage/partials/remote-read-replicas.adoc | 1 - modules/manage/partials/tiered-storage.adoc | 1 - 2 files changed, 2 deletions(-) diff --git a/modules/manage/partials/remote-read-replicas.adoc b/modules/manage/partials/remote-read-replicas.adoc index 9ebbae114..b99c95c15 100644 --- a/modules/manage/partials/remote-read-replicas.adoc +++ b/modules/manage/partials/remote-read-replicas.adoc @@ -3,7 +3,6 @@ include::shared:partial$enterprise-license.adoc[] ==== -:env-kubernetes: true ifdef::env-kubernetes[] :tiered-storage-link: manage:kubernetes/storage/tiered-storage/k-tiered-storage.adoc diff --git a/modules/manage/partials/tiered-storage.adoc b/modules/manage/partials/tiered-storage.adoc index 8cb037749..80697192b 100644 --- a/modules/manage/partials/tiered-storage.adoc +++ b/modules/manage/partials/tiered-storage.adoc @@ -351,7 +351,6 @@ To configure access to Google Cloud Storage with access keys instead of an IAM r . Use a Google-managed encryption key. . Set a https://cloud.google.com/storage/docs/migrating#defaultproj[default project^]. . Create a service user with https://cloud.google.com/storage/docs/authentication/managing-hmackeys[HMAC keys^] and make a note of the access key and secret key. -. Make a note of the access key and secret key. . Create a Secret in which to store the access key and secret key. + [source,yaml]