From e05154e997d4bd231fbd4458126b7beabaddb176 Mon Sep 17 00:00:00 2001 From: ncabatoff Date: Thu, 6 Feb 2025 16:20:00 -0500 Subject: [PATCH] Revert "Add docs for new iteration of ent operator import feature (#29406)" This reverts commit 9c0f2fbfe573f5c6fcea47f238661043b7009e6a. --- .../content/docs/commands/operator/import.mdx | 34 +---- website/content/docs/import/awssm.mdx | 39 ------ website/content/docs/import/azurekv.mdx | 49 ------- website/content/docs/import/gcpsm.mdx | 8 +- website/content/docs/import/index.mdx | 120 ++++++---------- website/content/docs/import/mappings.mdx | 132 ++++++++++++++++++ website/content/docs/import/vault.mdx | 38 ----- website/data/docs-nav-data.json | 18 +-- 8 files changed, 190 insertions(+), 248 deletions(-) delete mode 100644 website/content/docs/import/awssm.mdx delete mode 100644 website/content/docs/import/azurekv.mdx create mode 100644 website/content/docs/import/mappings.mdx delete mode 100644 website/content/docs/import/vault.mdx diff --git a/website/content/docs/commands/operator/import.mdx b/website/content/docs/commands/operator/import.mdx index 4901a90f10ae..8144b1c92713 100644 --- a/website/content/docs/commands/operator/import.mdx +++ b/website/content/docs/commands/operator/import.mdx @@ -54,48 +54,28 @@ Output: ## Configuration The `operator import` command uses a dedicated configuration file to specify the source, -destination, and mapping rules. To learn more about these types and secrets importing in +destination, and mapping rules. To learn more about these types and secrets importing in general, refer to the [Secrets Import documentation](/vault/docs/import). ```hcl source_gcp { - name = "my-src-1" - credentials_file = "/path/to/service-account-key.json" -} - -source_aws { - name = "my-src-2" - credentials_profile = "my-profile-name" -} - -source_azure { - name = "my-src-3" - # Use default credentials from doing an az login + name = "my-gcp-source-1" + credentials = "@/path/to/service-account-key.json" } destination_vault { name = "my-dest-1" address = "http://127.0.0.1:8200/" + token = "root" namespace = "ns-1" mount = "mount-1" } -mapping { +mapping_passthrough { name = "my-map-1" - source = "my-src-1" - destination = "my-dest-1" -} - -mapping { - name = "my-map-2" - source = "my-src-2" - destination = "my-dest-1" -} - -mapping { - name = "my-map-3" - source = "my-src-3" + source = "my-gcp-1" destination = "my-dest-1" + priority = 1 } ``` diff --git a/website/content/docs/import/awssm.mdx b/website/content/docs/import/awssm.mdx deleted file mode 100644 index 630fab1c998f..000000000000 --- a/website/content/docs/import/awssm.mdx +++ /dev/null @@ -1,39 +0,0 @@ ---- -layout: docs -page_title: AWS secret import source -description: The AWS Secret Manager source imports secrets from AWS to Vault. ---- - -# AWS secret import source - -Use the AWS source to import secret data from AWS Secret Manager into your Vault instance. - -## Argument reference - -Refer to the [HCL syntax](/vault/docs/import#hcl-syntax-1) for arguments common to all source types. - -## Additional arguments - -- `credentials_profile` `(string: "")` - The name of the profile in your credentials file to authenticate with. - If not set, Vault uses the default credential provider mechanisms. - -## Example - -Define and configure the `my-aws-source-1` AWS source: - -```hcl -source_aws { - name = "my-aws-source-1" -} -``` - -## Permissions - -To use AWS import, you must grant the associated AWS identity permissions to read secrets: - -```shell-session -"secretsmanager:DescribeSecret", -"secretsmanager:GetSecretValue", -"secretsmanager:BatchGetSecretValue", -"secretsmanager:ListSecrets", -``` diff --git a/website/content/docs/import/azurekv.mdx b/website/content/docs/import/azurekv.mdx deleted file mode 100644 index 86f534da86ad..000000000000 --- a/website/content/docs/import/azurekv.mdx +++ /dev/null @@ -1,49 +0,0 @@ ---- -layout: docs -page_title: Azure secret import source -description: The Azure KeyVault source imports secrets from Azure to Vault. ---- - -# Azure secret import source - -Use the Azure source to import secret data from Azure KeyVault into your Vault instance. - -## Argument reference - -Refer to the [HCL syntax](/vault/docs/import#hcl-syntax-1) for arguments common to all source types. - -## Additional arguments - -Required: - -- `key_vault_uri` `(string)` - The URI of the Azure KeyVault you want to import from. - -The following parameters are optional. If you leave these parameters unset, -Vault uses the default credential provider mechanisms, e.g. the credentials persisted to disk -by a preceding `az login`. - -- `cloud_name` `(string: "AzureCloud")` - Azure cloud to connect to -- `tenant_id` `(string: "")` - Tenant ID to use -- `client_id` `(string: "")` - Client ID to use -- `credentials_file` `(string: "")` - Path to a file with the client secret - -## Example - -Define and configure the `my-azure-source-1` Azure source: - -```hcl -source_aws { - name = "my-azure-source-1" - key_vault_uri = "https://keyvault-1234abcd.vault.azure.net" -} -``` - -## Permissions - -To use Azure import, you must grant the associated Azure identity permission to read secrets -from the specified KeyVault: - -```shell-session -"Get", -"List", -``` diff --git a/website/content/docs/import/gcpsm.mdx b/website/content/docs/import/gcpsm.mdx index 21ca2cbca0bd..c9e2b3ebbdb3 100644 --- a/website/content/docs/import/gcpsm.mdx +++ b/website/content/docs/import/gcpsm.mdx @@ -7,7 +7,8 @@ description: The Google Cloud Platform Secret Manager source imports secrets fro # GCP secret import source Use the GCP source to import secret data from GCP Secret Manager into your Vault instance. To use dynamic -credentials with GCP import, ensure the [GCP secrets engine](/vault/docs/secrets/gcp) is already configured. +credentials with GCP import, ensure the [GCP secrets engine](/vault/docs/secrets/gcp) is +already configured. ## Argument reference @@ -15,8 +16,9 @@ Refer to the [HCL syntax](/vault/docs/import#hcl-syntax-1) for arguments common ## Additional arguments -- `credentials_file` `(string: "")` - The path to the service account key credentials file to authenticate with. - If `credentials_file` is set, then `vault_mount_path` and `vault_role_name` must be unset. +- `credentials` `(string: "")` - The path to the service account key credentials file for the service account + with the [necessary permissions](#permissions). If `credentials` is set, then `vault_mount_path` and + `vault_role_name` must be unset. - `vault_mount_path` `(string: "")` - The Vault mount path to a pre-configured GCP secrets engine used to generate dynamic credentials for the importer. If one of diff --git a/website/content/docs/import/index.mdx b/website/content/docs/import/index.mdx index 82a282fb5a59..2303b248a85e 100644 --- a/website/content/docs/import/index.mdx +++ b/website/content/docs/import/index.mdx @@ -18,9 +18,9 @@ several challenges, including: - Increased exposure risk from data sprawl. - Increased risk of outdated and out-of-sync information. -Using Vault as a single source of truth for sensitive data increases +Using Vault as a single source of truth (SSOT) for sensitive data increases security and reduces management overhead, but migrating preexisting data from multiple -and/or varied sources can be complex and costly. +and/or varied sources can be complex and costly. The secrets import process helps you automate and streamline your sensitive data migration with codified import plans as HCL files. Import plans tell Vault which KVv2 secrets @@ -29,8 +29,8 @@ read from, and how to filter this data. Three HCL blocks make this possible: - The `destination` block defines target KVv2 mounts. - The `source` block provides credentials for connecting to the external system. -- The `mapping` block defines how Vault should decide which data gets imported (and possibly - transformed) before writing the information to KVv2. +- The `mapping` block defines how Vault should decide which data gets imported before + writing the information to KVv2. ## Destinations @@ -41,6 +41,7 @@ Vault instance you want the information stored. ### HCL syntax + ```hcl destination_vault { name = "my-dest-1" @@ -57,25 +58,23 @@ destination_vault { - `address` `(string)` - Optional network address of the Vault server with the KVv2 secrets engine enabled. By default, the Vault client's address will be used. -- `credentials_file` `(string)` - Optional path to authentication token for the Vault - server at the specified address. By default, the Vault client's token will be used. +- `token` `(string)` - Optional authentication token for the Vault server at the + specified address. By default, the Vault client's token will be used. - `namespace` `(string)` - Optional namespace path containing the specified KVv2 mount. By default, Vault looks for the KVv2 mount under the root namespace. + + ## Sources Vault can import secrets from the following sources: - [GCP Secret Manager](/vault/docs/import/gcpsm) -- [AWS Secret Manager](/vault/docs/import/awssm) -- [Azure KeyVault](/vault/docs/import/azurekv) -- [HashiCorp Vault](/vault/docs/import/vault) -To pull data from a source during import, Vault needs credentials for the +To pull data from a source during import, Vault needs read credentials for the external system. You can provide credentials directly as part of the import plan, or use Vault to automatically generate dynamic credentials if you already -have the corresponding secrets engine configured. Otherwise credentials will be -fetched from the default locations appropriate to the external system. +have the corresponding secrets engine configured. ### HCL syntax @@ -84,45 +83,46 @@ information required by the target system or the secrets engine to leverage. For ```hcl source_gcp { - name = "my-gcp-source-1" - credentials_file = "/path/to/service-account-key.json" + name = "my-gcp-source-1" + credentials = "@/path/to/service-account-key.json" } ``` - `name` `(string: )` - A unique name for the source block that can be referenced in subsequent mapping blocks. -- `credentials_file` `(string: )` - Path to a credential file with +- `credentials` `(string: )` - Path to a credential file or token with read permissions for the target system. Depending on the source system, additional information may be required. Refer to the connection documentation for your source system to determine the full set of required fields for that system type. + + ## Mappings Mappings glue the source and destination together and filter the migrated data, -to determine what is imported and what is ignored. +to determine what is imported and what is ignored. Vault currently supports the +following mapping methods: + +- [mapping_passthrough](/vault/docs/import/mappings#passthrough) +- [mapping_metadata](/vault/docs/import/mappings#metadata) +- [mapping_regex](/vault/docs/import/mappings#regex) ### HCL syntax -Mapping blocks require a name, a source name, a destination name, and any corresponding -transformations or filters that apply for each mapping type. Example: +Mapping blocks start with `mapping_` and require a source name, +destination name, an execution priority, and any corresponding transformations +or filters that apply for each mapping type. For example: ```hcl -mapping { +mapping_regex { name = "my-map-1" source = "my-gcp-source-1" destination = "my-dest-1" - filter = "Secret.Name matches `(foo|bar)`" - transform "exact" { - from = "foo" - to = "foosball" - } - transform "regexp" { - from = "foo(.*)" - to = "bar$1" - } + priority = 1 + expression = "^database/.*$" } ``` @@ -134,58 +134,20 @@ mapping { - `destination` `(string: )` - The name of a previously defined destination block **to** which the data should be written. -- `filter` `(string: "")` - Optional string containing a (`bexpr` style boolean expression)[https://github.com/hashicorp/go-bexpr] - that limits which secrets Vault imports from the source. - -#### Filters - -Filters describes conditions for secret fields that a source secret must meet to be imported. You can filter against the following secret fields: - -- `Secret.Name` - The name of the source secret -- `Secret.Tags` - A map of key-value user-defined metadata associated with the source secret - -You can also apply simple binary conditions to input values: +- `priority` `(integer: )` - The order in which Vault should apply the + mapping block during the import process. The lower the number, the higher the + priority. For example, a mapping with priority 1 executes before a mapping + with priority 2. -- `Secret.Name != "foo"` -- `Secret.Tags.Team matches "my-dept.*"` -- `Secret.Tags contains "my-required-tag-name"` +Depending on the filter type, additional fields may be required or possible. Refer +to the [import mappings documentation](/vault/docs/import/mappings) for the available +supported options and for a list of each mapping's specific fields. -These can be combined using `and`, `or`, and `not`: + -- `(Secret.Name == "foo" or Secret.Tags.Team == "my-team") and Secret.Tags.Category = "my-cat"` - -#### Transformers - -Transform stanzas come in two forms: `exact` and `regexp`. - -An exact transform allows renaming a secret during import, so that the `from` secret name -is imported into Vault as a secret named `to`. In the following example it, takes a source secret -named `foo` and transforms it to `foosball` during import. - -```hcl - transform "exact" { - from = "foo" - to = "foosball" - } -``` - -`regexp` transforms rename secrets during import using -[Go regular expression syntax](https://github.com/google/re2/wiki/Syntax). For -example, the transform below imports any secret in the source whose name starts -with `foo` and replaces the `foo` prefix with `bar`. - -```hcl - transform "regexp" { - from = "foo(.*)" - to = "bar$1" - } - transform "regexp" { - from = "foo(?P.*)" - to = "bar$suffix" - } -``` + Vault applies mapping definitions in priority order and a given secret only + matches to the first mapping that applies. Once Vault imports a secret with a + particular mapping, subsequent reads from the same source will ignore that + secret. See the [priority section](/vault/docs/import/mappings#priority) for an example. -The `from` value supports parentheses to bookend capture groups and named -capture groups using the syntax `(?Pre)`. When you use named capture -groups, you can reference the named group in the `to` value. For example, -`$name` instead of `$1`. + diff --git a/website/content/docs/import/mappings.mdx b/website/content/docs/import/mappings.mdx new file mode 100644 index 000000000000..50050e8154ee --- /dev/null +++ b/website/content/docs/import/mappings.mdx @@ -0,0 +1,132 @@ +--- +layout: docs +page_title: Secrets import mappings +description: Mappings lets users apply various filtering methods to secrets being imported in to Vault. +--- + +# Secrets import mappings + +Vault supports multiple filter types for mapping blocks. Each of the types provides a different mechanism +used to filter the scanned secrets and determine which will be imported in to Vault. + + +## Argument reference + +Refer to the [HCL syntax](/vault/docs/import#hcl-syntax-2) for arguments common to all mapping types. + +## Passthrough mapping filters + +The passthrough mapping block `mapping_passthrough` allows all secrets through from the specified source to the +specified destination. For example, one use case is setting it as a base-case for imported secrets. By assigning +it the lowest priority in the import plan, all other mapping blocks will be applied first. Secrets that fail +to match any of the previous mappings will fall through to the passthrough block and be collected in a single +KVv2 location. + +### Additional arguments + +There are no extra arguments to specify in a `mapping_passthrough` block. + +### Example + +In this example, every single secret that `my-gcp-source-1` scans from GCP Secret Manager will be imported +to the KVv2 secrets engine mount defined in `my-dest-1`. + +```hcl +mapping_passthrough { + name = "my-map-1" + source = "my-gcp-source-1" + destination = "my-dest-1" + priority = 1 +} +``` + +## Metadata + +The metadata mapping block `mapping_metadata` allows secrets through from the specified source to the specified +destination if they contain matching metadata key-value pairs. Metadata is not supported in all external secret +management systems, and ones that do may use different terminology for metadata. For example, AWS allows tags +on secrets while [GCP](/vault/docs/import/gcpsm) allows labels. + +### Additional arguments + +* `tags` `(string: )` - A set of key-value pairs to match on secrets from the external system. All of the specified +keys must be found on a secret and all of the values must be exact matches. Specifying a key in this mapping with +an empty value, i.e. `""`, acts as a wildcard match to the external system's key's value. + +### Example + +In this example, `my-map-1` will only import the secrets into the destination `my-dest-1` that contain a tag with +a key named `importable` and its value set to `true`. + +```hcl +mapping_metadata { + name = "my-map-1" + source = "my-gcp-source-1" + destination = "my-dest-1" + priority = 1 + + tags = { + "importable" = "true" + } +} +``` + +## Regex + +The regex mapping block `mapping_regex` allows secrets through from the specified source to the specified +destination if their secret name passes a regular expression check. + +### Additional arguments + +* `expression` `(string: )` - The regular expression used to match secrets' names from the external system. + +### Example + +In this example, any secret in the GCP source whose name begins with `database/` will be imported into Vault. + +```hcl +mapping_regex { + name = "my-map-1" + source = "my-gcp-source-1" + destination = "my-dest-1" + priority = 1 + expression = "^database/.*$" +} +``` + +## Priority + +Priority works in a "first match" fashion where lower values are higher priority. To explain in more detail, +consider the above metadata example with a second additional mapping. + +Below are two metadata mappings. The first, `my-map-1`, has a priority of 1. This will only import the secrets +into the destination `my-dest-1` that contain both tag keys `database` and `importable`. Each of these keys' values +must also match to `users` and `true` respectively. The second, `my-map-2`, has a priority of 2. Even though all +the secrets in the first mapping would also qualify for the second mapping's filtering rule, those secrets will only +be imported into `my-dest-1` because of `my-map-2`'s lower priority. All remaining secrets that have the tag +`importable` with a value of `true` will be imported into `my-dest-2`. + +```hcl +mapping_metadata { + name = "my-map-1" + source = "my-gcp-source-1" + destination = "my-dest-1" + priority = 1 + + tags = { + "database" = "users" + "importable" = "true" + } +} + +mapping_metadata { + name = "my-map-2" + source = "my-gcp-source-1" + destination = "my-dest-2" + priority = 2 + + tags = { + "importable" = "true" + } +} +``` diff --git a/website/content/docs/import/vault.mdx b/website/content/docs/import/vault.mdx deleted file mode 100644 index 909a1a7c0c17..000000000000 --- a/website/content/docs/import/vault.mdx +++ /dev/null @@ -1,38 +0,0 @@ ---- -layout: docs -page_title: Vault secret import source -description: The Vault source imports secrets from Vault to Vault. ---- - -# Vault secret import source - -Use the Vault source to import secret data from a Vault cluster into your Vault instance. - -## Argument reference - -Refer to the [HCL syntax](/vault/docs/import#hcl-syntax-1) for arguments common to all source types. - -## Additional arguments - -- `address` `(string: required)` - API address of source vault cluster. -- `namespace` `(string: "")` - Optional namespace to read from. -- `mount` `(string: required)` - KV-v2 mount to read from. -- `credentials_file` `(string: "")` - Path to a file containing a vault token. If absent, the CLI uses the -environment variable VAULT_TOKEN. However, VAULT_TOKEN will be used for the Vault destination as well, so -unless you're using the same cluster as both source and destination (e.g. from one mount to another), the source -`credentials_file` should be populated with the source-specific token. - -## Example - -Define and configure the `my-vault-source-1` Vault source: - -```hcl -source_vault { - name = "my-vault-source-1" - address = "https://source-vault-address:8200" - namespace = "ns1" - mount = "my-kvv2" - credentials_file = "/path/to/vault/token" -} -``` - diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index 165659fc8a36..ae7ddfead62d 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -404,10 +404,10 @@ "color": "neutral" }, "path": "configuration/entropy-augmentation" - }, - + }, + { "heading": "Configuration stanzas" }, - + { "title": "adaptive_overload_protection", "badge": { @@ -1882,21 +1882,13 @@ "color": "highlight" } }, - { - "title": "AWS Secret Manager", - "path": "import/awssm" - }, - { - "title": "Azure KeyVault", - "path": "import/azurekv" - }, { "title": "GCP Secret Manager", "path": "import/gcpsm" }, { - "title": "HashiCorp Vault", - "path": "import/vault" + "title": "Mappings", + "path": "import/mappings" } ] },