From 9d4b70ad24a45ccda642318be7d641f3126bc7a7 Mon Sep 17 00:00:00 2001 From: dmytro_velychko3 Date: Wed, 17 May 2023 18:04:11 +0300 Subject: [PATCH 1/7] feat: delete unity catalog BREAKING CHANGE: unity catalog in individual module --- README.md | 130 --------------------------------------------------- main.tf | 7 +++ outputs.tf | 13 ------ variables.tf | 53 --------------------- 4 files changed, 7 insertions(+), 196 deletions(-) diff --git a/README.md b/README.md index 92bacb9..52ca144 100644 --- a/README.md +++ b/README.md @@ -134,117 +134,6 @@ module "databricks_runtime_premium" { } ``` -### Create Unity Catalog metastore -An example below explains to create Unity Catalog Metastore. -It is highly recommended to create Metastore on separate environment or even Azure subscription. - -```hcl -# Prerequisite resources - -# Databricks Workspace with Premium SKU -data "azurerm_databricks_workspace" "example" { - name = "example-workspace" - resource_group_name = "example-rg" -} - -# Databricks Provider configuration -provider "databricks" { - alias = "main" - host = data.azurerm_databricks_workspace.example.workspace_url - azure_workspace_resource_id = data.azurerm_databricks_workspace.example.id -} - -# This Access connector cloud be created with Databricks Workspace module -resource "azurerm_databricks_access_connector" "example" { - name = "databrickstest" - resource_group_name = "example-rg" - location = "eastus" - - identity { - type = "SystemAssigned" - } -} - -# Storage Account where metastore would be created -data "azurerm_storage_account" "example" { - name = "metastore" - resource_group_name = "example-rg" -} - -# Example usage of module for Unity Catalog Metastore creation -module "databricks_runtime_premium" { - source = "data-platform-hq/databricks-runtime-premium/databricks" - - project = "datahq" - env = "example" - location = "eastus" - - ip_rules = { - "example_devops-0" = "10.128.0.0/16", - "example_devops-1" = "10.33.0.0/16", - } - user_object_ids = { - "example-app-id" = "ebfasddf-05sd-4sdc-aasa-ddffgs83c299" - "user1@example.com" = "ebfasddf-05sd-4sdc-aasa-ddffgs83c256" - } - - # Unity Catalog - create_metastore = true - metastore_grants = { "account users" = ["CREATE_CATALOG"] } - - access_connector_id = azurerm_databricks_access_connector.example.id - workspace_id = data.azurerm_databricks_workspace.example.id - - - catalog = { - catalog-one-data = { - catalog_grants = { - "user1@example.com" = ["USE_CATALOG", "USE_SCHEMA", "CREATE_SCHEMA", "CREATE_TABLE", "SELECT", "MODIFY"] - "account users" = ["USE_CATALOG", "USE_SCHEMA", "SELECT"] - } - catalog_comment = "This catalog is created by Terraform" - schema_name = ["schema1", "schema2", "schema3"] - schema_grants = { - "account_users" = ["USE_SCHEMA", "CREATE_TABLE","CREATE_VIEW", "MODIFY"] - } - schema_comment = "Created by terraform. Allowed for SELECT operations" - schema_properties = { allowed = "all users"} - } - - catalog-two-admin = { - catalog_grants = { - "user1@example.com" = ["USE_CATALOG", "USE_SCHEMA", "CREATE_SCHEMA", "CREATE_TABLE", "SELECT", "MODIFY"] - } - catalog_comment = "This catalog is created by Terraform" - schema_name = ["schema1"] - schema_properties = { allowed = "admin only"} - } -} - - # Storage Account where Metastore would be created - storage_account_id = data.azurerm_storage_account.example.id - storage_account_name = data.azurerm_storage_account.example.name - - # Permissions - workspace_admins = { - user = ["user1@example.com"] - service_principal = ["example-app-id"] - } - - # Cluster for Unity Catalog access - databricks_cluster_configs = [{ - cluster_name = "Unity Catalog" - availability = "SPOT_AZURE" - spot_bid_max_price = -1 - }] - - providers = { - databricks = databricks.main - } -} -``` - - ## Requirements @@ -289,15 +178,7 @@ No modules. | [databricks_ip_access_list.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/ip_access_list) | resource | | [databricks_sql_global_config.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/sql_global_config) | resource | | [databricks_sql_endpoint.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/sql_endpoint) | resource | -| [azurerm_storage_data_lake_gen2_filesystem.this](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_data_lake_gen2_filesystem) | resource | -| [databricks_metastore.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/metastore) | resource | -| [databricks_grants.metastore](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grants) | resource | -| [databricks_metastore_data_access.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/metastore_data_access) | resource | | [databricks_metastore_assignment.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/metastore_assignment) | resource | -| [databricks_catalog.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/catalog) | resource | -| [databricks_grants.catalog](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grants) | resource | -| [databricks_schema.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/sql_endpoint) | resource | -| [databricks_grants.schema](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/schema) | resource | | [databricks_mount.adls](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/mount) | resource | | [databricks_secret_scope.main](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/secret_scope) | resource | | [databricks_secret_scope.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/secret_scope) | resource | @@ -311,21 +192,12 @@ No modules. | Name | Description | Type | Default | Required | |------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| :------: | -| [project](#input\_project) | Project name | `string` | n/a | yes | -| [env](#input\_env) | Environment name | `string` | n/a | yes | -| [location](#input\_location) | Azure location | `string` | n/a | yes | | [workspace\_id](#input\_workspace\_id) | Id of Azure Databricks workspace | `string` | n/a | yes | | [ip\_rules](#input\_ip\_rules) | Map of IP addresses permitted for access to DB | `map(string)` | {} | no | | [user\_object\_ids](#input\_user\_object\_ids) | Map of AD usernames and corresponding object IDs | `map(string)` | {} | no | | [workspace\_admins](#input\_workspace\_admins) | Provide users or service principals to grant them Admin permissions in Workspace. | 
 object({ 
    user              = list(string) 
    service_principal = list(string)
  })
| 
 { 
   user              = null 
   service_principal = null 
 }
| no | | [iam](#input\_iam) | Used to create workspace group. Map of group name and its parameters, such as users and service principals added to the group. Also possible to configure group entitlements. | 
 map(object({ 
   user              = optional(list(string)) 
   service_principal = optional(list(string)) 
   entitlements      = optional(list(string)) 
 }))
| {} | no | | [sql\_endpoint](#input\_sql\_endpoint) | Set of objects with parameters to configure SQL Endpoint and assign permissions to it for certain custom groups | 
 map(object({ 
   cluster_size              = string 
   min_num_clusters          = optional(number) 
   max_num_clusters          = optional(number) 
   auto_stop_mins            = optional(string) 
   enable_photon             = optional(bool) 
   enable_serverless_compute = optional(bool) 
 }))
| {} | no | -| [create\_metastore](#input\_create\_metastore) | Boolean flag for Unity Catalog Metastore current in this environment. One Metastore per region | `bool` | false | no | -| [access\_connector\_id](#input\_access\_connector\_id) | Databricks Access Connector Id that lets you to connect managed identities to an Azure Databricks account. Provides an ability to access Unity Catalog with assigned identity | `string` | " " | no | -| [storage\_account\_id](#input\_storage\_account\_id) | Storage Account Id where Unity Catalog Metastore would be provisioned | `string` | " " | no | -| [storage\_account\_name](#input\_storage\_account\_name) | Storage Account Name where Unity Catalog Metastore would be provisioned | `string` | " " | no | -| [catalog](#input\_catalog) | Map of SQL Endpoints to be deployed in Databricks Workspace | 
 map(object({ 
   catalog_grants     = optional(map(list(string))) 
   catalog_comment    = optional(string) 
   catalog_properties = optional(map(string)) 
   schema_name        = optional(list(string)) 
   schema_grants      = optional(map(list(string))) 
   schema_comment     = optional(string) 
   schema_properties  = optional(map(string))
}))
| {} | no | -| [suffix](#input\_suffix) | Optional suffix that would be added to the end of resources names. | `string` | " " | no | | [external\_metastore\_id](#input\_external\_metastore\_id) | Unity Catalog Metastore Id that is located in separate environment. Provide this value to associate Databricks Workspace with target Metastore | `string` | " " | no | | [metastore\_grants](#input\_metastore\_grants) | Permissions to give on metastore to group | `map(list(string))` | {} | no | | [sp\_client\_id\_secret\_name](#input\_sp\_client\_id\_secret\_name) | The name of Azure Key Vault secret that contains ClientID of Service Principal to access in Azure Key Vault | `string` | n/a | yes | @@ -349,9 +221,7 @@ No modules. | ----------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------- | | [sql\_endpoint\_jdbc\_url](#output\_sql\_endpoint\_jdbc\_url) | JDBC connection string of SQL Endpoint | | [sql\_endpoint\_data\_source\_id](#output\_sql\_endpoint\_data\_source\_id) | ID of the data source for this endpoint | -| [metastore\_id](#output\_metastore\_id) | Unity Catalog Metastore Id | | [token](#output\_token) | Databricks Personal Authorization Token | -| [clusters](#output\_clusters) | Provides name and unique identifier for the clusters | ## License diff --git a/main.tf b/main.tf index a788e2f..98270c3 100644 --- a/main.tf +++ b/main.tf @@ -66,3 +66,10 @@ resource "databricks_sql_endpoint" "this" { } depends_on = [databricks_sql_global_config.this] } + +resource "databricks_metastore_assignment" "this" { + count = length(var.external_metastore_id) != 0 ? 1 : 0 + + workspace_id = var.workspace_id + metastore_id = var.external_metastore_id +} diff --git a/outputs.tf b/outputs.tf index d067de7..25ba68e 100644 --- a/outputs.tf +++ b/outputs.tf @@ -8,21 +8,8 @@ output "sql_endpoint_data_source_id" { description = "ID of the data source for this endpoint" } -output "metastore_id" { - value = var.create_metastore ? databricks_metastore.this[0].id : "" - description = "Unity Catalog Metastore Id" -} - output "token" { value = databricks_token.pat.token_value description = "Databricks Personal Authorization Token" sensitive = true } - -output "clusters" { - value = [for param in var.clusters : { - name = param.cluster_name - id = databricks_cluster.cluster[param.cluster_name].id - } if length(var.clusters) != 0] - description = "Provides name and unique identifier for the clusters" -} diff --git a/variables.tf b/variables.tf index 4e651d5..1ef9132 100644 --- a/variables.tf +++ b/variables.tf @@ -1,18 +1,3 @@ -variable "project" { - type = string - description = "Project name" -} - -variable "env" { - type = string - description = "Environment name" -} - -variable "location" { - type = string - description = "Azure location" -} - variable "workspace_id" { type = string description = "Id of Azure Databricks workspace" @@ -81,44 +66,6 @@ variable "sql_endpoint" { default = [] } -# Unity Catalog variables -variable "create_metastore" { - type = bool - description = "Boolean flag for Unity Catalog Metastore current in this environment. One Metastore per region" - default = false -} - -variable "access_connector_id" { - type = string - description = "Databricks Access Connector Id that lets you to connect managed identities to an Azure Databricks account. Provides an ability to access Unity Catalog with assigned identity" - default = "" -} - -variable "storage_account_id" { - type = string - description = "Storage Account Id where Unity Catalog Metastore would be provisioned" - default = "" -} -variable "storage_account_name" { - type = string - description = "Storage Account Name where Unity Catalog Metastore would be provisioned" - default = "" -} - -variable "catalog" { - type = map(object({ - catalog_grants = optional(map(list(string))) - catalog_comment = optional(string) - catalog_properties = optional(map(string)) - schema_name = optional(list(string)) - schema_grants = optional(map(list(string))) - schema_comment = optional(string) - schema_properties = optional(map(string)) - })) - description = "Map of catalog name and its parameters" - default = {} -} - variable "suffix" { type = string description = "Optional suffix that would be added to the end of resources names." From 94cf091bf6a3a5ece1e380f7e570f76ef2d914d2 Mon Sep 17 00:00:00 2001 From: dmytro_velychko3 Date: Wed, 17 May 2023 18:07:07 +0300 Subject: [PATCH 2/7] feat: delete unity.tf --- unity.tf | 135 ------------------------------------------------------- 1 file changed, 135 deletions(-) delete mode 100644 unity.tf diff --git a/unity.tf b/unity.tf deleted file mode 100644 index 714d660..0000000 --- a/unity.tf +++ /dev/null @@ -1,135 +0,0 @@ -resource "azurerm_storage_data_lake_gen2_filesystem" "this" { - count = var.create_metastore ? 1 : 0 - - name = "meta-${var.project}-${var.env}" - storage_account_id = var.storage_account_id - - lifecycle { - precondition { - condition = alltrue([ - for variable in [var.storage_account_id, var.access_connector_id, var.storage_account_name] : false if length(variable) == 0 - ]) - error_message = "To create Metastore in a Region it is required to provide proper values for these variables: access_connector_id, storage_account_id, storage_account_name" - } - } -} - -resource "databricks_metastore" "this" { - count = var.create_metastore ? 1 : 0 - - name = "meta-${var.project}-${var.env}-${var.location}${local.suffix}" - storage_root = format("abfss://%s@%s.dfs.core.windows.net/", azurerm_storage_data_lake_gen2_filesystem.this[0].name, var.storage_account_name) - force_destroy = true -} - -resource "databricks_grants" "metastore" { - for_each = !var.create_metastore && length(var.external_metastore_id) == 0 ? {} : { - for k, v in var.metastore_grants : k => v - if v != null - } - - metastore = length(var.external_metastore_id) == 0 ? databricks_metastore.this[0].id : var.external_metastore_id - grant { - principal = each.key - privileges = each.value - } -} - -resource "databricks_metastore_data_access" "this" { - count = var.create_metastore ? 1 : 0 - - metastore_id = databricks_metastore.this[0].id - name = "data-access-${var.project}-${var.env}-${var.location}${local.suffix}" - azure_managed_identity { - access_connector_id = var.access_connector_id - } - is_default = true -} - -resource "databricks_metastore_assignment" "this" { - count = !var.create_metastore && length(var.external_metastore_id) == 0 ? 0 : 1 - - workspace_id = var.workspace_id - metastore_id = length(var.external_metastore_id) == 0 ? databricks_metastore.this[0].id : var.external_metastore_id - default_catalog_name = "hive_metastore" -} - -# Catalog -resource "databricks_catalog" "this" { - for_each = !var.create_metastore && length(var.external_metastore_id) == 0 ? {} : var.catalog - - metastore_id = length(var.external_metastore_id) == 0 ? databricks_metastore.this[0].id : var.external_metastore_id - name = each.key - comment = lookup(each.value, "catalog_comment", "default comment") - properties = merge(lookup(each.value, "catalog_properties", {}), { env = var.env }) - force_destroy = true - - depends_on = [databricks_metastore_assignment.this[0]] -} - -# Catalog grants -resource "databricks_grants" "catalog" { - for_each = !var.create_metastore && length(var.external_metastore_id) == 0 ? {} : { - for name, params in var.catalog : name => params.catalog_grants - if params.catalog_grants != null - } - - catalog = databricks_catalog.this[each.key].name - dynamic "grant" { - for_each = each.value - content { - principal = grant.key - privileges = grant.value - } - } -} - -# Schema -locals { - schema = flatten([ - for catalog, params in var.catalog : [ - for schema in params.schema_name : { - catalog = catalog, - schema = schema, - comment = lookup(params, "schema_comment", "default comment"), - properties = lookup(params, "schema_properties", {}) - } - ] if params.schema_name != null - ]) -} - -resource "databricks_schema" "this" { - for_each = !var.create_metastore && length(var.external_metastore_id) == 0 ? {} : { - for entry in local.schema : "${entry.catalog}.${entry.schema}" => entry - } - - catalog_name = databricks_catalog.this[each.value.catalog].name - name = each.value.schema - comment = each.value.comment - properties = merge(each.value.properties, { env = var.env }) - force_destroy = true -} - -# Schema grants -locals { - schema_grants = flatten([ - for catalog, params in var.catalog : [for schema in params.schema_name : [for principal in flatten(keys(params.schema_grants)) : { - catalog = catalog, - schema = schema, - principal = principal, - permission = flatten(values(params.schema_grants)), - }]] if params.schema_grants != null - ]) -} - -resource "databricks_grants" "schema" { - for_each = !var.create_metastore && length(var.external_metastore_id) == 0 ? {} : { - for entry in local.schema_grants : "${entry.catalog}.${entry.schema}.${entry.principal}" => entry - } - - schema = databricks_schema.this["${each.value.catalog}.${each.value.schema}"].id - grant { - principal = each.value.principal - privileges = each.value.permission - } -} From 1285f91b0d29a7f151e37332683424f421a5e117 Mon Sep 17 00:00:00 2001 From: owlleg6 Date: Fri, 19 May 2023 15:19:47 +0300 Subject: [PATCH 3/7] fix: updated variables; and condition --- main.tf | 7 ++++--- variables.tf | 41 +++++++++++++++++------------------------ 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/main.tf b/main.tf index 98270c3..35ae1dd 100644 --- a/main.tf +++ b/main.tf @@ -68,8 +68,9 @@ resource "databricks_sql_endpoint" "this" { } resource "databricks_metastore_assignment" "this" { - count = length(var.external_metastore_id) != 0 ? 1 : 0 + count = alltrue([var.assign_unity_catalog_metastore, length(var.external_metastore_id) != 0]) ? 1 : 0 - workspace_id = var.workspace_id - metastore_id = var.external_metastore_id + workspace_id = var.workspace_id + metastore_id = var.external_metastore_id + default_catalog_name = "hive_metastore" } diff --git a/variables.tf b/variables.tf index 1ef9132..34adbfc 100644 --- a/variables.tf +++ b/variables.tf @@ -72,30 +72,6 @@ variable "suffix" { default = "" } -variable "external_metastore_id" { - type = string - description = "Unity Catalog Metastore Id that is located in separate environment. Provide this value to associate Databricks Workspace with target Metastore" - default = "" - validation { - condition = length(var.external_metastore_id) == 36 || length(var.external_metastore_id) == 0 - error_message = "UUID has to be either in nnnnnnnn-nnnn-nnnn-nnnn-nnnnnnnnnnnn format or empty string" - } -} - -variable "metastore_grants" { - type = map(list(string)) - description = "Permissions to give on metastore to group" - default = {} - validation { - condition = values(var.metastore_grants) != null ? alltrue([ - for item in toset(flatten([for group, params in var.metastore_grants : params if params != null])) : contains([ - "CREATE_CATALOG", "CREATE_EXTERNAL_LOCATION", "CREATE_SHARE", "CREATE_RECIPIENT", "CREATE_PROVIDER" - ], item) - ]) : true - error_message = "Metastore permission validation. The only possible values for permissions are: CREATE_CATALOG, CREATE_EXTERNAL_LOCATION, CREATE_SHARE, CREATE_RECIPIENT, CREATE_PROVIDER" - } -} - variable "sp_client_id_secret_name" { type = string description = "The name of Azure Key Vault secret that contains ClientID of Service Principal to access in Azure Key Vault" @@ -151,6 +127,23 @@ variable "mountpoints" { default = {} } +# Unity Catalog Metastore assignment variables +variable "assign_unity_catalog_metastore" { + type = bool + description = "Boolean flag provides an ability to assign Unity Catalog Metastore to this Workspace" + default = false +} + +variable "external_metastore_id" { + type = string + description = "Unity Catalog Metastore Id that is located in separate environment. Provide this value to associate Databricks Workspace with target Metastore" + default = "" + validation { + condition = anytrue([length(var.external_metastore_id) == 36, length(var.external_metastore_id) == 0]) + error_message = "UUID has to be either in nnnnnnnn-nnnn-nnnn-nnnn-nnnnnnnnnnnn format or empty string" + } +} + variable "custom_cluster_policies" { type = list(object({ name = string From 5dc2c94b476b7b3a11c9cc8624493ab25cfb0faf Mon Sep 17 00:00:00 2001 From: dmytro_velychko3 Date: Fri, 19 May 2023 15:51:33 +0300 Subject: [PATCH 4/7] docs: changed variable description --- README.md | 7 ++++--- variables.tf | 24 ++++++++++++------------ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 52ca144..be13ab8 100644 --- a/README.md +++ b/README.md @@ -193,19 +193,20 @@ No modules. | Name | Description | Type | Default | Required | |------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| :------: | | [workspace\_id](#input\_workspace\_id) | Id of Azure Databricks workspace | `string` | n/a | yes | -| [ip\_rules](#input\_ip\_rules) | Map of IP addresses permitted for access to DB | `map(string)` | {} | no | +| [ip\_rules](#input\_ip\_rules)| Map of IP addresses permitted for access to DB| `map(string)`| {}| no | +| [suffix](#input\_suffix)| Optional suffix that would be added to the end of resources names. | `string`| "" | no | | [user\_object\_ids](#input\_user\_object\_ids) | Map of AD usernames and corresponding object IDs | `map(string)` | {} | no | | [workspace\_admins](#input\_workspace\_admins) | Provide users or service principals to grant them Admin permissions in Workspace. | 
 object({ 
    user              = list(string) 
    service_principal = list(string)
  })
| 
 { 
   user              = null 
   service_principal = null 
 }
| no | | [iam](#input\_iam) | Used to create workspace group. Map of group name and its parameters, such as users and service principals added to the group. Also possible to configure group entitlements. | 
 map(object({ 
   user              = optional(list(string)) 
   service_principal = optional(list(string)) 
   entitlements      = optional(list(string)) 
 }))
| {} | no | | [sql\_endpoint](#input\_sql\_endpoint) | Set of objects with parameters to configure SQL Endpoint and assign permissions to it for certain custom groups | 
 map(object({ 
   cluster_size              = string 
   min_num_clusters          = optional(number) 
   max_num_clusters          = optional(number) 
   auto_stop_mins            = optional(string) 
   enable_photon             = optional(bool) 
   enable_serverless_compute = optional(bool) 
 }))
| {} | no | | [external\_metastore\_id](#input\_external\_metastore\_id) | Unity Catalog Metastore Id that is located in separate environment. Provide this value to associate Databricks Workspace with target Metastore | `string` | " " | no | -| [metastore\_grants](#input\_metastore\_grants) | Permissions to give on metastore to group | `map(list(string))` | {} | no | | [sp\_client\_id\_secret\_name](#input\_sp\_client\_id\_secret\_name) | The name of Azure Key Vault secret that contains ClientID of Service Principal to access in Azure Key Vault | `string` | n/a | yes | | [sp\_key\_secret\_name](#input\_sp\_key\_secret\_name) | The name of Azure Key Vault secret that contains client secret of Service Principal to access in Azure Key Vault | `string` | n/a | yes | | [secret\_scope](#input\_secret\_scope) | Provides an ability to create custom Secret Scope, store secrets in it and assigning ACL for access management | 
list(object({
  scope_name = string
  acl = optional(list(object({
    principal  = string
    permission = string
  })))
  secrets = optional(list(object({
    key          = string
    string_value = string
  })))
}))
| 
default = [{
  scope_name = null
  acl        = null
  secrets    = null
}]
| yes | | [key\_vault\_id](#input\_key\_vault\_id) | ID of the Key Vault instance where the Secret resides | `string` | n/a | yes | | [tenant\_id\_secret\_name](#input\_tenant\_id\_secret\_name) | The name of Azure Key Vault secret that contains tenant ID secret of Service Principal to access in Azure Key Vault | `string` | n/a | yes | -| [mountpoints](#input\_mountpoints) | Mountpoints for databricks | 
map(object({
  storage_account_name = string
  container_name       = string
}))
| {} | no | +| [mountpoints](#input\_mountpoints) | Mountpoints for databricks | 
map(object({
  storage_account_name = string
  container_name       = string
}))
| {} | no | +| [assign\_unity\_catalog\_metastore](#input\_assign\_unity\_catalog\_metastore) | Boolean flag provides an ability to assign Unity Catalog Metastore to this Workspace | `bool` | false | no | | [custom\_cluster\_policies](#input\_custom\_cluster\_policies) | Provides an ability to create custom cluster policy, assign it to cluster and grant CAN_USE permissions on it to certain custom groups | 
list(object({
  name       = string
  can_use    = list(string)
  definition = any
}))
| 
[{
  name       = null
  can_use    = null
  definition = null
}]
| no | | [clusters](#input\_clusters) | Set of objects with parameters to configure Databricks clusters and assign permissions to it for certain custom groups | 
set(object({
    cluster_name                 = string
    spark_version                = optional(string)
    spark_conf                   = optional(map(any))
    cluster_conf_passthrought    = optional(bool)
    spark_env_vars               = optional(map(any))
    data_security_mode           = optional(string)
    node_type_id                 = optional(string)
    autotermination_minutes      = optional(number)
    min_workers                  = optional(number)
    max_workers                  = optional(number)
    max_workers                  = optional(number)
    availability                 = optional(string)
    first_on_demand              = optional(number)
    spot_bid_max_price           = optional(number)
    cluster_log_conf_destination = optional(string)
    permissions = optional(set(object({
      group_name       = string
      permission_level = string
    })), [])
}))
| 
set(object({
    cluster_name                 = string
    spark_version                = optional(string, "11.3.x-scala2.12")
    spark_conf                   = optional(map(any), {})
    cluster_conf_passthrought    = optional(bool, false)
    spark_env_vars               = optional(map(any), {})
    data_security_mode           = optional(string, "USER_ISOLATION")
    node_type_id                 = optional(string, "Standard_D3_v2")
    autotermination_minutes      = optional(number, 30)
    min_workers                  = optional(number, 1)
    max_workers                  = optional(number, 2)
    max_workers                  = optional(number, 2)
    availability                 = optional(string, "ON_DEMAND_AZURE")
    first_on_demand              = optional(number, 0)
    spot_bid_max_price           = optional(number, 1)
    cluster_log_conf_destination = optional(string, null)
    permissions = optional(set(object({
      group_name       = string
      permission_level = string
    })), [])
}))
| no | | [pat\_token\_lifetime\_seconds](#input\_pat\_token\_lifetime\_seconds) | The lifetime of the token, in seconds. If no lifetime is specified, the token remains valid indefinitely | `number` | 315569520 | no | diff --git a/variables.tf b/variables.tf index 34adbfc..055794f 100644 --- a/variables.tf +++ b/variables.tf @@ -9,6 +9,12 @@ variable "ip_rules" { default = {} } +variable "suffix" { + type = string + description = "Optional suffix that would be added to the end of resources names." + default = "" +} + # Identity Access Management variables variable "user_object_ids" { type = map(string) @@ -66,10 +72,14 @@ variable "sql_endpoint" { default = [] } -variable "suffix" { +variable "external_metastore_id" { type = string - description = "Optional suffix that would be added to the end of resources names." + description = "Unity Catalog Metastore Id that is located in separate environment. Provide this value to associate Databricks Workspace with target Metastore" default = "" + validation { + condition = anytrue([length(var.external_metastore_id) == 36, length(var.external_metastore_id) == 0]) + error_message = "UUID has to be either in nnnnnnnn-nnnn-nnnn-nnnn-nnnnnnnnnnnn format or empty string" + } } variable "sp_client_id_secret_name" { @@ -134,16 +144,6 @@ variable "assign_unity_catalog_metastore" { default = false } -variable "external_metastore_id" { - type = string - description = "Unity Catalog Metastore Id that is located in separate environment. Provide this value to associate Databricks Workspace with target Metastore" - default = "" - validation { - condition = anytrue([length(var.external_metastore_id) == 36, length(var.external_metastore_id) == 0]) - error_message = "UUID has to be either in nnnnnnnn-nnnn-nnnn-nnnn-nnnnnnnnnnnn format or empty string" - } -} - variable "custom_cluster_policies" { type = list(object({ name = string From 5290a02ccd060fa8576ed50cc1f08da993332c74 Mon Sep 17 00:00:00 2001 From: Oleh Mykolaishyn Date: Fri, 19 May 2023 16:22:30 +0300 Subject: [PATCH 5/7] chore: temp dryrun --- .releaserc.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.releaserc.json b/.releaserc.json index bdc8bf9..236d6b7 100644 --- a/.releaserc.json +++ b/.releaserc.json @@ -4,7 +4,7 @@ "master" ], "ci": true, - "dryRun": false, + "dryRun": true, "plugins": [ "@semantic-release/commit-analyzer", "@semantic-release/release-notes-generator", From 4021f1610ee651e50a923c85becf97c1b607990e Mon Sep 17 00:00:00 2001 From: Oleh Mykolaishyn Date: Fri, 19 May 2023 16:24:59 +0300 Subject: [PATCH 6/7] chore: temp dry run --- .releaserc.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.releaserc.json b/.releaserc.json index 236d6b7..9f23bde 100644 --- a/.releaserc.json +++ b/.releaserc.json @@ -1,6 +1,7 @@ { "branches": [ "main", + "remove_unity", "master" ], "ci": true, From d489c8d57e6f4b2af42c42b004ef3a9f7866518e Mon Sep 17 00:00:00 2001 From: Oleh Mykolaishyn Date: Fri, 19 May 2023 16:27:15 +0300 Subject: [PATCH 7/7] chore: revert changes --- .releaserc.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.releaserc.json b/.releaserc.json index 9f23bde..bdc8bf9 100644 --- a/.releaserc.json +++ b/.releaserc.json @@ -1,11 +1,10 @@ { "branches": [ "main", - "remove_unity", "master" ], "ci": true, - "dryRun": true, + "dryRun": false, "plugins": [ "@semantic-release/commit-analyzer", "@semantic-release/release-notes-generator",