diff --git a/README.md b/README.md
index 2790a57..73e8549 100644
--- a/README.md
+++ b/README.md
@@ -164,14 +164,14 @@ module "metastore_assignment" {
|------|---------|
| [terraform](#requirement\_terraform) | >=1.0.0 |
| [azurerm](#requirement\_azurerm) | >=3.40.0 |
-| [databricks](#requirement\_databricks) | >=1.14.2 |
+| [databricks](#requirement\_databricks) | >=1.30.0 |
## Providers
| Name | Version |
|------|---------|
| [azurerm](#provider\_azurerm) | >=3.40.0 |
-| [databricks](#provider\_databricks) | >=1.14.2 |
+| [databricks](#provider\_databricks) | >=1.30.0 |
## Modules
@@ -213,7 +213,7 @@ No modules.
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
-| [clusters](#input\_clusters) | Set of objects with parameters to configure Databricks clusters and assign permissions to it for certain custom groups | set(object({
cluster_name = string
spark_version = optional(string, "13.3.x-scala2.12")
spark_conf = optional(map(any), {})
cluster_conf_passthrought = optional(bool, false)
spark_env_vars = optional(map(any), {})
data_security_mode = optional(string, "USER_ISOLATION")
node_type_id = optional(string, "Standard_D3_v2")
autotermination_minutes = optional(number, 30)
min_workers = optional(number, 1)
max_workers = optional(number, 2)
availability = optional(string, "ON_DEMAND_AZURE")
first_on_demand = optional(number, 0)
spot_bid_max_price = optional(number, 1)
cluster_log_conf_destination = optional(string, null)
init_scripts_workspace = optional(set(string), [])
init_scripts_volumes = optional(set(string), [])
init_scripts_dbfs = optional(set(string), [])
init_scripts_abfss = optional(set(string), [])
single_user_name = optional(string, null)
permissions = optional(set(object({
group_name = string
permission_level = string
})), [])
pypi_library_repository = optional(set(string), [])
maven_library_repository = optional(set(object({
coordinates = string
exclusions = set(string)
})), [])
})) | `[]` | no |
+| [clusters](#input\_clusters) | Set of objects with parameters to configure Databricks clusters and assign permissions to it for certain custom groups | set(object({
cluster_name = string
spark_version = optional(string, "13.3.x-scala2.12")
spark_conf = optional(map(any), {})
cluster_conf_passthrought = optional(bool, false)
spark_env_vars = optional(map(any), {})
data_security_mode = optional(string, "USER_ISOLATION")
node_type_id = optional(string, "Standard_D3_v2")
autotermination_minutes = optional(number, 30)
min_workers = optional(number, 1)
max_workers = optional(number, 2)
availability = optional(string, "ON_DEMAND_AZURE")
first_on_demand = optional(number, 0)
spot_bid_max_price = optional(number, 1)
cluster_log_conf_destination = optional(string, null)
init_scripts_workspace = optional(set(string), [])
init_scripts_volumes = optional(set(string), [])
init_scripts_dbfs = optional(set(string), [])
init_scripts_abfss = optional(set(string), [])
single_user_name = optional(string, null)
single_node_enable = optional(bool, false)
custom_tags = optional(map(string), {})
permissions = optional(set(object({
group_name = string
permission_level = string
})), [])
pypi_library_repository = optional(set(string), [])
maven_library_repository = optional(set(object({
coordinates = string
exclusions = set(string)
})), [])
})) | `[]` | no |
| [create\_databricks\_access\_policy\_to\_key\_vault](#input\_create\_databricks\_access\_policy\_to\_key\_vault) | Boolean flag to enable creation of Key Vault Access Policy for Databricks Global Service Principal. | `bool` | `true` | no |
| [custom\_cluster\_policies](#input\_custom\_cluster\_policies) | Provides an ability to create custom cluster policy, assign it to cluster and grant CAN\_USE permissions on it to certain custom groups
name - name of custom cluster policy to create
can\_use - list of string, where values are custom group names, there groups have to be created with Terraform;
definition - JSON document expressed in Databricks Policy Definition Language. No need to call 'jsonencode()' function on it when providing a value; | list(object({
name = string
can_use = list(string)
definition = any
})) | [
{
"can_use": null,
"definition": null,
"name": null
}
]
| no |
| [global\_databricks\_sp\_object\_id](#input\_global\_databricks\_sp\_object\_id) | Global 'AzureDatabricks' SP object id. Used to create Key Vault Access Policy for Secret Scope | `string` | `"9b38785a-6e08-4087-a0c4-20634343f21f"` | no |
diff --git a/cluster.tf b/cluster.tf
index 3fe51a1..dc486cd 100644
--- a/cluster.tf
+++ b/cluster.tf
@@ -1,24 +1,31 @@
+locals {
+ spark_conf_single_node = {
+ "spark.master" = "local[*]"
+ "spark.databricks.cluster.profile" = "singleNode"
+ }
+ conf_passthrought = {
+ "spark.databricks.cluster.profile" : "serverless",
+ "spark.databricks.repl.allowedLanguages" : "python,sql",
+ "spark.databricks.passthrough.enabled" : "true",
+ "spark.databricks.pyspark.enableProcessIsolation" : "true"
+ }
+}
+
resource "databricks_cluster" "cluster" {
for_each = { for cluster in var.clusters : cluster.cluster_name => cluster }
cluster_name = each.value.cluster_name
spark_version = each.value.spark_version
- spark_conf = each.value.cluster_conf_passthrought ? merge({
- "spark.databricks.cluster.profile" : "serverless",
- "spark.databricks.repl.allowedLanguages" : "python,sql",
- "spark.databricks.passthrough.enabled" : "true",
- "spark.databricks.pyspark.enableProcessIsolation" : "true"
- }, each.value.spark_conf) : each.value.spark_conf
+ spark_conf = merge(
+ each.value.cluster_conf_passthrought ? local.conf_passthrought : {},
+ each.value.single_node_enable == true ? local.spark_conf_single_node : {},
+ each.value.spark_conf)
spark_env_vars = each.value.spark_env_vars
data_security_mode = each.value.cluster_conf_passthrought ? null : each.value.data_security_mode
node_type_id = each.value.node_type_id
autotermination_minutes = each.value.autotermination_minutes
single_user_name = each.value.single_user_name
-
- autoscale {
- min_workers = each.value.min_workers
- max_workers = each.value.max_workers
- }
+ custom_tags = merge(each.value.single_node_enable ? { "ResourceClass" = "SingleNode" } : {}, each.value.custom_tags)
azure_attributes {
availability = each.value.availability
@@ -26,6 +33,14 @@ resource "databricks_cluster" "cluster" {
spot_bid_max_price = each.value.spot_bid_max_price
}
+ dynamic "autoscale" {
+ for_each = each.value.single_node_enable ? [] : [1]
+ content {
+ min_workers = each.value.min_workers
+ max_workers = each.value.max_workers
+ }
+ }
+
dynamic "cluster_log_conf" {
for_each = each.value.cluster_log_conf_destination != null ? [each.value.cluster_log_conf_destination] : []
content {
diff --git a/variables.tf b/variables.tf
index 515600f..96b478e 100644
--- a/variables.tf
+++ b/variables.tf
@@ -166,6 +166,8 @@ variable "clusters" {
init_scripts_dbfs = optional(set(string), [])
init_scripts_abfss = optional(set(string), [])
single_user_name = optional(string, null)
+ single_node_enable = optional(bool, false)
+ custom_tags = optional(map(string), {})
permissions = optional(set(object({
group_name = string
permission_level = string
diff --git a/versions.tf b/versions.tf
index 60901f7..471f531 100644
--- a/versions.tf
+++ b/versions.tf
@@ -8,7 +8,7 @@ terraform {
}
databricks = {
source = "databricks/databricks"
- version = ">=1.14.2"
+ version = ">=1.30.0"
}
}
}