fix: move cluster policy to cluster tf

Author: dmytro_velychko3

README.md

@@ -10,11 +10,14 @@ Here we provide some examples of how to provision it with a different options.
 
 ### In example below, these features of given module would be covered:
 1. Workspace admins assignment, custom Workspace group creation, group assignments, group entitlements
-2. Workspace IP Access list creation
-3. SQL Endpoint creation and configuration 
-4. Create Cluster policy and assign permissions to custom groups 
-5. Create Secret Scope and assign permissions to custom groups
-6. Connect to already existing Unity Catalog Metastore
+2. Default Shared Autoscaling cluster
+3. Workspace IP Access list creation
+4. ADLS Gen2 Mount
+5. Secret scope and its secrets
+6. SQL Endpoint creation and configuration 
+7. Create Cluster policy and assign permissions to custom groups 
+8. Create Secret Scope and assign permissions to custom groups
+9. Connect to already existing Unity Catalog Metastore
 
 ```hcl
 # Prerequisite resources
@@ -39,8 +42,8 @@ data "azurerm_key_vault" "example" {
 }
 
 # Given module is tightly coupled with this "Runtime Premium" module, it's usage is prerequisite.
-module "databricks_runtime_core" {
-  source  = "data-platform-hq/databricks-runtime/databricks"
+module "databricks_runtime_premium" {
+  source  = "data-platform-hq/databricks-runtime-premium/databricks"
 
   sku          = data.databricks_workspace.example.sku
   workspace_id = data.databricks_workspace.example.workspace_id
@@ -52,10 +55,18 @@ module "databricks_runtime_core" {
   sp_key_secret_name       = "sp-key" # secret's name that stores Service Principal Secret Key
   tenant_id_secret_name    = "infra-arm-tenant-id" # secret's name that stores tenant id value
 
-  # Default cluster parameters
+  # Databricks clusters configuration  
+  databricks_cluster_configs = [{
+    cluster_name       = "shared autoscaling"
+    data_security_mode = "NONE"
+    availability       = "SPOT_AZURE"
+    spot_bid_max_price = -1
+    permissions = [{group_name = "dev", permission_level = "CAN_MANAGE"}]
+  }]
+
+  # Databricks cluster policies
   custom_cluster_policies = [{
     name     = "custom_policy_1",
-    assigned = true, # automatically assigns this policy to default shared cluster if set 'true'
     can_use  =  "DEVELOPERS", # custom workspace group name, that is allowed to use this policy
     definition = {
       "autoscale.max_workers": {
@@ -97,6 +108,12 @@ module "databricks_runtime_premium" {
     "ip_range_2" = "10.33.0.0/16",
   }
   
+  # ADLS Gen2 Mount
+  mountpoints = {
+    storage_account_name = data.azurerm_storage_account.example.name
+    container_name       = "example_container"
+  }
+
   # Here is the map of users and theirs object ids. 
   # This step is optional, in case of Service Principal assignment to workspace, 
   # please only required to provide APP ID as it's value
@@ -130,13 +147,9 @@ module "databricks_runtime_premium" {
       ]
     "service_principal" = []
     entitlements = ["allow_instance_pool_create","allow_cluster_create","databricks_sql_access"]
-    default_cluster_permission = "CAN_RESTART" # assigns certain permission on default cluster to created group
     }
   }
-    
-  # Assigns acls on secret scope to a custom group ("DEVELOPERS" in this example) 
-  secret_scope_object     = module.databricks_runtime_core.secret_scope_object
-  
+
   providers = {
     databricks = databricks.main
   }
@@ -290,7 +303,8 @@ No modules.
 | [azurerm_key_vault_secret.sp_client_id](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault_secret)                        | data     |
 | [azurerm_key_vault_secret.sp_key](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault_secret)                              | data     |
 | [azurerm_key_vault_secret.tenant_id](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault_secret)                           | data     |
-| [databricks_workspace_conf.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/workspace_conf)                                | resource |
+| [databricks_workspace_conf.pat](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/workspace_conf)                                 | resource |
+| [databricks_token.pat](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/token)                                                   | resource |
 | [databricks_ip_access_list.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/ip_access_list)                                | resource |
 | [databricks_sql_global_config.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/sql_global_config)                          | resource |
 | [databricks_sql_endpoint.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/sql_endpoint)                                    | resource |
@@ -332,14 +346,13 @@ No modules.
 | <a name="input_suffix"></a> [suffix](#input\_suffix)                                                                      | Optional suffix that would be added to the end of resources names.                                                                                                             | `string`                                                                                                                                                                                                                                                                                                                                                                                          | " "                                                                                                                                                                                                                                                                     |    no    |
 | <a name="input_external_metastore_id"></a> [external\_metastore\_id](#input\_external\_metastore\_id)                     | Unity Catalog Metastore Id that is located in separate environment. Provide this value to associate Databricks Workspace with target Metastore                                 | `string`                                                                                                                                                                                                                                                                                                                                                                                          | " "                                                                                                                                                                                                                                                                     |    no    |
 | <a name="input_metastore_grants"></a> [metastore\_grants](#input\_metastore\_grants)                                      | Permissions to give on metastore to group                                                                                                                                      | `map(list(string))`                                                                                                                                                                                                                                                                                                                                                                               | {}                                                                                                                                                                                                                                                                      |    no    |
-| <a name="input_secret_scope_object"></a> [secret\_scope\_object](#input\_secret\_scope\_object)                           | List of objects, where 'scope_name' param is a Secret scope name and 'acl' are list of objects with 'principals' and one of allowed 'permission' ('READ', 'WRITE' or 'MANAGE') | <pre>list(object({<br> scope_name = string<br> acl = list(object({<br>   principal  = string<br>   permission = string<br>   }))<br>}))</pre>                                                                                                                                                                                                                                                     | <pre>[{<br>  scope_name = null<br>  acl        = null<br>}]</pre>                                                                                                                                                                                                       |    no    |
 | <a name="input_sp_client_id_secret_name"></a> [sp\_client\_id\_secret\_name](#input\_sp\_client\_id\_secret\_name)  | The name of Azure Key Vault secret that contains ClientID of Service Principal to access in Azure Key Vault   | `string` | n/a | yes |
 | <a name="input_sp_key_secret_name"></a> [sp\_key\_secret\_name](#input\_sp\_key\_secret\_name)  | The name of Azure Key Vault secret that contains client secret of Service Principal to access in Azure Key Vault   | `string` | n/a | yes |
 | <a name="input_secret_scope"></a> [secret\_scope](#input\_secret\_scope)  | Provides an ability to create custom Secret Scope, store secrets in it and assigning ACL for access management   | <pre>list(object({<br>  scope_name = string<br>  acl = optional(list(object({<br>    principal  = string<br>    permission = string<br>  })))<br>  secrets = optional(list(object({<br>    key          = string<br>    string_value = string<br>  })))<br>}))<br></pre> | <pre>default = [{<br>  scope_name = null<br>  acl        = null<br>  secrets    = null<br>}]<br></pre> | yes |
 | <a name="input_key_vault_id"></a> [key\_vault\_id](#input\_key\_vault\_id)  | ID of the Key Vault instance where the Secret resides | `string` | n/a | yes |
 | <a name="input_tenant_id_secret_name"></a> [tenant\_id\_secret\_name](#input\_tenant\_id\_secret\_name)  | The name of Azure Key Vault secret that contains tenant ID secret of Service Principal to access in Azure Key Vault | `string` | n/a | yes |
 | <a name="input_mountpoints"></a> [mountpoints](#input\_mountpoints)  | Mountpoints for databricks | <pre>map(object({<br>  storage_account_name = string<br>  container_name       = string<br>}))<br></pre> |{}| no |
-| <a name="input_custom_cluster_policies"></a> [custom\_cluster\_policies](#input\_custom\_cluster\_policies)  | Provides an ability to create custom cluster policy, assign it to cluster and grant CAN_USE permissions on it to certain custom groups | <pre>list(object({<br>  name       = string<br>  can_use    = list(string)<br>  definition = any<br>  assigned   = bool<br>}))<br></pre> |<pre>[{<br>  name       = null<br>  can_use    = null<br>  definition = null<br>  assigned   = false<br>}]<br></pre>| no |
+| <a name="input_custom_cluster_policies"></a> [custom\_cluster\_policies](#input\_custom\_cluster\_policies)  | Provides an ability to create custom cluster policy, assign it to cluster and grant CAN_USE permissions on it to certain custom groups | <pre>list(object({<br>  name       = string<br>  can_use    = list(string)<br>  definition = any<br>}))<br></pre> |<pre>[{<br>  name       = null<br>  can_use    = null<br>  definition = null<br>}]<br></pre>| no |
 | <a name="input_clusters"></a> [clusters](#input\_clusters)  | Set of objects with parameters to configure Databricks clusters and assign permissions to it for certain custom groups  | <pre>set(object({<br>    cluster_name                 = string<br>    spark_version                = optional(string)<br>    spark_conf                   = optional(map(any))<br>    spark_env_vars               = optional(map(any))<br>    data_security_mode           = optional(string)<br>    node_type_id                 = optional(string)<br>    autotermination_minutes      = optional(number)<br>    min_workers                  = optional(number)<br>    max_workers                  = optional(number)<br>    max_workers                  = optional(number)<br>    availability                 = optional(string)<br>    first_on_demand              = optional(number)<br>    spot_bid_max_price           = optional(number)<br>    cluster_log_conf_destination = optional(string)<br>    permissions = optional(set(object({<br>      group_name       = string<br>      permission_level = string<br>    })), [])<br>}))<br></pre> | <pre>set(object({<br>    cluster_name                 = string<br>    spark_version                = optional(string, "11.3.x-scala2.12")<br>    spark_conf                   = optional(map(any), {})<br>    spark_env_vars               = optional(map(any), {})<br>    data_security_mode           = optional(string, "USER_ISOLATION")<br>    node_type_id                 = optional(string, "Standard_D3_v2")<br>    autotermination_minutes      = optional(number, 30)<br>    min_workers                  = optional(number, 1)<br>    max_workers                  = optional(number, 2)<br>    max_workers                  = optional(number, 2)<br>    availability                 = optional(string, "ON_DEMAND_AZURE")<br>    first_on_demand              = optional(number, 0)<br>    spot_bid_max_price           = optional(number, 1)<br>    cluster_log_conf_destination = optional(string, null)<br>    permissions = optional(set(object({<br>      group_name       = string<br>      permission_level = string<br>    })), [])<br>}))<br></pre> | no |
 
 
@@ -351,6 +364,7 @@ No modules.
 | <a name="output_sql_endpoint_jdbc_url"></a> [sql\_endpoint\_jdbc\_url](#output\_sql\_endpoint\_jdbc\_url)                     | JDBC connection string of SQL Endpoint  |
 | <a name="output_sql_endpoint_data_source_id"></a> [sql\_endpoint\_data\_source\_id](#output\_sql\_endpoint\_data\_source\_id) | ID of the data source for this endpoint |
 | <a name="output_metastore_id"></a> [metastore\_id](#output\_metastore\_id)                                                    | Unity Catalog Metastore Id              |
+| <a name="output_token"></a> [token](#output\_token)                                                                           | Databricks Personal Authorization Token |
 <!-- END_TF_DOCS -->
 
 ## License

cluster.tf

@@ -35,3 +35,13 @@ resource "databricks_cluster" "cluster" {
     ]
   }
 }
+
+resource "databricks_cluster_policy" "this" {
+  for_each = {
+    for param in var.custom_cluster_policies : (param.name) => param.definition
+    if param.definition != null
+  }
+
+  name       = each.key
+  definition = jsonencode(each.value)
+}

main.tf

@@ -26,6 +26,11 @@ resource "databricks_workspace_conf" "this" {
   }
 }
 
+resource "databricks_token" "pat" {
+  comment          = "Terraform Provisioning"
+  lifetime_seconds = var.pat_token_lifetime_seconds
+}
+
 resource "databricks_ip_access_list" "this" {
   count = local.ip_rules == null ? 0 : 1
 

outputs.tf

@@ -12,3 +12,8 @@ output "metastore_id" {
   value       = var.create_metastore ? databricks_metastore.this[0].id : ""
   description = "Unity Catalog Metastore Id"
 }
+
+output "token" {
+  value       = databricks_token.pat.token_value
+  description = "Databricks Personal Authorization Token"
+}

permissions.tf

@@ -1,21 +1,18 @@
 locals {
-  secrets_acl_objects_list = flatten([for param in var.secret_scope_object : [
+  secret_scope_object =  {
+  value = [for param in var.secret_scope : {
+    scope_name = databricks_secret_scope.this[param.scope_name].name
+    acl        = param.acl
+  } if param.acl != null] 
+  }
+
+  secrets_acl_objects_list = flatten([for param in local.secret_scope_object : [
     for permission in param.acl : {
       scope = param.scope_name, principal = permission.principal, permission = permission.permission
     }] if param.acl != null
   ])
 }
 
-resource "databricks_cluster_policy" "this" {
-  for_each = {
-    for param in var.custom_cluster_policies : (param.name) => param.definition
-    if param.definition != null
-  }
-
-  name       = each.key
-  definition = jsonencode(each.value)
-}
-
 resource "databricks_permissions" "clusters" {
   for_each = {
     for v in var.clusters : (v.cluster_name) => v

variables.tf

@@ -48,7 +48,6 @@ variable "iam" {
     user                       = optional(list(string))
     service_principal          = optional(list(string))
     entitlements               = optional(list(string))
-    default_cluster_permission = optional(string)
   }))
   description = "Used to create workspace group. Map of group name and its parameters, such as users and service principals added to the group. Also possible to configure group entitlements."
   default     = {}
@@ -150,22 +149,6 @@ variable "metastore_grants" {
   }
 }
 
-# Secret Scope ACLs variables
-variable "secret_scope_object" {
-  type = list(object({
-    scope_name = string
-    acl = list(object({
-      principal  = string
-      permission = string
-    }))
-  }))
-  description = "List of objects, where 'scope_name' param is a Secret scope name and 'acl' are list of objects with 'principals' and one of allowed 'permission' ('READ', 'WRITE' or 'MANAGE')"
-  default = [{
-    scope_name = null
-    acl        = null
-  }]
-}
-
 variable "sp_client_id_secret_name" {
   type        = string
   description = "The name of Azure Key Vault secret that contains ClientID of Service Principal to access in Azure Key Vault"
@@ -226,25 +209,18 @@ variable "custom_cluster_policies" {
     name       = string
     can_use    = list(string)
     definition = any
-    assigned   = bool
   }))
   description = <<-EOT
 Provides an ability to create custom cluster policy, assign it to cluster and grant CAN_USE permissions on it to certain custom groups
 name - name of custom cluster policy to create
 can_use - list of string, where values are custom group names, there groups have to be created with Terraform;
 definition - JSON document expressed in Databricks Policy Definition Language. No need to call 'jsonencode()' function on it when providing a value;
-assigned - boolean flag which assigns policy to default 'shared autoscaling' cluster, only single custom policy could be assigned;
 EOT
   default = [{
     name       = null
     can_use    = null
     definition = null
-    assigned   = false
   }]
-  validation {
-    condition     = length([for policy in var.custom_cluster_policies : policy.assigned if policy.assigned]) <= 1
-    error_message = "Only single cluster policy assignment allowed. Please set 'assigned' parameter to 'true' for exact one or none policy"
-  }
 }
 
 variable "clusters" {