Merge pull request #12 from data-platform-hq/secret-scope-resources

feat: custom secret scope

Author: owlleg6

README.md

@@ -10,14 +10,14 @@ Terraform module used for Databricks Workspace configuration and Resources creat
 | ---------------------------------------------------------------------------- | --------- |
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform)    | >= 1.0.0  |
 | <a name="requirement_azurerm"></a> [azurerm](#requirement\_azurerm)          | >= 3.40.0 |
-| <a name="requirement_databricks"></a> [databricks](#requirement\_databricks) | >= 1.8.0  |
+| <a name="requirement_databricks"></a> [databricks](#requirement\_databricks) | >= 1.9.2  |
 
 ## Providers
 
 | Name                                                                   | Version |
 | ---------------------------------------------------------------------- | ------- |
 | <a name="provider_azurerm"></a> [azurerm](#provider\_azurerm)          | 3.40.0  |
-| <a name="provider_databricks"></a> [databricks](#provider\_databricks) | 1.8.0   |
+| <a name="provider_databricks"></a> [databricks](#provider\_databricks) | 1.9.2   |
 
 ## Modules
 
@@ -36,6 +36,8 @@ No modules.
 | [databricks_cluster_policy.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/cluster_policy)      | resource |
 | [databricks_cluster.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/cluster)                    | resource |
 | [databricks_mount.adls](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/mount)                        | resource |
+| [databricks_secret_scope.main](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/secret_scope)          | resource |
+| [databricks_secret.main](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/secret)          | resource |
 | [databricks_secret_scope.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/secret_scope)          | resource |
 | [databricks_secret.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/secret)                      | resource |
 
@@ -50,24 +52,24 @@ No modules.
 | <a name="input_key_vault_id"></a> [key\_vault\_id](#input\_key\_vault\_id) | ID of the Key Vault instance where the Secret resides | `string` | n/a                                                                                                              | yes |
 | <a name="input_sku"></a> [sku](#input\_sku) | The sku to use for the Databricks Workspace: [standard \ premium \ trial] | `string` | "standard"                                                                                                       |    no    |
 | <a name="input_pat_token_lifetime_seconds"></a> [pat\_token\_lifetime\_seconds](#input\_pat\_token\_lifetime\_seconds) | The lifetime of the token, in seconds. If no lifetime is specified, the token remains valid indefinitely | `number` | 315569520                                                                                                        | no |
+| <a name="input_users"></a> [users](#input\_users)| List of users to access Databricks | `list(string)` | [] | no |
+| <a name="input_permissions"></a> [permissions](#input\_permissions)| Databricks Workspace permission maps | `list(map(string))` | <pre> [{   <br>   object_id = null   <br>   role      = null <br> }] </pre> | no |
+| <a name="input_custom_cluster_policies"></a> [custom\_cluster\_policies](#input\_custom\_cluster\_policies)  | Provides an ability to create custom cluster policy, assign it to cluster and grant CAN_USE permissions on it to certain custom groups | <pre>list(object({<br>  name       = string<br>  can_use    = list(string)<br>  definition = any<br>  assigned   = bool<br>}))</pre>  | <pre>[{<br>  name       = null<br>  can_use    = null<br>  definition = null<br>  assigned   = false<br>}]</pre> | no |
 | <a name="input_cluster_nodes_availability"></a> [cluster\_nodes\_availability](#input\_cluster\_nodes\_availability) | Availability type used for all subsequent nodes past the first_on_demand ones: [SPOT_AZURE \  SPOT_WITH_FALLBACK_AZURE \  ON_DEMAND_AZURE] | `string` | null                                                                                                             | no |
 | <a name="input_first_on_demand"></a> [first\_on\_demand](#input\_first\_on\_demand)| The first first_on_demand nodes of the cluster will be placed on on-demand instances: [[ \:number ]] | `number` | 0                                                                                                                | no |
 | <a name="input_spot_bid_max_price"></a> [spot\_bid\_max\_price](#input\_spot\_bid\_max\_price) | The max price for Azure spot instances. Use -1 to specify lowest price | `number` | -1                                                                                                               | no |
 | <a name="input_autotermination_minutes"></a> [autotermination\_minutes](#input\_autotermination\_minutes) | Automatically terminate the cluster after being inactive for this time in minutes. If not set, Databricks won't automatically terminate an inactive cluster. If specified, the threshold must be between 10 and 10000 minutes. You can also set this value to 0 to explicitly disable automatic termination | `number`| 15                                                                                                               | no |
 | <a name="input_min_workers"></a> [min\_workers](#input\_min\_workers)| The minimum number of workers to which the cluster can scale down when underutilized. It is also the initial number of workers the cluster will have after creation | `number` | 1                                                                                                                | no |
 | <a name="input_max_workers"></a> [max\_workers](#input\_max\_workers) | The maximum number of workers to which the cluster can scale up when overloaded. max_workers must be strictly greater than min_workers  | `number` | 2                                                                                                                | no |
-| <a name="input_users"></a> [users](#input\_users)| List of users to access Databricks | `list(string)` | []                                                                                                               | no |
-| <a name="input_secrets"></a> [secrets](#input\_secrets) | Map of secrets to create in Databricks | `map(any)`| {}                                                                                                               | no |
-| <a name="input_use_local_secret_scope"></a> [use\_local\_secret\_scope](#input\_use\_local\_secret\_scope) | Create databricks secret scope and create secrets | `bool` | false                                                                                                            | no |
-| <a name="input_permissions"></a> [permissions](#input\_permissions)| Databricks Workspace permission maps | `list(map(string))` | <pre> [{   <br>   object_id = null   <br>   role      = null <br> }] </pre>                                      | no |
-| <a name="input_custom_cluster_policies"></a> [custom\_cluster\_policies](#input\_custom\_cluster\_policies)  | Provides an ability to create custom cluster policy, assign it to cluster and grant CAN_USE permissions on it to certain custom groups | <pre>list(object({<br>  name       = string<br>  can_use    = list(string)<br>  definition = any<br>  assigned   = bool<br>}))</pre>  | <pre>[{<br>  name       = null<br>  can_use    = null<br>  definition = null<br>  assigned   = false<br>}]</pre> | no |
-| <a name="input_data_security_mode"></a> [data\_security\_mode](#input\_data\_security\_mode) | Security features of the cluster  | `string` | "NONE"                                                                                                           | no |
+| <a name="input_data_security_mode"></a> [data\_security\_mode](#input\_data\_security\_mode) | Security features of the cluster  | `string` | "NONE" | no |
 | <a name="input_spark_version"></a> [spark\_version](#input\_spark\_version)  | Runtime version | `string` | "11.3.x-scala2.12"                                                                                               | no |
 | <a name="input_spark_conf"></a> [spark\_conf](#input\_spark\_conf)| Map with key-value pairs to fine-tune Spark clusters, where you can provide custom Spark configuration properties in a cluster configuration. | `map(any)`  | {}                                                                                                               | no |
 | <a name="input_spark_env_vars"></a> [spark_env_vars](#input\_spark_env_vars)| Map with environment variable key-value pairs to fine-tune Spark clusters. Key-value pairs of the form (X,Y) are exported (i.e., X='Y') while launching the driver and workers.| `map(any)`| {}                                                                                                               | no |
-| <a name="input_cluster_log_conf_destination"></a> [cluster\_log\_conf\_destination](#input\_cluster\_log\_conf\_destination)  | Provide a dbfs location, example 'dbfs:/cluster-logs', to push all cluster logs to certain location | `string` | ""                                                                                                               | no |
-| <a name="input_node_type"></a> [spark\_node\_type](#input\_node\_type)| Databricks_node_type id | `string` | "Standard_D3_v2"                                                                                                 | no |
+| <a name="input_cluster_log_conf_destination"></a> [cluster\_log\_conf\_destination](#input\_cluster\_log\_conf\_destination)  | Provide a dbfs location, example 'dbfs:/cluster-logs', to push all cluster logs to certain location | `string` | " "                                                                                                               | no |
+| <a name="input_node_type"></a> [node\_type](#input\_node\_type)| Databricks_node_type id | `string` | "Standard_D3_v2"                                                                                                 | no |
 | <a name="input_mountpoints"></a> [mountpoints](#input\_mountpoints) | Mountpoints for databricks | `map(any)`| null                                                                                                             | no |
+| <a name="input_secret_scope"></a> [secret\_scope](#input\_secret\_scope) | Provides an ability to create custom Secret Scope, store secrets in it and assigning ACL for access management | <pre>list(object({<br>  scope_name = string<br>  acl = optional(list(object({<br>    principal  = string<br>    permission = string<br>  secrets = optional(list(object({<br>    key          = string<br>    string_value = string<br>})))<br></pre>  |  <pre>default = [{<br>  scope_name = null<br>  acl        = null<br>  can_use    = null<br>  secrets    = null<br>}]</pre>  | no |
+
 
 
 ## Outputs
@@ -77,6 +79,7 @@ No modules.
 | <a name="output_token"></a> [token](#output\_token)                  | Databricks Personal Authorization Token |
 | <a name="output_cluster_id"></a> [cluster\_id](#output\_cluster\_id) | Databricks Cluster Id                   |
 | <a name="output_cluster_policies_object"></a> [cluster\_policies\_object](#output\_cluster\_policies\_object) | Databricks Cluster Policies object map |
+| <a name="output_secret_scope_object"></a> [secret_scope\_object](#output\_secret_scope\_object) | Databricks-managed Secret Scope object map to create ACLs |
 <!-- END_TF_DOCS -->
 
 ## License

main.tf

@@ -13,28 +13,21 @@ data "azurerm_key_vault_secret" "tenant_id" {
   key_vault_id = var.key_vault_id
 }
 
-locals {
-  secrets = merge(var.secrets, {
-    (var.sp_client_id_secret_name) = { value = data.azurerm_key_vault_secret.sp_client_id.value }
-    (var.sp_key_secret_name)       = { value = data.azurerm_key_vault_secret.sp_key.value }
-  })
-}
-
 resource "databricks_token" "pat" {
   comment          = "Terraform Provisioning"
   lifetime_seconds = var.pat_token_lifetime_seconds
 }
 
 resource "databricks_user" "this" {
-  for_each  = var.sku == "standard" ? toset(var.users) : []
+  for_each  = var.sku == "premium" ? [] : toset(var.users)
   user_name = each.value
   lifecycle { ignore_changes = [external_id] }
 }
 
 resource "azurerm_role_assignment" "this" {
   for_each = {
-    for permision in var.permissions : "${permision.object_id}-${permision.role}" => permision
-    if permision.role != null
+    for permission in var.permissions : "${permission.object_id}-${permission.role}" => permission
+    if permission.role != null
   }
   scope                = var.workspace_id
   role_definition_name = each.value.role

mount.tf

@@ -1,8 +1,3 @@
-locals {
-  secret_scope_name = var.use_local_secret_scope ? databricks_secret_scope.this[0].name : "main"
-  mount_secret_name = var.use_local_secret_scope ? databricks_secret.this[var.sp_key_secret_name].config_reference : "{{secrets/${local.secret_scope_name}/${data.azurerm_key_vault_secret.sp_key.name}}}"
-}
-
 resource "databricks_mount" "adls" {
   for_each = var.mountpoints
 
@@ -12,10 +7,10 @@ resource "databricks_mount" "adls" {
     "fs.azure.account.auth.type" : "OAuth",
     "fs.azure.account.oauth.provider.type" : "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider",
     "fs.azure.account.oauth2.client.id" : data.azurerm_key_vault_secret.sp_client_id.value,
-    "fs.azure.account.oauth2.client.secret" : local.mount_secret_name,
+    "fs.azure.account.oauth2.client.secret" : databricks_secret.main[data.azurerm_key_vault_secret.sp_key.name].config_reference,
     "fs.azure.account.oauth2.client.endpoint" : "https://login.microsoftonline.com/${data.azurerm_key_vault_secret.tenant_id.value}/oauth2/token",
     "fs.azure.createRemoteFileSystemDuringInitialization" : "false",
     "spark.databricks.sqldw.jdbc.service.principal.client.id" : data.azurerm_key_vault_secret.sp_client_id.value,
-    "spark.databricks.sqldw.jdbc.service.principal.client.secret" : local.mount_secret_name,
+    "spark.databricks.sqldw.jdbc.service.principal.client.secret" : databricks_secret.main[data.azurerm_key_vault_secret.sp_key.name].config_reference
   }
 }

outputs.tf

@@ -16,3 +16,11 @@ output "cluster_policies_object" {
   } if policy.definition != null]
   description = "Databricks Cluster Policies object map"
 }
+
+output "secret_scope_object" {
+  value = [for param in var.secret_scope : {
+    scope_name = databricks_secret_scope.this[param.scope_name].name
+    acl        = param.acl
+  } if param.acl != null]
+  description = "Databricks-managed Secret Scope object map to create ACLs"
+}

secrets.tf

@@ -1,16 +1,74 @@
-resource "databricks_secret_scope" "this" {
-  count = var.use_local_secret_scope ? 1 : 0
+locals {
+  sp_secrets = {
+    (var.sp_client_id_secret_name) = { value = data.azurerm_key_vault_secret.sp_client_id.value }
+    (var.sp_key_secret_name)       = { value = data.azurerm_key_vault_secret.sp_key.value }
+  }
+
+  secrets_objects_list = flatten([for param in var.secret_scope : [
+    for secret in param.secrets : {
+      scope_name = param.scope_name, key = secret.key, string_value = secret.string_value
+    }] if param.secrets != null
+  ])
+}
 
+# Secret Scope with SP secrets for mounting Azure Data Lake Storage
+resource "databricks_secret_scope" "main" {
   name                     = "main"
-  initial_manage_principal = "users"
+  initial_manage_principal = var.sku == "premium" ? null : "users"
 }
 
-resource "databricks_secret" "this" {
-  for_each = var.use_local_secret_scope ? local.secrets : {}
+resource "databricks_secret" "main" {
+  for_each = local.sp_secrets
 
   key          = each.key
   string_value = each.value["value"]
-  scope        = databricks_secret_scope.this[0].id
+  scope        = databricks_secret_scope.main.id
+}
+
+# Custom additional Databricks Secret Scope
+resource "databricks_secret_scope" "this" {
+  for_each = {
+    for param in var.secret_scope : (param.scope_name) => param
+    if param.scope_name != null
+  }
 
-  depends_on = [databricks_secret_scope.this]
+  name                     = each.key
+  initial_manage_principal = var.sku == "premium" ? null : "users"
 }
+
+resource "databricks_secret" "this" {
+  for_each = { for entry in local.secrets_objects_list : "${entry.scope_name}.${entry.key}" => entry }
+
+  key          = each.value.key
+  string_value = each.value.string_value
+  scope        = databricks_secret_scope.this[each.value.scope_name].id
+}
+
+# At the nearest future, Azure will allow acquiring AAD tokens by service principals,
+# thus providing an ability to create Azure backed Key Vault with Terraform
+# https://github.com/databricks/terraform-provider-databricks/pull/1965
+
+## Azure Key Vault-backed Scope
+#resource "azurerm_key_vault_access_policy" "databricks" {
+#  count = var.key_vault_secret_scope.key_vault_id != null ? 1 : 0
+
+#  key_vault_id = var.key_vault_secret_scope.key_vault_id
+#  object_id    = "9b38785a-6e08-4087-a0c4-20634343f21f" # Global 'AzureDatabricks' SP object id
+#  tenant_id    = data.azurerm_key_vault_secret.tenant_id.value
+#
+#  secret_permissions = [
+#    "Get",
+#    "List",
+#  ]
+#}
+#
+#resource "databricks_secret_scope" "external" {
+#  count = var.key_vault_secret_scope.key_vault_id != null ? 1 : 0
+#
+#  name = "external"
+#  keyvault_metadata {
+#    resource_id = var.key_vault_secret_scope.key_vault_id
+#    dns_name    = var.key_vault_secret_scope.dns_name
+#  }
+#  depends_on = [azurerm_key_vault_access_policy.databricks]
+#}