feat: mount feature optionality

Author: owlleg6

main.tf

@@ -3,25 +3,6 @@ locals {
   suffix   = length(var.suffix) == 0 ? "" : "-${var.suffix}"
 }
 
-data "azurerm_key_vault_secret" "sp_client_id" {
-  count = var.mount_enabled ? 1 : 0
-
-  name         = var.sp_client_id_secret_name
-  key_vault_id = var.key_vault_id
-}
-
-data "azurerm_key_vault_secret" "sp_key" {
-  count = var.mount_enabled ? 1 : 0
-
-  name         = var.sp_key_secret_name
-  key_vault_id = var.key_vault_id
-}
-
-data "azurerm_key_vault_secret" "tenant_id" {
-  name         = var.tenant_id_secret_name
-  key_vault_id = var.key_vault_id
-}
-
 resource "databricks_workspace_conf" "this" {
   count = local.ip_rules == null ? 0 : 1
 

mount.tf

@@ -10,11 +10,9 @@ resource "databricks_mount" "adls" {
     } : {
     "fs.azure.account.auth.type" : "OAuth",
     "fs.azure.account.oauth.provider.type" : "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider",
-    "fs.azure.account.oauth2.client.id" : data.azurerm_key_vault_secret.sp_client_id[0].value,
-    "fs.azure.account.oauth2.client.secret" : databricks_secret.main[data.azurerm_key_vault_secret.sp_key[0].name].config_reference,
-    "fs.azure.account.oauth2.client.endpoint" : "https://login.microsoftonline.com/${data.azurerm_key_vault_secret.tenant_id.value}/oauth2/token",
+    "fs.azure.account.oauth2.client.id" : var.mount_service_principal_client_id,
+    "fs.azure.account.oauth2.client.secret" : databricks_secret.main["mount-sp-secret"].config_reference,
+    "fs.azure.account.oauth2.client.endpoint" : "https://login.microsoftonline.com/${var.mount_service_principal_tenant_id}/oauth2/token",
     "fs.azure.createRemoteFileSystemDuringInitialization" : "false",
-    "spark.databricks.sqldw.jdbc.service.principal.client.id" : data.azurerm_key_vault_secret.sp_client_id[0].value,
-    "spark.databricks.sqldw.jdbc.service.principal.client.secret" : databricks_secret.main[data.azurerm_key_vault_secret.sp_key[0].name].config_reference
   }
 }

secrets.tf

@@ -1,8 +1,8 @@
 locals {
-  sp_secrets = var.mount_enabled ? {
-    (var.sp_client_id_secret_name) = { value = data.azurerm_key_vault_secret.sp_client_id[0].value }
-    (var.sp_key_secret_name)       = { value = data.azurerm_key_vault_secret.sp_key[0].value }
-  } : {}
+  mount_sp_secrets = {
+    mount-sp-client-id = { value = var.mount_service_principal_client_id }
+    mount-sp-secret    = { value = var.mount_service_principal_secret }
+  }
 
   secrets_objects_list = flatten([for param in var.secret_scope : [
     for secret in param.secrets : {
@@ -13,16 +13,25 @@ locals {
 
 # Secret Scope with SP secrets for mounting Azure Data Lake Storage
 resource "databricks_secret_scope" "main" {
+  count = var.mount_enabled ? 1 : 0
+
   name                     = "main"
   initial_manage_principal = null
 }
 
 resource "databricks_secret" "main" {
-  for_each = local.sp_secrets
+  for_each = var.mount_enabled ? local.mount_sp_secrets : {}
 
   key          = each.key
   string_value = each.value["value"]
-  scope        = databricks_secret_scope.main.id
+  scope        = databricks_secret_scope.main[0].id
+
+  lifecycle {
+    precondition {
+      condition     = var.mount_enabled ? length(compact([var.mount_service_principal_client_id, var.mount_service_principal_secret, var.mount_service_principal_tenant_id])) == 3 : true
+      error_message = "To mount ADLS Storage, please provide prerequisite Service Principal values - 'mount_service_principal_object_id', 'mount_service_principal_secret', 'mount_service_principal_tenant_id'."
+    }
+  }
 }
 
 # Custom additional Databricks Secret Scope
@@ -52,8 +61,8 @@ resource "azurerm_key_vault_access_policy" "databricks" {
   } : {}
 
   key_vault_id = each.value.key_vault_id
-  object_id    = "9b38785a-6e08-4087-a0c4-20634343f21f" # Global 'AzureDatabricks' SP object id
-  tenant_id    = data.azurerm_key_vault_secret.tenant_id.value
+  object_id    = var.global_databricks_sp_object_id
+  tenant_id    = each.value.tenant_id
 
   secret_permissions = [
     "Get",

variables.tf

@@ -76,18 +76,6 @@ variable "sql_endpoint" {
   default     = []
 }
 
-variable "sp_client_id_secret_name" {
-  type        = string
-  description = "The name of Azure Key Vault secret that contains ClientID of Service Principal to access in Azure Key Vault"
-  default     = ""
-}
-
-variable "sp_key_secret_name" {
-  type        = string
-  description = "The name of Azure Key Vault secret that contains client secret of Service Principal to access in Azure Key Vault"
-  default     = ""
-}
-
 # Secret Scope variables
 variable "secret_scope" {
   type = list(object({
@@ -114,12 +102,13 @@ EOT
   }]
 }
 
-variable "key_vault_id" {
+# Azure Key Vault-backed Secret Scope
+variable "global_databricks_sp_object_id" {
   type        = string
-  description = "ID of the Key Vault instance where the Secret resides"
+  description = "Global 'AzureDatabricks' SP object id. Used to create Key Vault Access Policy for Secret Scope"
+  default     = "9b38785a-6e08-4087-a0c4-20634343f21f"
 }
 
-# Azure Key Vault-backed Secret Scope
 variable "create_databricks_access_policy_to_key_vault" {
   type        = bool
   description = "Boolean flag to enable creation of Key Vault Access Policy for Databricks Global Service Principal."
@@ -128,28 +117,15 @@ variable "create_databricks_access_policy_to_key_vault" {
 
 variable "key_vault_secret_scope" {
   type = list(object({
-    name         = optional(string)
-    key_vault_id = optional(string)
-    dns_name     = optional(string)
+    name         = string
+    key_vault_id = string
+    dns_name     = string
+    tenant_id    = string
   }))
   description = "Object with Azure Key Vault parameters required for creation of Azure-backed Databricks Secret scope"
   default     = []
 }
 
-variable "tenant_id_secret_name" {
-  type        = string
-  description = "The name of Azure Key Vault secret that contains tenant ID secret of Service Principal to access in Azure Key Vault"
-}
-
-variable "mountpoints" {
-  type = map(object({
-    storage_account_name = string
-    container_name       = string
-  }))
-  description = "Mountpoints for databricks"
-  default     = {}
-}
-
 variable "custom_cluster_policies" {
   type = list(object({
     name       = string
@@ -201,20 +177,48 @@ variable "pat_token_lifetime_seconds" {
   default     = 315569520
 }
 
-variable "mount_adls_passthrough" {
+# Mount ADLS Gen2 Filesystem
+variable "mount_enabled" {
   type        = bool
-  description = "Boolean flag to use mount options for credentals passthrough. Should be used with mount_cluster_name, specified cluster should have option cluster_conf_passthrought == true"
+  description = "Boolean flag that determines whether mount point for storage account filesystem is created"
   default     = false
 }
 
-variable "mount_cluster_name" {
+variable "mount_service_principal_client_id" {
   type        = string
-  description = "Name of the cluster that will be used during storage mounting. If mount_adls_passthrough == true, cluster should also have option cluster_conf_passthrought == true"
+  description = "Application(client) Id of Service Principal used to perform storage account mounting"
+  default     = null
+}
+variable "mount_service_principal_secret" {
+  type        = string
+  description = "Service Principal Secret used to perform storage account mounting"
   default     = null
+  sensitive   = true
 }
 
-variable "mount_enabled" {
+variable "mount_service_principal_tenant_id" {
+  type        = string
+  description = "Service Principal tenant id used to perform storage account mounting"
+  default     = null
+}
+
+variable "mountpoints" {
+  type = map(object({
+    storage_account_name = string
+    container_name       = string
+  }))
+  description = "Mountpoints for databricks"
+  default     = {}
+}
+
+variable "mount_adls_passthrough" {
   type        = bool
-  description = "Boolean flag to enable ADLS mount to Databricks"
+  description = "Boolean flag to use mount options for credentials passthrough. Should be used with mount_cluster_name, specified cluster should have option cluster_conf_passthrought == true"
   default     = false
 }
+
+variable "mount_cluster_name" {
+  type        = string
+  description = "Name of the cluster that will be used during storage mounting. If mount_adls_passthrough == true, cluster should also have option cluster_conf_passthrought == true"
+  default     = null
+}