feat: added credential passthrough

Author: Leonid_Frolov1

cluster.tf

@@ -3,7 +3,13 @@ resource "databricks_cluster" "cluster" {
 
   cluster_name            = each.value.cluster_name
   spark_version           = each.value.spark_version
-  spark_conf              = each.value.spark_conf
+  spark_conf              = each.value.enabled_adls_passthrought ? merge(each.value.spark_conf,
+    {
+      "spark.databricks.cluster.profile" : "serverless",
+      "spark.databricks.repl.allowedLanguages" : "python,sql",
+      "spark.databricks.passthrough.enabled" : "true",
+      "spark.databricks.pyspark.enableProcessIsolation" : "true"
+    }) : each.value.spark_conf
   spark_env_vars          = each.value.spark_env_vars
   data_security_mode      = each.value.data_security_mode
   node_type_id            = each.value.node_type_id

mount.tf

@@ -3,7 +3,10 @@ resource "databricks_mount" "adls" {
 
   name = each.key
   uri  = "abfss://${each.value["container_name"]}@${each.value["storage_account_name"]}.dfs.core.windows.net"
-  extra_configs = {
+  extra_configs = var.mount_adls_passthrough ? {
+    "fs.azure.account.auth.type" : "CustomAccessToken",
+    "fs.azure.account.custom.token.provider.class" : "{{sparkconf/spark.databricks.passthrough.adls.gen2.tokenProviderClassName}}"
+    } : {
     "fs.azure.account.auth.type" : "OAuth",
     "fs.azure.account.oauth.provider.type" : "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider",
     "fs.azure.account.oauth2.client.id" : data.azurerm_key_vault_secret.sp_client_id.value,

outputs.tf

@@ -16,4 +16,5 @@ output "metastore_id" {
 output "token" {
   value       = databricks_token.pat.token_value
   description = "Databricks Personal Authorization Token"
+  sensitive = true
 }

variables.tf

@@ -228,6 +228,7 @@ variable "clusters" {
     cluster_name                 = string
     spark_version                = optional(string, "11.3.x-scala2.12")
     spark_conf                   = optional(map(any), {})
+    enabled_adls_passthrought    = optional(bool, false)
     spark_env_vars               = optional(map(any), {})
     data_security_mode           = optional(string, "USER_ISOLATION")
     node_type_id                 = optional(string, "Standard_D3_v2")
@@ -252,3 +253,9 @@ variable "pat_token_lifetime_seconds" {
   description = "The lifetime of the token, in seconds. If no lifetime is specified, the token remains valid indefinitely"
   default     = 315569520
 }
+
+variable "mount_adls_passthrough" {
+  type        = bool
+  description = "Boolean flag for Unity Catalog Metastore current in this environment. One Metastore per region"
+  default     = false
+}