feat: added databricks runtime module

Author: owlleg6

main.tf

@@ -0,0 +1,69 @@
+data "azurerm_key_vault_secret" "sp_client_id" {
+  name         = var.sp_client_id_secret_name
+  key_vault_id = var.key_vault_id
+}
+
+data "azurerm_key_vault_secret" "sp_key" {
+  name         = var.sp_key_secret_name
+  key_vault_id = var.key_vault_id
+}
+
+data "azurerm_key_vault_secret" "tenant_id" {
+  name         = var.tenant_id_secret_name
+  key_vault_id = var.key_vault_id
+}
+
+locals {
+  secrets = merge(var.secrets, {
+    (var.sp_client_id_secret_name) = { value = data.azurerm_key_vault_secret.sp_client_id.value }
+    (var.sp_key_secret_name)       = { value = data.azurerm_key_vault_secret.sp_key.value }
+  })
+  secret_scope_name = var.use_local_secret_scope ? databricks_secret_scope.this[0].name : "main"
+  mount_secret_name = var.use_local_secret_scope ? databricks_secret.this[var.sp_key_secret_name].key : data.azurerm_key_vault_secret.sp_key.name
+}
+
+resource "databricks_token" "pat" {
+  comment          = "Terraform Provisioning"
+  lifetime_seconds = var.pat_token_lifetime_seconds
+}
+
+resource "databricks_user" "this" {
+  for_each  = var.sku == "standard" ? toset(var.users) : []
+  user_name = each.value
+  lifecycle { ignore_changes = [external_id] }
+}
+
+resource "azurerm_role_assignment" "this" {
+  for_each = {
+    for permision in var.permissions : "${permision.object_id}-${permision.role}" => permision
+    if permision.role != null
+  }
+  scope                = var.workspace_id
+  role_definition_name = each.value.role
+  principal_id         = each.value.object_id
+}
+
+resource "databricks_cluster" "this" {
+  cluster_name  = "shared autoscaling"
+  spark_version = var.spark_version
+
+  node_type_id            = var.node_type
+  autotermination_minutes = var.autotermination_minutes
+
+  autoscale {
+    min_workers = var.min_workers
+    max_workers = var.max_workers
+  }
+
+  azure_attributes {
+    availability       = var.cluster_nodes_availability
+    first_on_demand    = var.first_on_demand
+    spot_bid_max_price = var.spot_bid_max_price
+  }
+
+  lifecycle {
+    ignore_changes = [
+      state
+    ]
+  }
+}

mount.tf

@@ -0,0 +1,19 @@
+resource "databricks_mount" "adls" {
+  for_each = var.mountpoints
+
+  cluster_id = databricks_cluster.this.id
+  name       = each.key
+  uri        = "abfss://${each.value["container_name"]}@${each.value["storage_account_name"]}.dfs.core.windows.net/${each.value["root_path"]}"
+  extra_configs = {
+    "fs.azure.account.auth.type" : "OAuth",
+    "fs.azure.account.oauth.provider.type" : "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider",
+    "fs.azure.account.oauth2.client.id" : data.azurerm_key_vault_secret.sp_client_id.value,
+    "fs.azure.account.oauth2.client.secret" : "{{secrets/${local.secret_scope_name}/${data.azurerm_key_vault_secret.sp_key.name}}}",
+    "fs.azure.account.oauth2.client.secret" : "{{secrets/${local.secret_scope_name}/${local.mount_secret_name}}}",
+    "fs.azure.account.oauth2.client.endpoint" : "https://login.microsoftonline.com/${data.azurerm_key_vault_secret.tenant_id.value}/oauth2/token",
+    "fs.azure.createRemoteFileSystemDuringInitialization" : "false",
+    "spark.databricks.sqldw.jdbc.service.principal.client.id" : data.azurerm_key_vault_secret.sp_client_id.value,
+    "spark.databricks.sqldw.jdbc.service.principal.client.secret" : "{{secrets/${local.secret_scope_name}/${data.azurerm_key_vault_secret.sp_key.name}}}",
+    "spark.databricks.sqldw.jdbc.service.principal.client.secret" : "{{secrets/${local.secret_scope_name}/${local.mount_secret_name}}}",
+  }
+}

outputs.tf

@@ -0,0 +1,9 @@
+output "token" {
+  value       = databricks_token.pat.token_value
+  description = "Databricks Personal Authorization Token"
+}
+
+output "cluster_id" {
+  value       = databricks_cluster.this.id
+  description = "Databricks Cluster Id"
+}

secrets.tf

@@ -0,0 +1,14 @@
+resource "databricks_secret_scope" "this" {
+  count = var.use_local_secret_scope ? 1 : 0
+
+  name                     = "main"
+  initial_manage_principal = "users"
+}
+
+resource "databricks_secret" "this" {
+  for_each = var.use_local_secret_scope ? local.secrets : {}
+
+  key          = each.key
+  string_value = each.value["value"]
+  scope        = databricks_secret_scope.this[0].id
+}

variables.tf

@@ -0,0 +1,120 @@
+variable "workspace_id" {
+  type        = string
+  description = "Databricks Workspace ID"
+}
+
+variable "sp_client_id_secret_name" {
+  type        = string
+  description = "The name of Azure Key Vault secret that contains ClientID of Service Principal to access in Azure Key Vault"
+}
+
+variable "sp_key_secret_name" {
+  type        = string
+  description = "The name of Azure Key Vault secret that contains client secret of Service Principal to access in Azure Key Vault"
+}
+
+variable "tenant_id_secret_name" {
+  type        = string
+  description = "The name of Azure Key Vault secret that contains tenant ID secret of Service Principal to access in Azure Key Vault"
+}
+
+variable "key_vault_id" {
+  type        = string
+  description = "ID of the Key Vault instance where the Secret resides"
+}
+
+# Optional
+variable "sku" {
+  type        = string
+  description = "The sku to use for the Databricks Workspace: [standard|premium|trial]"
+  default     = "standard"
+}
+
+variable "pat_token_lifetime_seconds" {
+  type        = number
+  description = "The lifetime of the token, in seconds. If no lifetime is specified, the token remains valid indefinitely"
+  default     = 315569520
+}
+
+variable "cluster_nodes_availability" {
+  type        = string
+  description = "Availability type used for all subsequent nodes past the first_on_demand ones: [SPOT_AZURE|SPOT_WITH_FALLBACK_AZURE|ON_DEMAND_AZURE]"
+  default     = null
+}
+
+variable "first_on_demand" {
+  type        = number
+  description = "The first first_on_demand nodes of the cluster will be placed on on-demand instances: [[:number]]"
+  default     = 0
+}
+
+variable "spot_bid_max_price" {
+  type        = number
+  description = "The max price for Azure spot instances. Use -1 to specify lowest price."
+  default     = -1
+}
+
+variable "autotermination_minutes" {
+  type        = number
+  description = "Automatically terminate the cluster after being inactive for this time in minutes. If not set, Databricks won't automatically terminate an inactive cluster. If specified, the threshold must be between 10 and 10000 minutes. You can also set this value to 0 to explicitly disable automatic termination."
+  default     = 15
+}
+
+variable "min_workers" {
+  type        = number
+  description = "The minimum number of workers to which the cluster can scale down when underutilized. It is also the initial number of workers the cluster will have after creation."
+  default     = 0
+}
+
+variable "max_workers" {
+  type        = number
+  description = "The maximum number of workers to which the cluster can scale up when overloaded. max_workers must be strictly greater than min_workers."
+  default     = 1
+}
+
+variable "users" {
+  type        = list(string)
+  description = "List of users to access Databricks"
+  default     = []
+}
+
+variable "secrets" {
+  type        = map(any)
+  description = "Map of secrets to create in Databricks"
+  default     = {}
+}
+
+variable "use_local_secret_scope" {
+  type        = bool
+  description = "Create databricks secret scope and create secrets"
+  default     = false
+}
+
+variable "permissions" {
+  type        = list(map(string))
+  description = "Databricks Workspace permission maps"
+  default = [
+    {
+      object_id = null
+      role      = null
+    }
+  ]
+}
+
+variable "spark_version" {
+  type        = string
+  description = "Runtime version"
+  default     = "9.1.x-scala2.12"
+}
+
+variable "node_type" {
+  type        = string
+  description = "databricks_node_type id"
+  default     = "Standard_D3_v2"
+}
+
+variable "mountpoints" {
+  type        = map(any)
+  description = "mountpoints for databricks"
+  default     = null
+}

versions.tf

@@ -0,0 +1,10 @@
+terraform {
+  required_version = ">=1.0.0"
+
+  required_providers {
+    databricks = {
+      source  = "databricks/databricks"
+      version = "=1.4.0"
+    }
+  }
+}