From 728ad51927ca0d53759481ec6f52564e368289f6 Mon Sep 17 00:00:00 2001 From: MyroslavLevchyk Date: Fri, 7 Feb 2025 13:47:45 +0200 Subject: [PATCH 1/2] fix: upgrade to terraform v1.3 and update readme --- README.md | 310 ++++++++++++++++++++++++++++++++++++++++++---------- versions.tf | 2 +- 2 files changed, 252 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 759453a..f1cc1ee 100644 --- a/README.md +++ b/README.md @@ -8,20 +8,22 @@ The main idea behind this module is to deploy resources for Databricks Workspace Here we provide some examples of how to provision it with a different options. +### Example for Azure Cloud: + ### In example below, these features of given module would be covered: -1. Workspace admins assignment, custom Workspace group creation, group assignments, group entitlements -2. Clusters (i.e., for Unity Catalog and Shared Autoscaling) -3. Workspace IP Access list creation -4. ADLS Gen2 Mount -5. Create Secret Scope and assign permissions to custom groups -6. SQL Endpoint creation and configuration -7. Create Cluster policy -8. Create an Azure Key Vault-backed secret scope -9. Connect to already existing Unity Catalog Metastore +1. Clusters (i.e., for Unity Catalog and Shared Autoscaling) +2. Workspace IP Access list creation +3. ADLS Gen2 Mount +4. Create Secret Scope and assign permissions to custom groups +5. SQL Endpoint creation and configuration +6. Create Cluster policy +7. Create an Azure Key Vault-backed secret scope ```hcl # Prerequisite resources +variable "databricks_account_id" {} + # Databricks Workspace with Premium SKU data "azurerm_databricks_workspace" "example" { name = "example-workspace" @@ -35,46 +37,73 @@ provider "databricks" { azure_workspace_resource_id = data.azurerm_databricks_workspace.example.id } +# Databricks Account-Level Provider configuration +provider "databricks" { + alias = "account" + host = "https://accounts.azuredatabricks.net" + account_id = var.databricks_account_id +} + # Key Vault where Service Principal's secrets are stored. Used for mounting Storage Container data "azurerm_key_vault" "example" { name = "example-key-vault" resource_group_name = "example-rg" } +locals { + databricks_iam_account_groups = [{ + group_name = "example-gn" + permissions = ["ADMIN"] + entitlements = [ + "allow_instance_pool_create", + "allow_cluster_create", + "databricks_sql_access" + ] + }] +} + +# Assigns Databricks Account groups to Workspace. It is required to assign Unity Catalog Metastore before assigning Account groups to Workspace +module "databricks_account_groups" { + count = length(local.databricks_iam_account_groups) != 0 ? 1 : 0 + source = "data-platform-hq/databricks-account-groups/databricks" + version = "1.0.1" + + workspace_id = data.azurerm_databricks_workspace.example.id + workspace_group_assignment = local.databricks_iam_account_groups + + providers = { + databricks = databricks.account + } +} + # Example usage of module for Runtime Premium resources. -module "databricks_runtime_premium" { - source = "data-platform-hq/databricks-runtime-premium/databricks" +module "databricks_runtime_premium" { + source = "data-platform-hq/runtime/databricks" + version = "~>1.0" project = "datahq" env = "example" location = "eastus" - # Parameters of Service principal used for ADLS mount - # Imports App ID and Secret of Service Principal from target Key Vault - key_vault_id = data.azurerm_key_vault.example.id - sp_client_id_secret_name = "sp-client-id" # secret's name that stores Service Principal App ID - sp_key_secret_name = "sp-key" # secret's name that stores Service Principal Secret Key - tenant_id_secret_name = "infra-arm-tenant-id" # secret's name that stores tenant id value - - # 1.1 Workspace admins - workspace_admins = { - user = ["user1@example.com"] - service_principal = ["example-app-id"] + # Cloud provider + cloud_name = "azure" + + # Example configuration for Workspace Groups + iam_workspace_groups = { + dev = { + user = [ + "user1@example.com", + "user2@example.com" + ] + service_principal = [] + entitlements = ["allow_instance_pool_create","allow_cluster_create","databricks_sql_access"] + } } - # 1.2 Custom Workspace group with assignments. - # In addition, provides an ability to create group and entitlements. - iam = [{ - group_name = "DEVELOPERS" - permissions = ["ADMIN"] - entitlements = [ - "allow_instance_pool_create", - "allow_cluster_create", - "databricks_sql_access" - ] - }] + # Example configuration for Account Groups + iam_account_groups = local.databricks_iam_account_groups - # 2. Databricks clusters configuration, and assign permission to a custom group on clusters. + # 1. Databricks clusters configuration, and assign permission to a custom group on clusters. databricks_cluster_configs = [ { cluster_name = "Unity Catalog" data_security_mode = "USER_ISOLATION" @@ -90,33 +119,39 @@ module "databricks_runtime_premium" { permissions = [{group_name = "DEVELOPERS", permission_level = "CAN_MANAGE"}] }] - # 3. Workspace could be accessed only from these IP Addresses: + # 2. Workspace could be accessed only from these IP Addresses: ip_rules = { "ip_range_1" = "10.128.0.0/16", "ip_range_2" = "10.33.0.0/16", } - # 4. ADLS Gen2 Mount + # 3. ADLS Gen2 Mount mountpoints = { storage_account_name = data.azurerm_storage_account.example.name container_name = "example_container" } - # 5. Create Secret Scope and assign permissions to custom groups + # Parameters of Service principal used for ADLS mount + # Imports App ID and Secret of Service Principal from target Key Vault + sp_client_id_secret_name = "sp-client-id" # secret's name that stores Service Principal App ID + sp_key_secret_name = "sp-key" # secret's name that stores Service Principal Secret Key + tenant_id_secret_name = "infra-arm-tenant-id" # secret's name that stores tenant id value + + # 4. Create Secret Scope and assign permissions to custom groups secret_scope = [{ scope_name = "extra-scope" acl = [{ principal = "DEVELOPERS", permission = "READ" }] # Only custom workspace group names are allowed. If left empty then only Workspace admins could access these keys secrets = [{ key = "secret-name", string_value = "secret-value"}] }] - # 6. SQL Warehouse Endpoint + # 5. SQL Warehouse Endpoint databricks_sql_endpoint = [{ name = "default" enable_serverless_compute = true permissions = [{ group_name = "DEVELOPERS", permission_level = "CAN_USE" },] }] - # 7. Databricks cluster policies + # 6. Databricks cluster policies custom_cluster_policies = [{ name = "custom_policy_1", can_use = "DEVELOPERS", # custom workspace group name, that is allowed to use this policy @@ -129,7 +164,7 @@ module "databricks_runtime_premium" { } }] - # 8. Azure Key Vault-backed secret scope + # 7. Azure Key Vault-backed secret scope key_vault_secret_scope = [{ name = "external" key_vault_id = data.azurerm_key_vault.example.id @@ -141,17 +176,174 @@ module "databricks_runtime_premium" { } } -# 9 Assignment already existing Unity Catalog Metastore -module "metastore_assignment" { - source = "data-platform-hq/metastore-assignment/databricks" +``` + +### Example for AWS Cloud: + +### In example below, these features of given module would be covered: +1. Clusters (i.e., for Unity Catalog and Shared Autoscaling) +2. Workspace IP Access list creation +3. Create Secret Scope and assign permissions to custom groups +4. SQL Endpoint creation and configuration +5. Create Cluster policy + +```hcl + +# Prerequisite resources + +variable "databricks_account_id" {} +variable "region" {} + +# Databricks Workspace ID +data "databricks_mws_workspaces" "example" { + account_id = var.databricks_account_id +} + +# Provider configuration for SSM +provider "aws" { + alias = "ssm" + region = var.region +} + +# Databricks Account-Level Provider configuration +provider "databricks" { + alias = "mws" + host = "https://accounts.cloud.databricks.com" + account_id = data.aws_ssm_parameter.this["databricks_account_id"].value + client_id = data.aws_ssm_parameter.this["databricks_admin_sp_id"].value + client_secret = data.aws_ssm_parameter.this["databricks_admin_sp_secret"].value +} + +# Databricks Provider configuration +provider "databricks" { + alias = "workspace" + host = module.databricks_workspace.workspace_url + client_id = data.aws_ssm_parameter.this["databricks_admin_sp_id"].value + client_secret = data.aws_ssm_parameter.this["databricks_admin_sp_secret"].value +} + +locals { + ssm_parameters = [ + "databricks_account_id", + "databricks_admin_sp_id", + "databricks_admin_sp_secret", + "github_pat_token" + ] + + ssm_parameters_prefix = "/example-prefix/" # Prefix for parameters stored in AWS SSM + + dbx_runtime = { + iam_account_groups_assignment = [ + { group_name = "example gm1", permissions = ["USER"] }, + { group_name = "example gm2", permissions = ["USER"] } + ] + + sql_endpoints = [{ + name = "example_test" + permissions = [ + { group_name = "example gm1", permission_level = "CAN_MANAGE" }, + ] + }] + + clusters = [{ + cluster_name = "example1" + permissions = [ + { group_name = "example gm2", permission_level = "CAN_RESTART" }, + ] + }, { + cluster_name = "example2" + permissions = [ + { group_name = "example gm2", permission_level = "CAN_RESTART" }, + { group_name = "example gm1", permission_level = "CAN_MANAGE" }, + ] + }] + } + + databricks_custom_cluster_policies = [{ + name = null + can_use = null + definition = null + }] + + dbx_inputs = { + vpc_id = "vpc-example" + subnet_ids = ["subnet-example1", "subnet-example2"] + security_group_ids = ["sg-example"] + } + + iam_default_permission_boundary_policy_arn = "arn:aws:iam::{ AWS Account ID }:policy/eo_role_boundary" +} + +# SSM Parameter +data "aws_ssm_parameter" "this" { + for_each = local.ssm_parameters + name = "${local.ssm_parameters_prefix}${each.key}" + provider = aws.ssm +} + +# Label configuration +module "label" { + source = "cloudposse/label/null" + version = "0.25.0" + + namespace = "example-namespace" + environment = "example-environment" + stage = "example-stage" +} + +# Databricks Workspace configuration +module "databricks_workspace" { + source = "data-platform-hq/aws-workspace/databricks" + version = "1.0.1" + + label = module.label.id + vpc_id = local.dbx_inputs.vpc_id + subnet_ids = local.dbx_inputs.subnet_ids + security_group_ids = local.dbx_inputs.security_group_ids + region = var.region + account_id = data.aws_ssm_parameter.this["databricks_account_id"].value + iam_cross_account_workspace_role_config = { + permission_boundary_arn = local.iam_default_permission_boundary_policy_arn + } + + providers = { + databricks = databricks.mws + } +} + +# Account level group assignment to the Workspace +module "databricks_account_groups" { + source = "data-platform-hq/databricks-account-groups/databricks" + version = "1.0.1" + + workspace_id = module.databricks_workspace.workspace_id + workspace_group_assignment = local.dbx_runtime.iam_account_groups_assignment + + providers = { + databricks = databricks.mws + } +} + +# Databricks Runtime resources configuration (clusters, sql, secrets, etc.) +module "databricks_runtime" { + source = "data-platform-hq/runtime/databricks" version = "1.0.0" - workspace_id = data.azurerm_databricks_workspace.example.workspace_id - metastore_id = "" + clusters = local.dbx_runtime.clusters + sql_endpoint = local.dbx_runtime.sql_endpoints + secret_scope = flatten([var.dbx_runtime.secret_scopes, local.demo_wwi_secret_scope]) + workspace_admin_token_enabled = var.workspace_admin_token_enabled + system_schemas_enabled = alltrue([var.databricks_system_schemas_enabled]) + + iam_account_groups = local.dbx_runtime.iam_account_groups_assignment + cloud_name = "aws" + custom_cluster_policies = local.databricks_custom_cluster_policies providers = { databricks = databricks.workspace } + + depends_on = [module.databricks_workspace, module.databricks_account_groups] } ``` @@ -161,7 +353,7 @@ module "metastore_assignment" { | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | +| [terraform](#requirement\_terraform) | >=1.3 | | [databricks](#requirement\_databricks) | ~>1.0 | ## Providers @@ -206,22 +398,22 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [cloud\_name](#input\_cloud\_name) | Cloud Name | `string` | n/a | yes | -| [clusters](#input\_clusters) | Set of objects with parameters to configure Databricks clusters and assign permissions to it for certain custom groups |
set(object({
cluster_name = string
spark_version = optional(string, "15.3.x-scala2.12")
spark_conf = optional(map(any), {})
spark_env_vars = optional(map(any), {})
data_security_mode = optional(string, "USER_ISOLATION")
aws_attributes = optional(object({
availability = optional(string)
zone_id = optional(string)
first_on_demand = optional(number)
spot_bid_price_percent = optional(number)
ebs_volume_count = optional(number)
ebs_volume_size = optional(number)
ebs_volume_type = optional(string)
}), {
availability = "ON_DEMAND"
zone_id = "auto"
first_on_demand = 0
spot_bid_price_percent = 100
ebs_volume_count = 1
ebs_volume_size = 100
ebs_volume_type = "GENERAL_PURPOSE_SSD"
})
azure_attributes = optional(object({
availability = optional(string)
first_on_demand = optional(number)
spot_bid_max_price = optional(number, 1)
}), {
availability = "ON_DEMAND_AZURE"
first_on_demand = 0
})
node_type_id = optional(string, null)
autotermination_minutes = optional(number, 20)
min_workers = optional(number, 1)
max_workers = optional(number, 2)
cluster_log_conf_destination = optional(string, null)
init_scripts_workspace = optional(set(string), [])
init_scripts_volumes = optional(set(string), [])
init_scripts_dbfs = optional(set(string), [])
init_scripts_abfss = optional(set(string), [])
single_user_name = optional(string, null)
single_node_enable = optional(bool, false)
custom_tags = optional(map(string), {})
permissions = optional(set(object({
group_name = string
permission_level = string
})), [])
pypi_library_repository = optional(set(string), [])
maven_library_repository = optional(set(object({
coordinates = string
exclusions = set(string)
})), [])
}))
| `[]` | no | -| [custom\_cluster\_policies](#input\_custom\_cluster\_policies) | Provides an ability to create custom cluster policy, assign it to cluster and grant CAN\_USE permissions on it to certain custom groups
name - name of custom cluster policy to create
can\_use - list of string, where values are custom group names, there groups have to be created with Terraform;
definition - JSON document expressed in Databricks Policy Definition Language. No need to call 'jsonencode()' function on it when providing a value; |
list(object({
name = string
can_use = list(string)
definition = any
}))
|
[
{
"can_use": null,
"definition": null,
"name": null
}
]
| no | -| [custom\_config](#input\_custom\_config) | Map of AD databricks workspace custom config | `map(string)` |
{
"enable-X-Content-Type-Options": "true",
"enable-X-Frame-Options": "true",
"enable-X-XSS-Protection": "true",
"enableDbfsFileBrowser": "false",
"enableExportNotebook": "false",
"enableIpAccessLists": "true",
"enableNotebookTableClipboard": "false",
"enableResultsDownloading": "false",
"enableUploadDataUis": "false",
"enableVerboseAuditLogs": "true",
"enforceUserIsolation": "true",
"storeInteractiveNotebookResultsInCustomerAccount": "true"
}
| no | -| [default\_cluster\_policies\_override](#input\_default\_cluster\_policies\_override) | Provides an ability to override default cluster policy
name - name of cluster policy to override
family\_id - family id of corresponding policy
definition - JSON document expressed in Databricks Policy Definition Language. No need to call 'jsonencode()' function on it when providing a value; |
list(object({
name = string
family_id = string
definition = any
}))
|
[
{
"definition": null,
"family_id": null,
"name": null
}
]
| no | -| [iam\_account\_groups](#input\_iam\_account\_groups) | List of objects with group name and entitlements for this group |
list(object({
group_name = optional(string)
entitlements = optional(list(string))
}))
| `[]` | no | -| [iam\_workspace\_groups](#input\_iam\_workspace\_groups) | Used to create workspace group. Map of group name and its parameters, such as users and service principals added to the group. Also possible to configure group entitlements. |
map(object({
user = optional(list(string))
service_principal = optional(list(string))
entitlements = optional(list(string))
}))
| `{}` | no | -| [ip\_addresses](#input\_ip\_addresses) | A map of IP address ranges | `map(string)` |
{
"all": "0.0.0.0/0"
}
| no | -| [key\_vault\_secret\_scope](#input\_key\_vault\_secret\_scope) | Object with Azure Key Vault parameters required for creation of Azure-backed Databricks Secret scope |
list(object({
name = string
key_vault_id = string
dns_name = string
tenant_id = string
}))
| `[]` | no | -| [mount\_configuration](#input\_mount\_configuration) | Configuration for mounting storage, including only service principal details |
object({
service_principal = object({
client_id = string
client_secret = string
tenant_id = string
})
})
|
{
"service_principal": {
"client_id": null,
"client_secret": null,
"tenant_id": null
}
}
| no | +| [clusters](#input\_clusters) | Set of objects with parameters to configure Databricks clusters and assign permissions to it for certain custom groups |
set(object({
cluster_name = string
spark_version = optional(string, "15.3.x-scala2.12")
spark_conf = optional(map(any), {})
spark_env_vars = optional(map(any), {})
data_security_mode = optional(string, "USER_ISOLATION")
aws_attributes = optional(object({
availability = optional(string)
zone_id = optional(string)
first_on_demand = optional(number)
spot_bid_price_percent = optional(number)
ebs_volume_count = optional(number)
ebs_volume_size = optional(number)
ebs_volume_type = optional(string)
}), {
availability = "ON_DEMAND"
zone_id = "auto"
first_on_demand = 0
spot_bid_price_percent = 100
ebs_volume_count = 1
ebs_volume_size = 100
ebs_volume_type = "GENERAL_PURPOSE_SSD"
})
azure_attributes = optional(object({
availability = optional(string)
first_on_demand = optional(number)
spot_bid_max_price = optional(number, 1)
}), {
availability = "ON_DEMAND_AZURE"
first_on_demand = 0
})
node_type_id = optional(string, null)
autotermination_minutes = optional(number, 20)
min_workers = optional(number, 1)
max_workers = optional(number, 2)
cluster_log_conf_destination = optional(string, null)
init_scripts_workspace = optional(set(string), [])
init_scripts_volumes = optional(set(string), [])
init_scripts_dbfs = optional(set(string), [])
init_scripts_abfss = optional(set(string), [])
single_user_name = optional(string, null)
single_node_enable = optional(bool, false)
custom_tags = optional(map(string), {})
permissions = optional(set(object({
group_name = string
permission_level = string
})), [])
pypi_library_repository = optional(set(string), [])
maven_library_repository = optional(set(object({
coordinates = string
exclusions = set(string)
})), [])
}))
| `[]` | no | +| [custom\_cluster\_policies](#input\_custom\_cluster\_policies) | Provides an ability to create custom cluster policy, assign it to cluster and grant CAN\_USE permissions on it to certain custom groups
name - name of custom cluster policy to create
can\_use - list of string, where values are custom group names, there groups have to be created with Terraform;
definition - JSON document expressed in Databricks Policy Definition Language. No need to call 'jsonencode()' function on it when providing a value; |
list(object({
name = string
can_use = list(string)
definition = any
}))
|
[
{
"can_use": null,
"definition": null,
"name": null
}
]
| no | +| [custom\_config](#input\_custom\_config) | Map of AD databricks workspace custom config | `map(string)` |
{
"enable-X-Content-Type-Options": "true",
"enable-X-Frame-Options": "true",
"enable-X-XSS-Protection": "true",
"enableDbfsFileBrowser": "false",
"enableExportNotebook": "false",
"enableIpAccessLists": "true",
"enableNotebookTableClipboard": "false",
"enableResultsDownloading": "false",
"enableUploadDataUis": "false",
"enableVerboseAuditLogs": "true",
"enforceUserIsolation": "true",
"storeInteractiveNotebookResultsInCustomerAccount": "true"
}
| no | +| [default\_cluster\_policies\_override](#input\_default\_cluster\_policies\_override) | Provides an ability to override default cluster policy
name - name of cluster policy to override
family\_id - family id of corresponding policy
definition - JSON document expressed in Databricks Policy Definition Language. No need to call 'jsonencode()' function on it when providing a value; |
list(object({
name = string
family_id = string
definition = any
}))
|
[
{
"definition": null,
"family_id": null,
"name": null
}
]
| no | +| [iam\_account\_groups](#input\_iam\_account\_groups) | List of objects with group name and entitlements for this group |
list(object({
group_name = optional(string)
entitlements = optional(list(string))
}))
| `[]` | no | +| [iam\_workspace\_groups](#input\_iam\_workspace\_groups) | Used to create workspace group. Map of group name and its parameters, such as users and service principals added to the group. Also possible to configure group entitlements. |
map(object({
user = optional(list(string))
service_principal = optional(list(string))
entitlements = optional(list(string))
}))
| `{}` | no | +| [ip\_addresses](#input\_ip\_addresses) | A map of IP address ranges | `map(string)` |
{
"all": "0.0.0.0/0"
}
| no | +| [key\_vault\_secret\_scope](#input\_key\_vault\_secret\_scope) | Object with Azure Key Vault parameters required for creation of Azure-backed Databricks Secret scope |
list(object({
name = string
key_vault_id = string
dns_name = string
tenant_id = string
}))
| `[]` | no | +| [mount\_configuration](#input\_mount\_configuration) | Configuration for mounting storage, including only service principal details |
object({
service_principal = object({
client_id = string
client_secret = string
tenant_id = string
})
})
|
{
"service_principal": {
"client_id": null,
"client_secret": null,
"tenant_id": null
}
}
| no | | [mount\_enabled](#input\_mount\_enabled) | Boolean flag that determines whether mount point for storage account filesystem is created | `bool` | `false` | no | -| [mountpoints](#input\_mountpoints) | Mountpoints for databricks |
map(object({
storage_account_name = string
container_name = string
}))
| `{}` | no | +| [mountpoints](#input\_mountpoints) | Mountpoints for databricks |
map(object({
storage_account_name = string
container_name = string
}))
| `{}` | no | | [pat\_token\_lifetime\_seconds](#input\_pat\_token\_lifetime\_seconds) | The lifetime of the token, in seconds. If no lifetime is specified, the token remains valid indefinitely | `number` | `315569520` | no | -| [secret\_scope](#input\_secret\_scope) | Provides an ability to create custom Secret Scope, store secrets in it and assigning ACL for access management
scope\_name - name of Secret Scope to create;
acl - list of objects, where 'principal' custom group name, this group is created in 'Premium' module; 'permission' is one of "READ", "WRITE", "MANAGE";
secrets - list of objects, where object's 'key' param is created key name and 'string\_value' is a value for it; |
list(object({
scope_name = string
scope_acl = optional(list(object({
principal = string
permission = string
})))
secrets = optional(list(object({
key = string
string_value = string
})))
}))
| `[]` | no | -| [sql\_endpoint](#input\_sql\_endpoint) | Set of objects with parameters to configure SQL Endpoint and assign permissions to it for certain custom groups |
set(object({
name = string
cluster_size = optional(string, "2X-Small")
min_num_clusters = optional(number, 0)
max_num_clusters = optional(number, 1)
auto_stop_mins = optional(string, "30")
enable_photon = optional(bool, false)
enable_serverless_compute = optional(bool, false)
spot_instance_policy = optional(string, "COST_OPTIMIZED")
warehouse_type = optional(string, "PRO")
permissions = optional(set(object({
group_name = string
permission_level = string
})), [])
}))
| `[]` | no | +| [secret\_scope](#input\_secret\_scope) | Provides an ability to create custom Secret Scope, store secrets in it and assigning ACL for access management
scope\_name - name of Secret Scope to create;
acl - list of objects, where 'principal' custom group name, this group is created in 'Premium' module; 'permission' is one of "READ", "WRITE", "MANAGE";
secrets - list of objects, where object's 'key' param is created key name and 'string\_value' is a value for it; |
list(object({
scope_name = string
scope_acl = optional(list(object({
principal = string
permission = string
})))
secrets = optional(list(object({
key = string
string_value = string
})))
}))
| `[]` | no | +| [sql\_endpoint](#input\_sql\_endpoint) | Set of objects with parameters to configure SQL Endpoint and assign permissions to it for certain custom groups |
set(object({
name = string
cluster_size = optional(string, "2X-Small")
min_num_clusters = optional(number, 0)
max_num_clusters = optional(number, 1)
auto_stop_mins = optional(string, "30")
enable_photon = optional(bool, false)
enable_serverless_compute = optional(bool, false)
spot_instance_policy = optional(string, "COST_OPTIMIZED")
warehouse_type = optional(string, "PRO")
permissions = optional(set(object({
group_name = string
permission_level = string
})), [])
}))
| `[]` | no | | [suffix](#input\_suffix) | Optional suffix that would be added to the end of resources names. | `string` | `""` | no | -| [system\_schemas](#input\_system\_schemas) | Set of strings with all possible System Schema names | `set(string)` |
[
"access",
"billing",
"compute",
"marketplace",
"storage"
]
| no | +| [system\_schemas](#input\_system\_schemas) | Set of strings with all possible System Schema names | `set(string)` |
[
"access",
"billing",
"compute",
"marketplace",
"storage"
]
| no | | [system\_schemas\_enabled](#input\_system\_schemas\_enabled) | System Schemas only works with assigned Unity Catalog Metastore. Boolean flag to enabled this feature | `bool` | `false` | no | | [workspace\_admin\_token\_enabled](#input\_workspace\_admin\_token\_enabled) | Boolean flag to specify whether to create Workspace Admin Token | `bool` | n/a | yes | diff --git a/versions.tf b/versions.tf index fcd2dd2..90304ac 100644 --- a/versions.tf +++ b/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.0" + required_version = ">=1.3" required_providers { databricks = { From d1744d117dbf31af83dd35181aad73089bb15bda Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 7 Feb 2025 11:48:13 +0000 Subject: [PATCH 2/2] terraform-docs: automated action --- README.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index f1cc1ee..31f109e 100644 --- a/README.md +++ b/README.md @@ -398,22 +398,22 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [cloud\_name](#input\_cloud\_name) | Cloud Name | `string` | n/a | yes | -| [clusters](#input\_clusters) | Set of objects with parameters to configure Databricks clusters and assign permissions to it for certain custom groups |
set(object({
cluster_name = string
spark_version = optional(string, "15.3.x-scala2.12")
spark_conf = optional(map(any), {})
spark_env_vars = optional(map(any), {})
data_security_mode = optional(string, "USER_ISOLATION")
aws_attributes = optional(object({
availability = optional(string)
zone_id = optional(string)
first_on_demand = optional(number)
spot_bid_price_percent = optional(number)
ebs_volume_count = optional(number)
ebs_volume_size = optional(number)
ebs_volume_type = optional(string)
}), {
availability = "ON_DEMAND"
zone_id = "auto"
first_on_demand = 0
spot_bid_price_percent = 100
ebs_volume_count = 1
ebs_volume_size = 100
ebs_volume_type = "GENERAL_PURPOSE_SSD"
})
azure_attributes = optional(object({
availability = optional(string)
first_on_demand = optional(number)
spot_bid_max_price = optional(number, 1)
}), {
availability = "ON_DEMAND_AZURE"
first_on_demand = 0
})
node_type_id = optional(string, null)
autotermination_minutes = optional(number, 20)
min_workers = optional(number, 1)
max_workers = optional(number, 2)
cluster_log_conf_destination = optional(string, null)
init_scripts_workspace = optional(set(string), [])
init_scripts_volumes = optional(set(string), [])
init_scripts_dbfs = optional(set(string), [])
init_scripts_abfss = optional(set(string), [])
single_user_name = optional(string, null)
single_node_enable = optional(bool, false)
custom_tags = optional(map(string), {})
permissions = optional(set(object({
group_name = string
permission_level = string
})), [])
pypi_library_repository = optional(set(string), [])
maven_library_repository = optional(set(object({
coordinates = string
exclusions = set(string)
})), [])
}))
| `[]` | no | -| [custom\_cluster\_policies](#input\_custom\_cluster\_policies) | Provides an ability to create custom cluster policy, assign it to cluster and grant CAN\_USE permissions on it to certain custom groups
name - name of custom cluster policy to create
can\_use - list of string, where values are custom group names, there groups have to be created with Terraform;
definition - JSON document expressed in Databricks Policy Definition Language. No need to call 'jsonencode()' function on it when providing a value; |
list(object({
name = string
can_use = list(string)
definition = any
}))
|
[
{
"can_use": null,
"definition": null,
"name": null
}
]
| no | -| [custom\_config](#input\_custom\_config) | Map of AD databricks workspace custom config | `map(string)` |
{
"enable-X-Content-Type-Options": "true",
"enable-X-Frame-Options": "true",
"enable-X-XSS-Protection": "true",
"enableDbfsFileBrowser": "false",
"enableExportNotebook": "false",
"enableIpAccessLists": "true",
"enableNotebookTableClipboard": "false",
"enableResultsDownloading": "false",
"enableUploadDataUis": "false",
"enableVerboseAuditLogs": "true",
"enforceUserIsolation": "true",
"storeInteractiveNotebookResultsInCustomerAccount": "true"
}
| no | -| [default\_cluster\_policies\_override](#input\_default\_cluster\_policies\_override) | Provides an ability to override default cluster policy
name - name of cluster policy to override
family\_id - family id of corresponding policy
definition - JSON document expressed in Databricks Policy Definition Language. No need to call 'jsonencode()' function on it when providing a value; |
list(object({
name = string
family_id = string
definition = any
}))
|
[
{
"definition": null,
"family_id": null,
"name": null
}
]
| no | -| [iam\_account\_groups](#input\_iam\_account\_groups) | List of objects with group name and entitlements for this group |
list(object({
group_name = optional(string)
entitlements = optional(list(string))
}))
| `[]` | no | -| [iam\_workspace\_groups](#input\_iam\_workspace\_groups) | Used to create workspace group. Map of group name and its parameters, such as users and service principals added to the group. Also possible to configure group entitlements. |
map(object({
user = optional(list(string))
service_principal = optional(list(string))
entitlements = optional(list(string))
}))
| `{}` | no | -| [ip\_addresses](#input\_ip\_addresses) | A map of IP address ranges | `map(string)` |
{
"all": "0.0.0.0/0"
}
| no | -| [key\_vault\_secret\_scope](#input\_key\_vault\_secret\_scope) | Object with Azure Key Vault parameters required for creation of Azure-backed Databricks Secret scope |
list(object({
name = string
key_vault_id = string
dns_name = string
tenant_id = string
}))
| `[]` | no | -| [mount\_configuration](#input\_mount\_configuration) | Configuration for mounting storage, including only service principal details |
object({
service_principal = object({
client_id = string
client_secret = string
tenant_id = string
})
})
|
{
"service_principal": {
"client_id": null,
"client_secret": null,
"tenant_id": null
}
}
| no | +| [clusters](#input\_clusters) | Set of objects with parameters to configure Databricks clusters and assign permissions to it for certain custom groups |
set(object({
cluster_name = string
spark_version = optional(string, "15.3.x-scala2.12")
spark_conf = optional(map(any), {})
spark_env_vars = optional(map(any), {})
data_security_mode = optional(string, "USER_ISOLATION")
aws_attributes = optional(object({
availability = optional(string)
zone_id = optional(string)
first_on_demand = optional(number)
spot_bid_price_percent = optional(number)
ebs_volume_count = optional(number)
ebs_volume_size = optional(number)
ebs_volume_type = optional(string)
}), {
availability = "ON_DEMAND"
zone_id = "auto"
first_on_demand = 0
spot_bid_price_percent = 100
ebs_volume_count = 1
ebs_volume_size = 100
ebs_volume_type = "GENERAL_PURPOSE_SSD"
})
azure_attributes = optional(object({
availability = optional(string)
first_on_demand = optional(number)
spot_bid_max_price = optional(number, 1)
}), {
availability = "ON_DEMAND_AZURE"
first_on_demand = 0
})
node_type_id = optional(string, null)
autotermination_minutes = optional(number, 20)
min_workers = optional(number, 1)
max_workers = optional(number, 2)
cluster_log_conf_destination = optional(string, null)
init_scripts_workspace = optional(set(string), [])
init_scripts_volumes = optional(set(string), [])
init_scripts_dbfs = optional(set(string), [])
init_scripts_abfss = optional(set(string), [])
single_user_name = optional(string, null)
single_node_enable = optional(bool, false)
custom_tags = optional(map(string), {})
permissions = optional(set(object({
group_name = string
permission_level = string
})), [])
pypi_library_repository = optional(set(string), [])
maven_library_repository = optional(set(object({
coordinates = string
exclusions = set(string)
})), [])
}))
| `[]` | no | +| [custom\_cluster\_policies](#input\_custom\_cluster\_policies) | Provides an ability to create custom cluster policy, assign it to cluster and grant CAN\_USE permissions on it to certain custom groups
name - name of custom cluster policy to create
can\_use - list of string, where values are custom group names, there groups have to be created with Terraform;
definition - JSON document expressed in Databricks Policy Definition Language. No need to call 'jsonencode()' function on it when providing a value; |
list(object({
name = string
can_use = list(string)
definition = any
}))
|
[
{
"can_use": null,
"definition": null,
"name": null
}
]
| no | +| [custom\_config](#input\_custom\_config) | Map of AD databricks workspace custom config | `map(string)` |
{
"enable-X-Content-Type-Options": "true",
"enable-X-Frame-Options": "true",
"enable-X-XSS-Protection": "true",
"enableDbfsFileBrowser": "false",
"enableExportNotebook": "false",
"enableIpAccessLists": "true",
"enableNotebookTableClipboard": "false",
"enableResultsDownloading": "false",
"enableUploadDataUis": "false",
"enableVerboseAuditLogs": "true",
"enforceUserIsolation": "true",
"storeInteractiveNotebookResultsInCustomerAccount": "true"
}
| no | +| [default\_cluster\_policies\_override](#input\_default\_cluster\_policies\_override) | Provides an ability to override default cluster policy
name - name of cluster policy to override
family\_id - family id of corresponding policy
definition - JSON document expressed in Databricks Policy Definition Language. No need to call 'jsonencode()' function on it when providing a value; |
list(object({
name = string
family_id = string
definition = any
}))
|
[
{
"definition": null,
"family_id": null,
"name": null
}
]
| no | +| [iam\_account\_groups](#input\_iam\_account\_groups) | List of objects with group name and entitlements for this group |
list(object({
group_name = optional(string)
entitlements = optional(list(string))
}))
| `[]` | no | +| [iam\_workspace\_groups](#input\_iam\_workspace\_groups) | Used to create workspace group. Map of group name and its parameters, such as users and service principals added to the group. Also possible to configure group entitlements. |
map(object({
user = optional(list(string))
service_principal = optional(list(string))
entitlements = optional(list(string))
}))
| `{}` | no | +| [ip\_addresses](#input\_ip\_addresses) | A map of IP address ranges | `map(string)` |
{
"all": "0.0.0.0/0"
}
| no | +| [key\_vault\_secret\_scope](#input\_key\_vault\_secret\_scope) | Object with Azure Key Vault parameters required for creation of Azure-backed Databricks Secret scope |
list(object({
name = string
key_vault_id = string
dns_name = string
tenant_id = string
}))
| `[]` | no | +| [mount\_configuration](#input\_mount\_configuration) | Configuration for mounting storage, including only service principal details |
object({
service_principal = object({
client_id = string
client_secret = string
tenant_id = string
})
})
|
{
"service_principal": {
"client_id": null,
"client_secret": null,
"tenant_id": null
}
}
| no | | [mount\_enabled](#input\_mount\_enabled) | Boolean flag that determines whether mount point for storage account filesystem is created | `bool` | `false` | no | -| [mountpoints](#input\_mountpoints) | Mountpoints for databricks |
map(object({
storage_account_name = string
container_name = string
}))
| `{}` | no | +| [mountpoints](#input\_mountpoints) | Mountpoints for databricks |
map(object({
storage_account_name = string
container_name = string
}))
| `{}` | no | | [pat\_token\_lifetime\_seconds](#input\_pat\_token\_lifetime\_seconds) | The lifetime of the token, in seconds. If no lifetime is specified, the token remains valid indefinitely | `number` | `315569520` | no | -| [secret\_scope](#input\_secret\_scope) | Provides an ability to create custom Secret Scope, store secrets in it and assigning ACL for access management
scope\_name - name of Secret Scope to create;
acl - list of objects, where 'principal' custom group name, this group is created in 'Premium' module; 'permission' is one of "READ", "WRITE", "MANAGE";
secrets - list of objects, where object's 'key' param is created key name and 'string\_value' is a value for it; |
list(object({
scope_name = string
scope_acl = optional(list(object({
principal = string
permission = string
})))
secrets = optional(list(object({
key = string
string_value = string
})))
}))
| `[]` | no | -| [sql\_endpoint](#input\_sql\_endpoint) | Set of objects with parameters to configure SQL Endpoint and assign permissions to it for certain custom groups |
set(object({
name = string
cluster_size = optional(string, "2X-Small")
min_num_clusters = optional(number, 0)
max_num_clusters = optional(number, 1)
auto_stop_mins = optional(string, "30")
enable_photon = optional(bool, false)
enable_serverless_compute = optional(bool, false)
spot_instance_policy = optional(string, "COST_OPTIMIZED")
warehouse_type = optional(string, "PRO")
permissions = optional(set(object({
group_name = string
permission_level = string
})), [])
}))
| `[]` | no | +| [secret\_scope](#input\_secret\_scope) | Provides an ability to create custom Secret Scope, store secrets in it and assigning ACL for access management
scope\_name - name of Secret Scope to create;
acl - list of objects, where 'principal' custom group name, this group is created in 'Premium' module; 'permission' is one of "READ", "WRITE", "MANAGE";
secrets - list of objects, where object's 'key' param is created key name and 'string\_value' is a value for it; |
list(object({
scope_name = string
scope_acl = optional(list(object({
principal = string
permission = string
})))
secrets = optional(list(object({
key = string
string_value = string
})))
}))
| `[]` | no | +| [sql\_endpoint](#input\_sql\_endpoint) | Set of objects with parameters to configure SQL Endpoint and assign permissions to it for certain custom groups |
set(object({
name = string
cluster_size = optional(string, "2X-Small")
min_num_clusters = optional(number, 0)
max_num_clusters = optional(number, 1)
auto_stop_mins = optional(string, "30")
enable_photon = optional(bool, false)
enable_serverless_compute = optional(bool, false)
spot_instance_policy = optional(string, "COST_OPTIMIZED")
warehouse_type = optional(string, "PRO")
permissions = optional(set(object({
group_name = string
permission_level = string
})), [])
}))
| `[]` | no | | [suffix](#input\_suffix) | Optional suffix that would be added to the end of resources names. | `string` | `""` | no | -| [system\_schemas](#input\_system\_schemas) | Set of strings with all possible System Schema names | `set(string)` |
[
"access",
"billing",
"compute",
"marketplace",
"storage"
]
| no | +| [system\_schemas](#input\_system\_schemas) | Set of strings with all possible System Schema names | `set(string)` |
[
"access",
"billing",
"compute",
"marketplace",
"storage"
]
| no | | [system\_schemas\_enabled](#input\_system\_schemas\_enabled) | System Schemas only works with assigned Unity Catalog Metastore. Boolean flag to enabled this feature | `bool` | `false` | no | | [workspace\_admin\_token\_enabled](#input\_workspace\_admin\_token\_enabled) | Boolean flag to specify whether to create Workspace Admin Token | `bool` | n/a | yes |