Skip to content

Commit 15c374d

Browse files
owlleg6owlleg6
authored
Merge pull request #11 from data-platform-hq/feat/sql-endpoint-refactor-and-serverless
feat: sql endpoint refactor and serverless feature
2 parents 5c1529e + 171f881 commit 15c374d

File tree

4 files changed

+48
-58
lines changed

4 files changed

+48
-58
lines changed

‎README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ No modules.
4040
| [databricks_secret_acl.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/secret_acl) | resource |
4141
| [databricks_workspace_conf.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/workspace_conf) | resource |
4242
| [databricks_ip_access_list.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/ip_access_list) | resource |
43+
| [databricks_sql_global_config.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/sql_global_config) | resource |
4344
| [databricks_sql_endpoint.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/sql_endpoint) | resource |
4445
| [azurerm_storage_data_lake_gen2_filesystem.this](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_data_lake_gen2_filesystem) | resource |
4546
| [databricks_metastore.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/metastore) | resource |
@@ -68,8 +69,7 @@ No modules.
6869
| <a name="input_iam_permissions"></a> [iam\_permissions](#input\_iam\_permissions) | Map of permission for groups. You can provide certain permission on services to groups. E.g., `'sql_endpoint'={'CAN_USE'=['group1', 'group2'] CAN_MANAGE=['group3']} | <pre> map(object({ <br> CAN_USE = list(string) <br> CAN_MANAGE = list(string)<br> })) </pre> | <pre> { <br> "sql_endpoint" = { <br> "CAN_USE" = ["default"] <br> "CAN_MANAGE" = [] <br> } <br> } </pre> | no |
6970
| <a name="input_default_cluster_id"></a> [default\_cluster\_id](#input\_default\_cluster\_id) | Single value of default Cluster id created by 'databricks-runtime' module | `map(string)` | {} | no |
7071
| <a name="input_cluster_policies_object"></a> [cluster\_policies\_object](#input\_cluster\_policies\_object) | List of objects that provides an ability to grant custom workspace group a permission to use(CAN_USE) cluster policy | <pre>list(object({<br> id = string<br> name = string<br> can_use = list(string)<br>}))</pre> | <pre>[{<br> id = null<br> name = null<br> can_use = null<br>}))</pre> | no |
71-
| <a name="input_sql_endpoint"></a> [sql\_endpoint](#input\_sql\_endpoint) | Map of SQL Endpoints to be deployed in Databricks Workspace | <pre> map(object({ <br> cluster_size = string <br> min_num_clusters = optional(number) <br> max_num_clusters = optional(number) <br> auto_stop_mins = optional(string) <br> enable_photon = optional(bool) <br> enable_serverless_compute = optional(bool) <br> })) </pre> | {} | no |
72-
| <a name="input_default_values_sql_endpoint"></a> [default\_values\_sql\_endpoint](#input\_default\_values\_sql\_endpoint) | Default values for SQL Endpoint | <pre> object({ <br> cluster_size = string <br> min_num_clusters = number <br> max_num_clusters = number <br> auto_stop_mins = string <br> enable_photon = bool <br> enable_serverless_compute = bool <br> }) </pre> | <pre> { <br> cluster_size = "2X-Small" <br> min_num_clusters = 0 <br> max_num_clusters = 1 <br> auto_stop_mins = "30" <br> enable_photon = false <br> enable_serverless_compute = false <br> } </pre> | no |
72+
| <a name="input_sql_endpoint"></a> [sql\_endpoint](#input\_sql\_endpoint) | Set of objects with parameters to configure SQL Endpoint and assign permissions to it for certain custom groups| <pre> map(object({ <br> cluster_size = string <br> min_num_clusters = optional(number) <br> max_num_clusters = optional(number) <br> auto_stop_mins = optional(string) <br> enable_photon = optional(bool) <br> enable_serverless_compute = optional(bool) <br> })) </pre> | {} | no |
7373
| <a name="input_create_metastore"></a> [create\_metastore](#input\_create\_metastore) | Boolean flag for Unity Catalog Metastore current in this environment. One Metastore per region | `bool` | false | no |
7474
| <a name="input_access_connector_id"></a> [access\_connector\_id](#input\_access\_connector\_id) | Databricks Access Connector Id that lets you to connect managed identities to an Azure Databricks account. Provides an ability to access Unity Catalog with assigned identity | `string` | " " | no |
7575
| <a name="input_storage_account_id"></a> [storage\_account\_id](#input\_storage\_account\_id) | Storage Account Id where Unity Catalog Metastore would be provisioned | `string` | " " | no |

‎main.tf

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,32 @@ resource "databricks_ip_access_list" "this" {
2121
depends_on = [databricks_workspace_conf.this]
2222
}
2323

24+
# SQL Endpoint
25+
resource "databricks_sql_global_config" "this" {
26+
count = anytrue(var.sql_endpoint[*].enable_serverless_compute) ? 1 : 0
27+
28+
enable_serverless_compute = true
29+
}
30+
2431
resource "databricks_sql_endpoint" "this" {
25-
for_each = var.sql_endpoint
26-
27-
name = "${each.key}-${var.project}-${var.env}${local.suffix}"
28-
cluster_size = lookup(each.value, "cluster_size", var.default_values_sql_endpoint["cluster_size"])
29-
min_num_clusters = lookup(each.value, "min_num_clusters", var.default_values_sql_endpoint["min_num_clusters"])
30-
max_num_clusters = lookup(each.value, "max_num_clusters", var.default_values_sql_endpoint["max_num_clusters"])
31-
auto_stop_mins = lookup(each.value, "auto_stop_mins", var.default_values_sql_endpoint["auto_stop_mins"])
32-
enable_photon = lookup(each.value, "enable_photon", var.default_values_sql_endpoint["enable_photon"])
33-
enable_serverless_compute = lookup(each.value, "enable_serverless_compute", var.default_values_sql_endpoint["enable_serverless_compute"])
32+
for_each = { for endpoint in var.sql_endpoint : (endpoint.name) => endpoint }
33+
34+
name = "${each.key}${local.suffix}"
35+
cluster_size = coalesce(each.value.cluster_size, "2X-Small")
36+
min_num_clusters = coalesce(each.value.min_num_clusters, 0)
37+
max_num_clusters = coalesce(each.value.max_num_clusters, 1)
38+
auto_stop_mins = coalesce(each.value.auto_stop_mins, "30")
39+
enable_photon = coalesce(each.value.enable_photon, false)
40+
enable_serverless_compute = coalesce(each.value.enable_serverless_compute, false)
41+
spot_instance_policy = coalesce(each.value.spot_instance_policy, "COST_OPTIMIZED")
42+
warehouse_type = coalesce(each.value.warehouse_type, "PRO")
43+
44+
lifecycle {
45+
ignore_changes = [state, num_clusters]
46+
precondition {
47+
condition = each.value.enable_serverless_compute != null ? (each.value.enable_serverless_compute ? each.value.warehouse_type == "PRO" : false) : true
48+
error_message = "To create Serverless SQL Endpoint, please set 'warehouse_type' parameter value to 'PRO' explicitly"
49+
}
50+
}
51+
depends_on = [databricks_sql_global_config.this]
3452
}

‎permissions.tf

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -38,21 +38,20 @@ resource "databricks_permissions" "cluster_policy" {
3838
}
3939

4040
resource "databricks_permissions" "sql_endpoint" {
41-
for_each = { for entry in databricks_sql_endpoint.this : (entry.name) => (entry.id) }
41+
for_each = {
42+
for endpoint in var.sql_endpoint : (endpoint.name) => endpoint
43+
if endpoint.permissions != null
44+
}
4245

43-
sql_endpoint_id = each.value
46+
sql_endpoint_id = databricks_sql_endpoint.this[each.key].id
4447

4548
dynamic "access_control" {
46-
for_each = { for entry in flatten([for resource, permissions in var.iam_permissions : [for permission, groups in permissions : [for group in groups : {
47-
resource = resource, permission = permission, group = group
48-
} if resource == "sql_endpoint"]]]) : "${entry.resource}.${entry.permission}.${entry.group}" => entry }
49+
for_each = each.value.permissions
4950
content {
50-
group_name = access_control.value.group
51-
permission_level = access_control.value.permission
51+
group_name = databricks_group.this[access_control.value.group_name].display_name
52+
permission_level = access_control.value.permission_level
5253
}
5354
}
54-
55-
depends_on = [databricks_group.this]
5655
}
5756

5857
resource "databricks_secret_acl" "this" {

‎variables.tf

Lines changed: 11 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -61,20 +61,6 @@ variable "iam" {
6161
}
6262
}
6363

64-
variable "iam_permissions" {
65-
type = map(object({
66-
CAN_USE = list(string)
67-
CAN_MANAGE = list(string)
68-
}))
69-
description = "Map of permission for groups. You can provide certain permission on services to groups. E.g., `'sql_endpoint'={'CAN_USE'=['group1', 'group2'] CAN_MANAGE=['group3']}"
70-
default = {
71-
"sql_endpoint" = {
72-
"CAN_USE" = ["default"]
73-
"CAN_MANAGE" = []
74-
}
75-
}
76-
}
77-
7864
# Default Cluster and Cluster Policy variables
7965
variable "default_cluster_id" {
8066
type = map(string)
@@ -98,36 +84,23 @@ variable "cluster_policies_object" {
9884

9985
# SQL Endpoint variables
10086
variable "sql_endpoint" {
101-
type = map(object({
102-
cluster_size = string
87+
type = set(object({
88+
name = string
89+
cluster_size = optional(string)
10390
min_num_clusters = optional(number)
10491
max_num_clusters = optional(number)
10592
auto_stop_mins = optional(string)
10693
enable_photon = optional(bool)
10794
enable_serverless_compute = optional(bool)
95+
spot_instance_policy = optional(string)
96+
warehouse_type = optional(string)
97+
permissions = optional(set(object({
98+
group_name = string
99+
permission_level = string
100+
})))
108101
}))
109-
description = "Map of SQL Endoints to be deployed in Databricks Workspace"
110-
default = {}
111-
}
112-
113-
variable "default_values_sql_endpoint" {
114-
description = "Default values for SQL Endpoint"
115-
type = object({
116-
cluster_size = string
117-
min_num_clusters = number
118-
max_num_clusters = number
119-
auto_stop_mins = string
120-
enable_photon = bool
121-
enable_serverless_compute = bool
122-
})
123-
default = {
124-
cluster_size = "2X-Small"
125-
min_num_clusters = 0
126-
max_num_clusters = 1
127-
auto_stop_mins = "30"
128-
enable_photon = false
129-
enable_serverless_compute = false
130-
}
102+
description = "Set of objects with parameters to configure SQL Endpoint and assign permissions to it for certain custom groups"
103+
default = []
131104
}
132105

133106
# Unity Catalog variables

0 commit comments

Comments
 (0)