Skip to content

Commit 58ffa3c

Browse files
owlleg6owlleg6
committed
feat: unity catalog
1 parent 180f550 commit 58ffa3c

File tree

6 files changed

+287
-53
lines changed

6 files changed

+287
-53
lines changed

‎README.md

Lines changed: 52 additions & 29 deletions
Large diffs are not rendered by default.

‎iam.tf

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -101,18 +101,3 @@ resource "databricks_permissions" "sql_endpoint" {
101101

102102
depends_on = [databricks_group.this]
103103
}
104-
105-
resource "databricks_permissions" "token" {
106-
authorization = "tokens"
107-
108-
dynamic "access_control" {
109-
for_each = { for entry in flatten([for resource, permissions in var.iam_permissions : [for permission, groups in permissions : [for group in groups : {
110-
resource = resource, permission = permission, group = group
111-
} if resource == "token"]]]) : "${entry.resource}.${entry.permission}.${entry.group}" => entry }
112-
content {
113-
group_name = access_control.value.group
114-
permission_level = access_control.value.permission
115-
}
116-
}
117-
depends_on = [databricks_group.this]
118-
}

‎main.tf

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
locals {
22
ip_rules = var.ip_rules == null ? null : values(var.ip_rules)
3+
suffix = length(var.suffix) == 0 ? "" : "-${var.suffix}"
34
}
45

56
resource "databricks_workspace_conf" "this" {
@@ -23,7 +24,7 @@ resource "databricks_ip_access_list" "this" {
2324
resource "databricks_sql_endpoint" "this" {
2425
for_each = var.sql_endpoint
2526

26-
name = "${each.key}-${var.project}-${var.env}"
27+
name = "${each.key}-${var.project}-${var.env}${local.suffix}"
2728
cluster_size = lookup(each.value, "cluster_size", var.default_values_sql_endpoint["cluster_size"])
2829
min_num_clusters = lookup(each.value, "min_num_clusters", var.default_values_sql_endpoint["min_num_clusters"])
2930
max_num_clusters = lookup(each.value, "max_num_clusters", var.default_values_sql_endpoint["max_num_clusters"])

‎outputs.tf

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,8 @@ output "sql_endpoint_data_source_id" {
77
value = [for n in databricks_sql_endpoint.this : n.data_source_id]
88
description = "ID of the data source for this endpoint"
99
}
10+
11+
output "metastore_id" {
12+
value = var.create_metastore ? databricks_metastore.this[0].id : ""
13+
description = "Unity Catalog Metastore Id"
14+
}

‎unity.tf

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
resource "azurerm_storage_data_lake_gen2_filesystem" "this" {
2+
count = var.create_metastore ? 1 : 0
3+
4+
name = "meta-${var.project}-${var.env}"
5+
storage_account_id = var.storage_account_id
6+
7+
lifecycle {
8+
precondition {
9+
condition = alltrue([
10+
for variable in [var.storage_account_id, var.access_connector_id, var.storage_account_name] : false if length(variable) == 0
11+
])
12+
error_message = "To create Metastore in a Region it is required to provide proper values for these variables: access_connector_id, storage_account_id, storage_account_name"
13+
}
14+
}
15+
}
16+
17+
resource "databricks_metastore" "this" {
18+
count = var.create_metastore ? 1 : 0
19+
20+
name = "meta-${var.project}-${var.env}-${var.location}${local.suffix}"
21+
storage_root = format("abfss://%s@%s.dfs.core.windows.net/", azurerm_storage_data_lake_gen2_filesystem.this[0].name, var.storage_account_name)
22+
force_destroy = true
23+
}
24+
25+
resource "databricks_grants" "metastore" {
26+
for_each = !var.create_metastore && length(var.external_metastore_id) == 0 ? {} : {
27+
for k, v in var.metastore_grants : k => v
28+
if v != null
29+
}
30+
31+
metastore = length(var.external_metastore_id) == 0 ? databricks_metastore.this[0].id : var.external_metastore_id
32+
grant {
33+
principal = each.key
34+
privileges = each.value
35+
}
36+
}
37+
38+
resource "databricks_metastore_data_access" "this" {
39+
count = var.create_metastore ? 1 : 0
40+
41+
metastore_id = databricks_metastore.this[0].id
42+
name = "data-access-${var.project}-${var.env}-${var.location}${local.suffix}"
43+
azure_managed_identity {
44+
access_connector_id = var.access_connector_id
45+
}
46+
is_default = true
47+
}
48+
49+
resource "databricks_metastore_assignment" "this" {
50+
count = !var.create_metastore && length(var.external_metastore_id) == 0 ? 0 : 1
51+
52+
workspace_id = var.workspace_id
53+
metastore_id = length(var.external_metastore_id) == 0 ? databricks_metastore.this[0].id : var.external_metastore_id
54+
default_catalog_name = "hive_metastore"
55+
}
56+
57+
# Catalog
58+
resource "databricks_catalog" "this" {
59+
for_each = !var.create_metastore && length(var.external_metastore_id) == 0 ? {} : var.catalog
60+
61+
metastore_id = length(var.external_metastore_id) == 0 ? databricks_metastore.this[0].id : var.external_metastore_id
62+
name = each.key
63+
comment = lookup(each.value, "catalog_comment", "default comment")
64+
properties = merge(lookup(each.value, "catalog_properties", {}), { env = var.env })
65+
force_destroy = true
66+
67+
depends_on = [databricks_metastore_assignment.this[0]]
68+
}
69+
70+
# Catalog grants
71+
resource "databricks_grants" "catalog" {
72+
for_each = !var.create_metastore && length(var.external_metastore_id) == 0 ? {} : {
73+
for name, params in var.catalog : name => params.catalog_grants
74+
if params.catalog_grants != null
75+
}
76+
77+
catalog = databricks_catalog.this[each.key].name
78+
dynamic "grant" {
79+
for_each = each.value
80+
content {
81+
principal = grant.key
82+
privileges = grant.value
83+
}
84+
}
85+
}
86+
87+
# Schema
88+
locals {
89+
schema = flatten([
90+
for catalog, params in var.catalog : [
91+
for schema in params.schema_name : {
92+
catalog = catalog,
93+
schema = schema,
94+
comment = lookup(params, "schema_comment", "default comment"),
95+
properties = lookup(params, "schema_properties", {})
96+
}
97+
] if params.schema_name != null
98+
])
99+
}
100+
101+
resource "databricks_schema" "this" {
102+
for_each = !var.create_metastore && length(var.external_metastore_id) == 0 ? {} : {
103+
for entry in local.schema : "${entry.catalog}.${entry.schema}" => entry
104+
}
105+
106+
catalog_name = databricks_catalog.this[each.value.catalog].name
107+
name = each.value.schema
108+
comment = each.value.comment
109+
properties = merge(each.value.properties, { env = var.env })
110+
force_destroy = true
111+
}
112+
113+
# Schema grants
114+
locals {
115+
schema_grants = flatten([
116+
for catalog, params in var.catalog : [for schema in params.schema_name : [for principal in flatten(keys(params.schema_grants)) : {
117+
catalog = catalog,
118+
schema = schema,
119+
principal = principal,
120+
permission = flatten(values(params.schema_grants)),
121+
}]] if params.schema_grants != null
122+
])
123+
}
124+
125+
resource "databricks_grants" "schema" {
126+
for_each = !var.create_metastore && length(var.external_metastore_id) == 0 ? {} : {
127+
for entry in local.schema_grants : "${entry.catalog}.${entry.schema}.${entry.principal}" => entry
128+
}
129+
130+
schema = databricks_schema.this["${each.value.catalog}.${each.value.schema}"].id
131+
grant {
132+
principal = each.value.principal
133+
privileges = each.value.permission
134+
}
135+
}

‎variables.tf

Lines changed: 93 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,26 @@
1+
variable "project" {
2+
type = string
3+
description = "Project name"
4+
}
5+
16
variable "env" {
27
type = string
38
description = "Environment name"
49
}
510

6-
variable "project" {
11+
variable "location" {
712
type = string
8-
description = "Project name"
13+
description = "Azure location"
14+
}
15+
16+
variable "sku" {
17+
type = string
18+
description = "The sku to use for the Databricks Workspace: [standard|premium|trial]"
19+
}
20+
21+
variable "workspace_id" {
22+
type = string
23+
description = "Id of Azure Databricks workspace"
924
}
1025

1126
variable "user_object_ids" {
@@ -19,7 +34,7 @@ variable "workspace_admins" {
1934
user = list(string)
2035
service_principal = list(string)
2136
})
22-
description = "Provide users or service principals to grant them Admin permissions."
37+
description = "Provide users or service principals to grant them Admin permissions in Workspace."
2338
default = {
2439
user = null
2540
service_principal = null
@@ -54,10 +69,6 @@ variable "iam_permissions" {
5469
"CAN_USE" = ["default"]
5570
"CAN_MANAGE" = []
5671
}
57-
"token" = {
58-
"CAN_USE" = ["default"]
59-
"CAN_MANAGE" = []
60-
}
6172
}
6273
}
6374

@@ -68,7 +79,14 @@ variable "ip_rules" {
6879
}
6980

7081
variable "sql_endpoint" {
71-
type = map(map(string))
82+
type = map(object({
83+
cluster_size = string
84+
min_num_clusters = optional(number)
85+
max_num_clusters = optional(number)
86+
auto_stop_mins = optional(string)
87+
enable_photon = optional(bool)
88+
enable_serverless_compute = optional(bool)
89+
}))
7290
description = "Map of SQL Endoints to be deployed in Databricks Workspace"
7391
default = {}
7492
}
@@ -92,3 +110,70 @@ variable "default_values_sql_endpoint" {
92110
enable_serverless_compute = false
93111
}
94112
}
113+
114+
variable "create_metastore" {
115+
type = bool
116+
description = "Boolean flag for Unity Catalog Metastore current in this environment. One Metastore per region"
117+
default = false
118+
}
119+
120+
variable "access_connector_id" {
121+
type = string
122+
description = "Databricks Access Connector Id that lets you to connect managed identities to an Azure Databricks account. Provides an ability to access Unity Catalog with assigned identity"
123+
default = ""
124+
}
125+
126+
variable "storage_account_id" {
127+
type = string
128+
description = "Storage Account Id where Unity Catalog Metastore would be provisioned"
129+
default = ""
130+
}
131+
variable "storage_account_name" {
132+
type = string
133+
description = "Storage Account Name where Unity Catalog Metastore would be provisioned"
134+
default = ""
135+
}
136+
137+
variable "catalog" {
138+
type = map(object({
139+
catalog_grants = optional(map(list(string)))
140+
catalog_comment = optional(string)
141+
catalog_properties = optional(map(string))
142+
schema_name = optional(list(string))
143+
schema_grants = optional(map(list(string)))
144+
schema_comment = optional(string)
145+
schema_properties = optional(map(string))
146+
}))
147+
description = "Map of catalog name and its parameters"
148+
default = {}
149+
}
150+
151+
variable "suffix" {
152+
type = string
153+
description = "Optional suffix that would be added to the end of resources names."
154+
default = ""
155+
}
156+
157+
variable "external_metastore_id" {
158+
type = string
159+
description = "Unity Catalog Metastore Id that is located in separate environment. Provide this value to associate Databricks Workspace with target Metastore"
160+
default = ""
161+
validation {
162+
condition = length(var.external_metastore_id) == 36 || length(var.external_metastore_id) == 0
163+
error_message = "UUID has to be either in nnnnnnnn-nnnn-nnnn-nnnn-nnnnnnnnnnnn format or empty string"
164+
}
165+
}
166+
167+
variable "metastore_grants" {
168+
type = map(list(string))
169+
description = "Permissions to give on metastore to group"
170+
default = {}
171+
validation {
172+
condition = values(var.metastore_grants) != null ? alltrue([
173+
for item in toset(flatten([for group, params in var.metastore_grants : params if params != null])) : contains([
174+
"CREATE_CATALOG", "CREATE_EXTERNAL_LOCATION", "CREATE_SHARE", "CREATE_RECIPIENT", "CREATE_PROVIDER"
175+
], item)
176+
]) : true
177+
error_message = "Metastore permission validation. The only possible values for permissions are: CREATE_CATALOG, CREATE_EXTERNAL_LOCATION, CREATE_SHARE, CREATE_RECIPIENT, CREATE_PROVIDER"
178+
}
179+
}

0 commit comments

Comments
 (0)