Skip to content

Commit a36c1f3

Browse files
authored
Merge pull request #10 from data-platform-hq/feat_removed_metastore_creation
feat: removed metastore creation resources
2 parents 063478f + 4a77af2 commit a36c1f3

File tree

5 files changed

+80
-191
lines changed

5 files changed

+80
-191
lines changed

README.md

Lines changed: 47 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -2,39 +2,28 @@
22
Terraform module for creation Azure Unity Catalog
33

44
## Usage
5+
This module manages Unity Catalog resources like Catalogs, Schemas. In addition, it is possible to manage permissions within Metastore, Catalog and Schemas.
56
```hcl
6-
# Prerequisite resources
7-
87
# Configure Databricks Provider
98
data "azurerm_databricks_workspace" "example" {
109
name = "example-workspace"
1110
resource_group_name = "example-rg"
1211
}
1312
1413
provider "databricks" {
15-
alias = "main"
14+
alias = "workspace"
1615
host = data.databricks_workspace.example.workspace_url
1716
azure_workspace_resource_id = data.databricks_workspace.example.id
1817
}
1918
20-
# Databricks Access Connector (managed identity)
21-
resource "azurerm_databricks_access_connector" "example" {
22-
name = "example-resource"
23-
resource_group_name = "example-rg"
24-
location = "eastus"
25-
26-
identity {
27-
type = "SystemAssigned"
28-
}
29-
}
30-
31-
# Storage Account
32-
data "azurerm_storage_account" "example" {
33-
name = "example-storage-account"
34-
resource_group_name = "example-rg"
35-
}
36-
3719
locals {
20+
metastore_id = "10000000-0000-0000-0000-0000000000"
21+
22+
metastore_grants = [
23+
{ principal = "<user1@email.com>", privileges = ["CREATE_CATALOG","CREATE_EXTERNAL_LOCATION"] },
24+
{ principal = "<user2@epam.com>", privileges = ["CREATE_SHARE", "CREATE_RECIPIENT", "CREATE_PROVIDER"] }
25+
]
26+
3827
catalog = {
3928
example_catalog = {
4029
catalog_grants = {
@@ -45,20 +34,34 @@ locals {
4534
}
4635
}
4736
37+
# Prerequisite module.
38+
# NOTE! It is required to assign Metastore to Workspace before creating Unity Catalog resources.
39+
module "metastore_assignment" {
40+
source = "data-platform-hq/metastore-assignment/databricks"
41+
version = "~> 1.0.0"
42+
43+
workspace_id = data.databricks_workspace.example.id
44+
metastore_id = local.metastore_id
45+
46+
providers = {
47+
databricks = databricks.workspace
48+
}
49+
}
50+
4851
module "unity_catalog" {
49-
source = "../environment/modules/unity"
52+
source = "data-platform/unity-catalog/databricks"
53+
version = "~> 1.1.0"
5054
51-
project = "datahq"
52-
env = "example"
53-
location = "eastus"
54-
access_connector_id = azurerm_databricks_access_connector.example.id
55-
storage_account_id = data.azurerm_storage_account.example.id
56-
storage_account_name = data.azurerm_storage_account.example.name
57-
catalog = local.catalog
55+
env = "example"
56+
metastore_id = local.metastore_id
57+
metastore_grants = local.metastore_grants
58+
catalog = local.catalog
5859
5960
providers = {
60-
databricks = databricks.main
61+
databricks = databricks.workspace
6162
}
63+
64+
depends_on = [module.metastore_assignment]
6265
}
6366
```
6467
<!-- BEGIN_TF_DOCS -->
@@ -68,14 +71,14 @@ module "unity_catalog" {
6871
| ------------------------------------------------------------------------- | --------- |
6972
| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0.0 |
7073
| <a name="requirement_databricks"></a> [databricks](#requirement\_databricks) | >= 1.14.2 |
71-
| <a name="requirement_azurerm"></a> [azurerm](#requirement\_azurerm) | >= 3.40.0 |
74+
7275

7376
## Providers
7477

7578
| Name | Version |
7679
| ------------------------------------------------------------- | --------- |
7780
| <a name="provider_databricks"></a> [databricks](#provider\_databricks) | 1.14.2 |
78-
| <a name="provider_azurerm"></a> [azurerm](#provider\_azurerm) | 3.40.0 |
81+
7982

8083
## Modules
8184

@@ -85,38 +88,29 @@ No modules.
8588

8689
| Name | Type |
8790
| ------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- |
88-
| [azurerm_storage_data_lake_gen2_filesystem.this](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_data_lake_gen2_filesystem) | resource |
89-
| [databricks_metastore.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/metastore) | resource |
90-
| [databricks_grants.metastore](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grants) | resource |
91-
| [databricks_metastore_data_access.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/metastore_data_access) | resource |
92-
| [databricks_catalog.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/catalog) | resource |
93-
| [databricks_grants.catalog](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grants) | resource |
94-
| [databricks_schema.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/sql_endpoint) | resource |
95-
| [databricks_grants.schema](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/schema) | resource |
91+
| [databricks_grants.metastore](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grants) | resource |
92+
| [databricks_catalog.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/catalog) | resource |
93+
| [databricks_grants.catalog](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grants) | resource |
94+
| [databricks_schema.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/schema) | resource |
95+
| [databricks_grants.schema](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grants) | resource |
96+
9697

9798
## Inputs
9899

99100
| Name | Description | Type | Default | Required |
100101
| -------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | -------------- | ------- | :------: |
101-
| <a name="input_project"></a> [project](#input\_project) | Project name | `string` | n/a | yes |
102-
| <a name="input_env"></a> [env](#input\_env) | Environment name | `string` | n/a | yes |
103-
| <a name="input_location"></a> [location](#input\_location) | Azure location | `string` | n/a | yes |
104-
| <a name="input_suffix"></a> [suffix](#input\_suffix) | Optional suffix that would be added to the end of resources names. | `string` | " " | no |
105-
| <a name="input_create_metastore"></a> [create\_metastore](#input\_create\_metastore) | Boolean flag for Unity Catalog Metastore current in this environment. One Metastore per region | `bool` | true | no |
106-
| <a name="input_access_connector_id"></a> [access\_connector\_id](#input\_access\_connector\_id) | Databricks Access Connector Id that lets you to connect managed identities to an Azure Databricks account. Provides an ability to access Unity Catalog with assigned identity | `string` | " " | no |
107-
| <a name="input_storage_account_id"></a> [storage\_account\_id](#input\_storage\_account\_id) | Storage Account Id where Unity Catalog Metastore would be provisioned | `string` | " " | no |
108-
| <a name="input_storage_account_name"></a> [storage\_account\_name](#input\_storage\_account\_name) | Storage Account Name where Unity Catalog Metastore would be provisioned | `string` | " " | no |
109-
| <a name="input_external_metastore_id"></a> [external\_metastore\_id](#input\_external\_metastore\_id) | Unity Catalog Metastore Id that is located in separate environment. Provide this value to associate Databricks Workspace with target Metastore | `string` | " " | no |
110-
| <a name="input_catalog"></a> [catalog](#input\_catalog) | Map of objects which parameters refers to certain catalog and schema attributes | <pre> map(object({ <br> catalog_grants = optional(map(list(string))) <br> catalog_comment = optional(string) <br> catalog_properties = optional(map(string)) <br> schema_name = optional(list(string)) <br> schema_grants = optional(map(list(string))) <br> schema_comment = optional(string) <br> schema_properties = optional(map(string))<br>})) </pre> | {} | no |
111-
| <a name="input_metastore_grants"></a> [metastore\_grants](#input\_metastore\_grants) | Permissions to give on metastore to group | `map(list(string))` | {} | no |
112-
| <a name="input_custom_databricks_metastore_name"></a> [custom\_databricks\_metastore\_name](#input\_custom\_databricks\_metastore\_name) | The name to provide for your Databricks Metastore | `string` | null | no |
102+
| <a name="input_env"></a> [env](#input\_project)| Environment name | `string`| n/a | yes |
103+
| <a name="input_metastore_id"></a> [metastore\_id](#input\_metastore\_id)| Unity Catalog Metastore Id that is located in separate environment. Provide this value to associate Databricks Workspace with target Metastore| `string` | n/a | yes |
104+
| <a name="input_metastore_grants"></a> [metastore\_grants](#input\_metastore\_grants)| Permissions to give on metastore to group | <pre>set(object({<br> principal = string<br> privileges = list(string)<br>}))</pre>| [] | no |
105+
| <a name="input_catalog"></a> [catalog](#input\_catalog)| Map of catalog name and its parameters | <pre>map(object({<br> catalog_grants = optional(map(list(string)))<br> catalog_comment = optional(string)<br> catalog_properties = optional(map(string))<br> schema_name = optional(list(string))<br> schema_grants = optional(map(list(string)))<br> schema_comment = optional(string)<br> schema_properties = optional(map(string))<br>}))</pre>|{} | no |
106+
113107

114108
## Outputs
115109

116110
| Name | Description |
117111
| -------------------------------------------------------------------------- | -------------------------------------- |
118-
| <a name="output_metastore_id"></a> [metastore\_id](#output\_metastore\_id) | Unity Catalog Metastore Id. |
119-
| <a name="output_data_lake_gen2_file_system_id"></a> [data\_lake\_gen2\_file\_syste_id](#output\_data\_lake\_gen2\_file\_syste_id) | The ID of the Data Lake Gen2 File System. |
112+
113+
No outputs.
120114
<!-- END_TF_DOCS -->
121115

122116
## License

main.tf

Lines changed: 19 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,63 +1,29 @@
1+
# Metastore grants
12
locals {
2-
# This optional suffix is added to the end of resource names.
3-
suffix = length(var.suffix) == 0 ? "" : "-${var.suffix}"
4-
databricks_metastore_name = var.custom_databricks_metastore_name == null ? "meta-${var.project}-${var.env}-${var.location}${local.suffix}" : var.custom_databricks_metastore_name
5-
databricks_metastore_container_name = var.custom_databricks_metastore_container_name == null ? "meta-${var.project}-${var.env}" : var.custom_databricks_metastore_container_name
6-
}
7-
8-
resource "azurerm_storage_data_lake_gen2_filesystem" "this" {
9-
count = var.create_metastore ? 1 : 0
10-
11-
name = local.databricks_metastore_container_name
12-
storage_account_id = var.storage_account_id
13-
14-
lifecycle {
15-
precondition {
16-
condition = alltrue([
17-
for variable in [var.storage_account_id, var.access_connector_id, var.storage_account_name] : false if length(variable) == 0
18-
])
19-
error_message = "To create Metastore in a Region it is required to provide proper values for these variables: access_connector_id, storage_account_id, storage_account_name"
20-
}
3+
mapped_metastore_grants = {
4+
for object in var.metastore_grants : object.principal => object
5+
if object.principal != null
216
}
227
}
238

24-
resource "databricks_metastore" "this" {
25-
count = var.create_metastore ? 1 : 0
26-
27-
name = local.databricks_metastore_name
28-
storage_root = format("abfss://%s@%s.dfs.core.windows.net/", azurerm_storage_data_lake_gen2_filesystem.this[0].name, var.storage_account_name)
29-
force_destroy = true
30-
}
31-
329
resource "databricks_grants" "metastore" {
33-
count = length(var.metastore_grants) != 0 ? 1 : 0
10+
count = length(local.mapped_metastore_grants) != 0 ? 1 : 0
3411

35-
metastore = length(var.external_metastore_id) == 0 ? databricks_metastore.this[0].id : var.external_metastore_id
12+
metastore = var.metastore_id
3613
dynamic "grant" {
37-
for_each = var.metastore_grants
14+
for_each = local.mapped_metastore_grants
3815
content {
39-
principal = grant.key
40-
privileges = grant.value
16+
principal = grant.value.principal
17+
privileges = grant.value.privileges
4118
}
4219
}
4320
}
4421

45-
resource "databricks_metastore_data_access" "this" {
46-
count = var.create_metastore ? 1 : 0
47-
48-
metastore_id = databricks_metastore.this[0].id
49-
name = "data-access-${var.project}-${var.env}-${var.location}${local.suffix}"
50-
azure_managed_identity {
51-
access_connector_id = var.access_connector_id
52-
}
53-
is_default = true
54-
}
55-
56-
# Catalog
22+
# Catalog creation
5723
resource "databricks_catalog" "this" {
58-
for_each = anytrue([var.create_metastore, length(var.external_metastore_id) != 0]) ? var.catalog : {}
24+
for_each = var.catalog
5925

60-
metastore_id = length(var.external_metastore_id) == 0 ? databricks_metastore.this[0].id : var.external_metastore_id
26+
metastore_id = var.metastore_id
6127
name = each.key
6228
comment = lookup(each.value, "catalog_comment", "default comment")
6329
properties = merge(lookup(each.value, "catalog_properties", {}), { env = var.env })
@@ -66,10 +32,10 @@ resource "databricks_catalog" "this" {
6632

6733
# Catalog grants
6834
resource "databricks_grants" "catalog" {
69-
for_each = anytrue([var.create_metastore, length(var.external_metastore_id) != 0]) ? {
35+
for_each = {
7036
for name, params in var.catalog : name => params.catalog_grants
7137
if params.catalog_grants != null
72-
} : {}
38+
}
7339

7440
catalog = databricks_catalog.this[each.key].name
7541
dynamic "grant" {
@@ -81,7 +47,7 @@ resource "databricks_grants" "catalog" {
8147
}
8248
}
8349

84-
# Schema
50+
# Schema creation
8551
locals {
8652
schema = flatten([
8753
for catalog, params in var.catalog : [
@@ -96,9 +62,9 @@ locals {
9662
}
9763

9864
resource "databricks_schema" "this" {
99-
for_each = anytrue([var.create_metastore, length(var.external_metastore_id) != 0]) ? {
65+
for_each = {
10066
for entry in local.schema : "${entry.catalog}.${entry.schema}" => entry
101-
} : {}
67+
}
10268

10369
catalog_name = databricks_catalog.this[each.value.catalog].name
10470
name = each.value.schema
@@ -120,9 +86,9 @@ locals {
12086
}
12187

12288
resource "databricks_grants" "schema" {
123-
for_each = anytrue([var.create_metastore, length(var.external_metastore_id) != 0]) ? {
89+
for_each = {
12490
for entry in local.schema_grants : "${entry.catalog}.${entry.schema}.${entry.principal}" => entry
125-
} : {}
91+
}
12692

12793
schema = databricks_schema.this["${each.value.catalog}.${each.value.schema}"].id
12894
grant {

outputs.tf

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +0,0 @@
1-
output "metastore_id" {
2-
value = var.create_metastore ? databricks_metastore.this[0].id : ""
3-
description = "Unity Catalog Metastore Id"
4-
}
5-
6-
output "data_lake_gen2_file_system_id" {
7-
value = var.create_metastore ? azurerm_storage_data_lake_gen2_filesystem.this[0].id : ""
8-
description = "The ID of the Data Lake Gen2 File System."
9-
}

variables.tf

Lines changed: 14 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -1,59 +1,28 @@
1-
variable "project" {
2-
type = string
3-
description = "Project name"
4-
}
5-
61
variable "env" {
72
type = string
83
description = "Environment name"
94
}
105

11-
variable "location" {
12-
type = string
13-
description = "Azure location"
14-
}
15-
16-
variable "suffix" {
17-
type = string
18-
description = "Optional suffix that would be added to the end of resources names."
19-
default = ""
20-
}
21-
22-
# Unity Catalog variables
23-
variable "create_metastore" {
24-
type = bool
25-
description = "Boolean flag for Unity Catalog Metastore current in this environment. One Metastore per region"
26-
default = false
27-
}
28-
29-
variable "access_connector_id" {
30-
type = string
31-
description = "Databricks Access Connector Id that lets you to connect managed identities to an Azure Databricks account. Provides an ability to access Unity Catalog with assigned identity"
32-
default = ""
33-
}
34-
35-
variable "storage_account_id" {
36-
type = string
37-
description = "Storage Account Id where Unity Catalog Metastore would be provisioned"
38-
default = ""
39-
}
40-
41-
variable "storage_account_name" {
42-
type = string
43-
description = "Storage Account Name where Unity Catalog Metastore would be provisioned"
44-
default = ""
45-
}
46-
47-
variable "external_metastore_id" {
6+
variable "metastore_id" {
487
type = string
498
description = "Unity Catalog Metastore Id that is located in separate environment. Provide this value to associate Databricks Workspace with target Metastore"
50-
default = ""
9+
5110
validation {
52-
condition = length(var.external_metastore_id) == 36 || length(var.external_metastore_id) == 0
53-
error_message = "UUID has to be either in nnnnnnnn-nnnn-nnnn-nnnn-nnnnnnnnnnnn format or empty string"
11+
condition = length(var.metastore_id) == 36
12+
error_message = "Create Metastore or connect to existing one in Init Layer using Private Endpoint. In case Unity Catalog is not required, remove 'databricks_catalog' variable from tfvars file."
5413
}
5514
}
5615

16+
# Metastore grants
17+
variable "metastore_grants" {
18+
type = set(object({
19+
principal = string
20+
privileges = list(string)
21+
}))
22+
description = "Permissions to give on metastore to group"
23+
default = []
24+
}
25+
5726
variable "catalog" {
5827
type = map(object({
5928
catalog_grants = optional(map(list(string)))
@@ -67,30 +36,3 @@ variable "catalog" {
6736
description = "Map of catalog name and its parameters"
6837
default = {}
6938
}
70-
71-
# Metastore grants
72-
variable "metastore_grants" {
73-
type = map(list(string))
74-
description = "Permissions to give on metastore to group"
75-
default = {}
76-
validation {
77-
condition = values(var.metastore_grants) != null ? alltrue([
78-
for item in toset(flatten([for group, params in var.metastore_grants : params if params != null])) : contains([
79-
"CREATE_CATALOG", "CREATE_EXTERNAL_LOCATION", "CREATE_SHARE", "CREATE_RECIPIENT", "CREATE_PROVIDER"
80-
], item)
81-
]) : true
82-
error_message = "Metastore permission validation. The only possible values for permissions are: CREATE_CATALOG, CREATE_EXTERNAL_LOCATION, CREATE_SHARE, CREATE_RECIPIENT, CREATE_PROVIDER"
83-
}
84-
}
85-
86-
variable "custom_databricks_metastore_name" {
87-
type = string
88-
description = "The name to provide for your Databricks Metastore"
89-
default = null
90-
}
91-
92-
variable "custom_databricks_metastore_container_name" {
93-
type = string
94-
description = "The name to provide for your Databricks Metastore Container"
95-
default = null
96-
}

0 commit comments

Comments
 (0)