data-platform-hq · owlleg6 · Dec 5, 2024 · Dec 5, 2024 · Dec 5, 2024 · Dec 5, 2024
diff --git a/README.md b/README.md
@@ -17,45 +17,56 @@ provider "databricks" {
 }
 
 locals {
-  metastore_id = "10000000-0000-0000-0000-0000000000"
-
-  metastore_grants = [
-    { principal = "<user1@email.com>", privileges = ["CREATE_CATALOG","CREATE_EXTERNAL_LOCATION"] }, 
-    { principal = "<user2@epam.com>", privileges = ["CREATE_SHARE", "CREATE_RECIPIENT", "CREATE_PROVIDER"] }
+  catalog_config = [
+
+    # Catalog w/o grants
+    {
+      catalog_name = "catalog_with_no_grants"
+    },
+
+    # Catalog with grants
+    {
+      catalog_name = "catalog_with_grants"
+      catalog_grants = [
+        { principal = "account users", privileges = ["USE_CATALOG", "APPLY_TAG", "CREATE_SCHEMA", "SELECT"] }
+      ]
+    },
+
+    # Catalog with grants and schemas
+    {
+      catalog_name = "catalog_with_schemas"
+      catalog_grants = [{ principal = "account users", privileges = ["USE_CATALOG", "APPLY_TAG", "SELECT"] }]
+      schema_configs = [
+        { schema_name = "schema_01" },
+        { schema_name = "schema_02" }
+      ]
+    },
+
+    # Catalog with schemas where 'schema_01' and 'schema_02' have a default set of grants from 'schema_default_grants' parameter
+    # and 'schema_03' has its own set of grants managed with 'schema_custom_grants' parameter
+    {
+      catalog_name = "catalog_custom_schema_grants"
+      catalog_grants = [{ principal = "account users", privileges = ["USE_CATALOG", "APPLY_TAG"] }]
+      schema_default_grants = [{ principal = "account users", privileges = ["CREATE_TABLE", "SELECT"] }]
+      schema_configs = [
+        { schema_name = "schema_01" },
+        { schema_name = "schema_02" },
+        { 
+          schema_name = "schema_03", 
+          schema_custom_grants = [
+            { principal = "account users", privileges = ["CREATE_VOLUME", "READ_VOLUME", "WRITE_VOLUME", "SELECT"] },
+          ]
+        },
+      ]
+    },
   ]
-
-  catalog = {
-    example_catalog = {
-      catalog_grants = {
-        "example@username.com" = ["USE_CATALOG", "USE_SCHEMA", "CREATE_SCHEMA", "CREATE_TABLE", "SELECT", "MODIFY"]
-      }
-      schema_name = ["raw", "refined", "data_product"]
-    }
-  }
-}
-
-# Prerequisite module.
-# NOTE! It is required to assign Metastore to Workspace before creating Unity Catalog resources.
-module "metastore_assignment" {
-  source  = "data-platform-hq/metastore-assignment/databricks"
-  version = "~> 1.0.0"
-
-  workspace_id = data.databricks_workspace.example.id
-  metastore_id = local.metastore_id
-
-  providers = {
-    databricks = databricks.workspace
-  }
 }
 
 module "unity_catalog" {
   source  = "data-platform/unity-catalog/databricks"
-  version = "~> 1.1.0"
+  version = "~> 2.0.0"
 
-  env              = "example"
-  metastore_id     = local.metastore_id
-  metastore_grants = local.metastore_grants
-  catalog          = local.catalog
+  catalog_config = local.catalog_config
 
   providers = {
     databricks = databricks.workspace
@@ -88,7 +99,6 @@ No modules.
 |------|------|
 | [databricks_catalog.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/catalog) | resource |
 | [databricks_grants.catalog](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grants) | resource |
-| [databricks_grants.metastore](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grants) | resource |
 | [databricks_grants.schema](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grants) | resource |
 | [databricks_schema.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/schema) | resource |
 | [databricks_workspace_binding.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/workspace_binding) | resource |
@@ -97,10 +107,8 @@ No modules.
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_catalog"></a> [catalog](#input\_catalog) | Map of catalog name and its parameters | <pre>map(object({<br/>    catalog_grants         = optional(map(list(string)))<br/>    catalog_owner          = optional(string)         # Username/groupname/sp application_id of the catalog owner.<br/>    catalog_storage_root   = optional(string)         # Location in cloud storage where data for managed tables will be stored<br/>    catalog_isolation_mode = optional(string, "OPEN") # Whether the catalog is accessible from all workspaces or a specific set of workspaces. Can be ISOLATED or OPEN.<br/>    catalog_comment        = optional(string)         # User-supplied free-form text<br/>    catalog_properties     = optional(map(string))    # Extensible Catalog Tags.<br/>    schema_name            = optional(list(string))   # List of Schema names relative to parent catalog.<br/>    schema_grants          = optional(map(list(string)))<br/>    schema_owner           = optional(string) # Username/groupname/sp application_id of the schema owner.<br/>    schema_comment         = optional(string)<br/>    schema_properties      = optional(map(string))<br/>  }))</pre> | `{}` | no |
-| <a name="input_isolated_unmanaged_catalog_bindings"></a> [isolated\_unmanaged\_catalog\_bindings](#input\_isolated\_unmanaged\_catalog\_bindings) | List of objects with parameters to configure Catalog Bindings | <pre>list(object({<br/>    catalog_name = string                                      # Name of ISOLATED catalog<br/>    binding_type = optional(string, "BINDING_TYPE_READ_WRITE") # Binding mode. Possible values are BINDING_TYPE_READ_ONLY, BINDING_TYPE_READ_WRITE<br/>  }))</pre> | `[]` | no |
-| <a name="input_metastore_grants"></a> [metastore\_grants](#input\_metastore\_grants) | Permissions to give on metastore to user, group or service principal | <pre>set(object({<br/>    principal  = string<br/>    privileges = list(string)<br/>  }))</pre> | `[]` | no |
-| <a name="input_metastore_id"></a> [metastore\_id](#input\_metastore\_id) | Unity Catalog Metastore Id that is located in separate environment. Provide this value to associate Databricks Workspace with target Metastore | `string` | n/a | yes |
+| <a name="input_catalog_config"></a> [catalog\_config](#input\_catalog\_config) |  | <pre>list(object({<br/><br/>    # Catalog config<br/>    catalog_name           = string<br/>    catalog_owner          = optional(string)         # Username/groupname/sp application_id of the catalog owner.<br/>    catalog_storage_root   = optional(string)         # Location in cloud storage where data for managed tables will be stored<br/>    catalog_isolation_mode = optional(string, "OPEN") # Whether the catalog is accessible from all workspaces or a specific set of workspaces. Can be ISOLATED or OPEN.<br/>    catalog_comment        = optional(string)         # User-supplied free-form text<br/>    catalog_properties     = optional(map(string))    # Extensible Catalog Tags.<br/>    catalog_grants = optional(list(object({           # List of objects to set catalog permissions<br/>      principal  = string                             # Account level group name, user or service principal app ID<br/>      privileges = list(string)<br/>    })), [])<br/><br/>    # Schemas<br/>    schema_default_grants = optional(list(object({ # Sets default grants for each schema created by 'schema_configs' block w/o 'schema_custom_grants' parameter set<br/>      principal  = string                          # Account level group name, user or service principal app ID<br/>      privileges = list(string)<br/>    })), [])<br/><br/>    schema_configs = optional(list(object({<br/>      schema_name       = string<br/>      schema_owner      = optional(string)<br/>      schema_comment    = optional(string)<br/>      schema_properties = optional(map(string))<br/>      schema_custom_grants = optional(list(object({ # Overwrites 'schema_default_grants'<br/>        principal  = string                         # Account level group name, user or service principal app ID<br/>        privileges = list(string)<br/>      })), [])<br/>    })), [])<br/>  }))</pre> | `[]` | no |
+| <a name="input_isolated_unmanaged_catalog_bindings"></a> [isolated\_unmanaged\_catalog\_bindings](#input\_isolated\_unmanaged\_catalog\_bindings) |  | <pre>list(object({<br/>    catalog_name = string                                      # Name of ISOLATED catalog<br/>    binding_type = optional(string, "BINDING_TYPE_READ_WRITE") # Binding mode. Possible values are BINDING_TYPE_READ_ONLY, BINDING_TYPE_READ_WRITE<br/>  }))</pre> | `[]` | no |
 | <a name="input_workspace_id"></a> [workspace\_id](#input\_workspace\_id) | ID of the target workspace. | `string` | `null` | no |
 
 ## Outputs

diff --git a/main.tf b/main.tf
@@ -1,30 +1,21 @@
-# Metastore grants
 locals {
-  mapped_metastore_grants = {
-    for object in var.metastore_grants : object.principal => object
-    if object.principal != null
-  }
-}
+  catalogs_config_mapped = { for object in var.catalog_config : object.catalog_name => object }
 
-resource "databricks_grants" "metastore" {
-  count = length(local.mapped_metastore_grants) != 0 ? 1 : 0
-
-  metastore = var.metastore_id
-  dynamic "grant" {
-    for_each = local.mapped_metastore_grants
-    content {
-      principal  = grant.value.principal
-      privileges = grant.value.privileges
-    }
+  schemas_config_mapped = {
+    for config in flatten([for object in var.catalog_config : [for schema in object.schema_configs : {
+      catalog       = object.catalog_name
+      schema        = schema.schema_name
+      schema_owner  = schema.schema_owner
+      schema_grants = try(coalescelist(schema.schema_custom_grants, object.schema_default_grants), [])
+    }]]) : "${config.catalog}:${config.schema}" => config
   }
 }
 
 # Catalog creation
 resource "databricks_catalog" "this" {
-  for_each = var.catalog
+  for_each = local.catalogs_config_mapped
 
-  metastore_id   = var.metastore_id
-  name           = each.key
+  name           = each.value.catalog_name
   owner          = each.value.catalog_owner
   storage_root   = each.value.catalog_storage_root
   isolation_mode = each.value.catalog_isolation_mode
@@ -35,78 +26,49 @@ resource "databricks_catalog" "this" {
 
 # Catalog grants
 resource "databricks_grants" "catalog" {
-  for_each = {
-    for name, params in var.catalog : name => params.catalog_grants
-    if params.catalog_grants != null
-  }
+  for_each = { for k, v in local.catalogs_config_mapped : k => v if length(v.catalog_grants) != 0 }
 
   catalog = databricks_catalog.this[each.key].name
+
   dynamic "grant" {
-    for_each = each.value
+    for_each = each.value.catalog_grants
     content {
-      principal  = grant.key
-      privileges = grant.value
+      principal  = grant.value.principal
+      privileges = grant.value.privileges
     }
   }
 }
 
 # Schema creation
-locals {
-  schema = flatten([
-    for catalog, params in var.catalog : [
-      for schema in params.schema_name : {
-        catalog    = catalog,
-        schema     = schema,
-        comment    = lookup(params, "schema_comment", "default comment"),
-        properties = lookup(params, "schema_properties", {})
-        owner      = lookup(params, "schema_owner", null)
-      }
-    ] if params.schema_name != null
-  ])
-}
-
 resource "databricks_schema" "this" {
-  for_each = {
-    for entry in local.schema : "${entry.catalog}.${entry.schema}" => entry
-  }
+  for_each = local.schemas_config_mapped
 
   catalog_name  = databricks_catalog.this[each.value.catalog].name
   name          = each.value.schema
-  owner         = each.value.owner
-  comment       = each.value.comment
-  properties    = each.value.properties
+  owner         = each.value.schema_owner
+  comment       = each.value.schema_comment
+  properties    = each.value.schema_properties
   force_destroy = true
 }
 
 # Schema grants
-locals {
-  schema_grants = flatten([
-    for catalog, params in var.catalog : [for schema in params.schema_name : [for principal in flatten(keys(params.schema_grants)) : {
-      catalog    = catalog,
-      schema     = schema,
-      principal  = principal,
-      permission = flatten(values(params.schema_grants)),
-    }]] if params.schema_grants != null
-  ])
-}
-
 resource "databricks_grants" "schema" {
-  for_each = {
-    for entry in local.schema_grants : "${entry.catalog}.${entry.schema}.${entry.principal}" => entry
-  }
+  for_each = { for k, v in local.schemas_config_mapped : k => v if length(v.schema_grants) != 0 }
+
+  schema = databricks_schema.this[each.key].id
 
-  schema = databricks_schema.this["${each.value.catalog}.${each.value.schema}"].id
-  grant {
-    principal  = each.value.principal
-    privileges = each.value.permission
+  dynamic "grant" {
+    for_each = each.value.schema_grants
+    content {
+      principal  = grant.value.principal
+      privileges = grant.value.privileges
+    }
   }
 }
 
 # ISOLATED Catalogs binding
 resource "databricks_workspace_binding" "this" {
-  for_each = {
-    for object in var.isolated_unmanaged_catalog_bindings : object.catalog_name => object
-  }
+  for_each = { for object in var.isolated_unmanaged_catalog_bindings : object.catalog_name => object }
 
   workspace_id   = var.workspace_id
   securable_name = each.value.catalog_name

diff --git a/variables.tf b/variables.tf
@@ -1,47 +1,50 @@
-variable "metastore_id" {
-  type        = string
-  description = "Unity Catalog Metastore Id that is located in separate environment. Provide this value to associate Databricks Workspace with target Metastore"
-
-  validation {
-    condition     = length(var.metastore_id) == 36
-    error_message = "Create Metastore or connect to existing one in Init Layer using Private Endpoint. In case Unity Catalog is not required, remove 'databricks_catalog' variable from tfvars file."
-  }
-}
-
-# Metastore grants
-variable "metastore_grants" {
-  type = set(object({
-    principal  = string
-    privileges = list(string)
-  }))
-  description = "Permissions to give on metastore to user, group or service principal"
-  default     = []
-}
+variable "catalog_config" {
+  type = list(object({
 
-variable "catalog" {
-  type = map(object({
-    catalog_grants         = optional(map(list(string)))
+    # Catalog config
+    catalog_name           = string
     catalog_owner          = optional(string)         # Username/groupname/sp application_id of the catalog owner.
     catalog_storage_root   = optional(string)         # Location in cloud storage where data for managed tables will be stored
     catalog_isolation_mode = optional(string, "OPEN") # Whether the catalog is accessible from all workspaces or a specific set of workspaces. Can be ISOLATED or OPEN.
     catalog_comment        = optional(string)         # User-supplied free-form text
     catalog_properties     = optional(map(string))    # Extensible Catalog Tags.
-    schema_name            = optional(list(string))   # List of Schema names relative to parent catalog.
-    schema_grants          = optional(map(list(string)))
-    schema_owner           = optional(string) # Username/groupname/sp application_id of the schema owner.
-    schema_comment         = optional(string)
-    schema_properties      = optional(map(string))
+    catalog_grants = optional(list(object({           # List of objects to set catalog permissions
+      principal  = string                             # Account level group name, user or service principal app ID
+      privileges = list(string)
+    })), [])
+
+    # Schemas
+    schema_default_grants = optional(list(object({ # Sets default grants for each schema created by 'schema_configs' block w/o 'schema_custom_grants' parameter set
+      principal  = string                          # Account level group name, user or service principal app ID
+      privileges = list(string)
+    })), [])
+
+    schema_configs = optional(list(object({
+      schema_name       = string
+      schema_owner      = optional(string)
+      schema_comment    = optional(string)
+      schema_properties = optional(map(string))
+      schema_custom_grants = optional(list(object({ # Overwrites 'schema_default_grants'
+        principal  = string                         # Account level group name, user or service principal app ID
+        privileges = list(string)
+      })), [])
+    })), [])
   }))
-  description = "Map of catalog name and its parameters"
-  default     = {}
+  description = <<DESCRIPTION
+
+
+  DESCRIPTION
+  default     = []
 }
 
 variable "isolated_unmanaged_catalog_bindings" {
   type = list(object({
     catalog_name = string                                      # Name of ISOLATED catalog
     binding_type = optional(string, "BINDING_TYPE_READ_WRITE") # Binding mode. Possible values are BINDING_TYPE_READ_ONLY, BINDING_TYPE_READ_WRITE
   }))
-  description = "List of objects with parameters to configure Catalog Bindings"
+  description = <<DESCRIPTION
+
+  DESCRIPTION
   default     = []
 }