feat: remove carbon footprint export setup

JohannesRudolph · meshkodiak[bot] · commit 783b3b79672a · 2023-02-16T07:55:10.000Z
see #10 for justification
diff --git a/README.md b/README.md
@@ -32,11 +32,12 @@ To run this module, you need the following:
     # Only required when you need your landing zone to invoke a cloud function
     "cloudfunctions.functions.getIamPolicy",
     "cloudfunctions.functions.setIamPolicy",
+
     # Only required for the optional submodule for exporting carbon data  
     "resourcemanager.projects.update",
     "serviceusage.services.enable",
-    "bigquery.transfers.update"
-
+    "bigquery.datasets.get",
+    "bigquery.datasets.update"
     ```
 
   Organization-level:
@@ -118,6 +119,12 @@ resource "google_project" "meshstack_root" {
     The replicator service account needs the "Groups Admin" role from the Admin Console (Workspace) to manage permissions for managed GCP projects.
     To authorize the Service Account **via the Google Admin Console** navigate to `@Account` in the sidebar and then `Admin Roles -> Groups Admin` and click `Assign Service Accounts`. In the prompt that appears, enter the service account email, which looks like `user@project.iam.gserviceaccount.com`.
 
+6. Optional: Enable GCP Cloud Carbon Footprint Export Transfer
+
+    When configuring the module with `carbon_export_module_enabled = true`, you need to manually set up the GCP
+    data transfer config.
+
+
 ## Example Usages
 
 Check [examples](./examples/) for different use cases. As a quick start we recommend using [basic-gcp-integration](./examples/basic-gcp-integration) example.
@@ -162,11 +169,11 @@ No resources.
 | <a name="input_billing_account_id"></a> [billing\_account\_id](#input\_billing\_account\_id) | The GCP billing account in your organization. | `string` | n/a | yes |
 | <a name="input_billing_org_id"></a> [billing\_org\_id](#input\_billing\_org\_id) | GCP organization ID that holds billing account. | `string` | n/a | yes |
 | <a name="input_carbon_export_module_enabled"></a> [carbon\_export\_module\_enabled](#input\_carbon\_export\_module\_enabled) | Determines whether or not to include the resources of the carbon footprint export module. | `bool` | `false` | no |
-| <a name="input_carbon_footprint_dataset_id"></a> [carbon\_footprint\_dataset\_id](#input\_carbon\_footprint\_dataset\_id) | Id of BigQuery dataset for carbon footprint. | `string` | `"carbon_footprint_data"` | no |
-| <a name="input_carbon_footprint_dataset_location"></a> [carbon\_footprint\_dataset\_location](#input\_carbon\_footprint\_dataset\_location) | Location of BigQuery dataset for carbon footprint. | `string` | `"us-west1"` | no |
-| <a name="input_cloud_billing_export_dataset_id"></a> [cloud\_billing\_export\_dataset\_id](#input\_cloud\_billing\_export\_dataset\_id) | GCP BigQuery dataset containing the Cloud Billing BigQuery export. This variable is only required to form the output for meshPlatform configuration. No resources are created or attached. | `string` | n/a | yes |
+| <a name="input_cloud_billing_export_dataset_id"></a> [cloud\_billing\_export\_dataset\_id](#input\_cloud\_billing\_export\_dataset\_id) | GCP BigQuery dataset containing the Cloud Billing BigQuery export. | `string` | n/a | yes |
 | <a name="input_cloud_billing_export_project_id"></a> [cloud\_billing\_export\_project\_id](#input\_cloud\_billing\_export\_project\_id) | GCP Project where the BiqQuery table resides that holds the Cloud Billing export to BigQuery. See https://cloud.google.com/billing/docs/how-to/export-data-bigquery | `string` | n/a | yes |
 | <a name="input_cloud_billing_export_table_id"></a> [cloud\_billing\_export\_table\_id](#input\_cloud\_billing\_export\_table\_id) | GCP BigQuery table containing the Cloud Billing BigQuery export. This variable is only required to form the output for meshPlatform configuration. No resources are created or attached. | `string` | n/a | yes |
+| <a name="input_cloud_carbon_export_dataset_id"></a> [cloud\_carbon\_export\_dataset\_id](#input\_cloud\_carbon\_export\_dataset\_id) | GCP BigQuery dataset containing the Cloud Carbon Footprint BigQuery export. | `string` | n/a | yes |
+| <a name="input_cloud_carbon_export_project_id"></a> [cloud\_carbon\_export\_project\_id](#input\_cloud\_carbon\_export\_project\_id) | GCP Project where the BiqQuery table resides that holds the Cloud Carbon Footprint export to BigQuery. | `string` | n/a | yes |
 | <a name="input_kraken_sa_name"></a> [kraken\_sa\_name](#input\_kraken\_sa\_name) | Name of the service account to create for Kraken. | `string` | `"mesh-kraken-service-tf"` | no |
 | <a name="input_landing_zone_folder_ids"></a> [landing\_zone\_folder\_ids](#input\_landing\_zone\_folder\_ids) | GCP Folders that make up the Landing Zone. The service account will only receive permissions on these folders. | `list(string)` | n/a | yes |
 | <a name="input_org_id"></a> [org\_id](#input\_org\_id) | GCP Organization ID that holds the projects that generate billing data that the service account should import. | `string` | n/a | yes |
@@ -177,7 +184,9 @@ No resources.
 
 | Name | Description |
 |------|-------------|
+| <a name="output_carbon_footprint_export_manual_setup"></a> [carbon\_footprint\_export\_manual\_setup](#output\_carbon\_footprint\_export\_manual\_setup) | GCP Cloud Carbon Footprint BigQuery export manual setup information. |
 | <a name="output_carbon_footprint_export_table_name"></a> [carbon\_footprint\_export\_table\_name](#output\_carbon\_footprint\_export\_table\_name) | The BigQuery table name containing the GCP Carbon Footprint BigQuery export. |
+| <a name="output_cloud_billing_export_manual_setup"></a> [cloud\_billing\_export\_manual\_setup](#output\_cloud\_billing\_export\_manual\_setup) | GCP Cloud Billing BigQuery export manual setup information. |
 | <a name="output_cloud_billing_export_table_name"></a> [cloud\_billing\_export\_table\_name](#output\_cloud\_billing\_export\_table\_name) | The BigQuery table name containing the GCP Cloud Billing BigQuery export. |
 | <a name="output_kraken_sa_email"></a> [kraken\_sa\_email](#output\_kraken\_sa\_email) | Kraken service account email. |
 | <a name="output_kraken_sa_key"></a> [kraken\_sa\_key](#output\_kraken\_sa\_key) | Kraken service account key. |
diff --git a/main.tf b/main.tf
@@ -1,11 +1,13 @@
 module "kraken_sa" {
   source = "./modules/meshcloud-kraken-service-account/"
 
-  sa_name                         = var.kraken_sa_name
-  org_id                          = var.org_id
-  meshstack_root_project_id       = var.project_id
+  sa_name                   = var.kraken_sa_name
+  org_id                    = var.org_id
+  meshstack_root_project_id = var.project_id
+  landing_zone_folder_ids   = var.landing_zone_folder_ids
+
   cloud_billing_export_project_id = var.cloud_billing_export_project_id
-  landing_zone_folder_ids         = var.landing_zone_folder_ids
+  cloud_billing_export_dataset_id = var.cloud_billing_export_dataset_id
 }
 
 module "replicator_sa" {
@@ -25,9 +27,8 @@ module "carbon_export" {
   source = "./modules/meshcloud-carbon-export/"
   count  = var.carbon_export_module_enabled ? 1 : 0
 
-  carbon_data_export_project_id = var.cloud_billing_export_project_id # using the same project as for billing
-  carbon_data_export_dataset_id = var.carbon_footprint_dataset_id
-  carbon_dataset_region         = var.carbon_footprint_dataset_location
+  kraken_sa_email = module.kraken_sa.sa_email
 
-  billing_account_id = var.billing_account_id
+  cloud_carbon_export_project_id = var.cloud_carbon_export_project_id # using the same project as for billing
+  cloud_carbon_export_dataset_id = var.cloud_carbon_export_dataset_id
 }
diff --git a/modules/meshcloud-carbon-export/README.md b/modules/meshcloud-carbon-export/README.md
@@ -19,23 +19,20 @@ No modules.
 
 | Name | Type |
 |------|------|
-| [google_bigquery_data_transfer_config.carbon_footprint_transfer_config](https://registry.terraform.io/providers/hashicorp/google/4.11.0/docs/resources/bigquery_data_transfer_config) | resource |
-| [google_bigquery_dataset.carbon_data_export_dataset](https://registry.terraform.io/providers/hashicorp/google/4.11.0/docs/resources/bigquery_dataset) | resource |
-| [google_project_service.bigquery_api](https://registry.terraform.io/providers/hashicorp/google/4.11.0/docs/resources/project_service) | resource |
-| [google_project_service.bigquerydatatransfer_api](https://registry.terraform.io/providers/hashicorp/google/4.11.0/docs/resources/project_service) | resource |
+| [google_bigquery_dataset_iam_member.read_carbon_export](https://registry.terraform.io/providers/hashicorp/google/4.11.0/docs/resources/bigquery_dataset_iam_member) | resource |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_billing_account_id"></a> [billing\_account\_id](#input\_billing\_account\_id) | The GCP Billing Account in your organization. | `string` | n/a | yes |
-| <a name="input_carbon_data_export_dataset_id"></a> [carbon\_data\_export\_dataset\_id](#input\_carbon\_data\_export\_dataset\_id) | GCP BigQuery dataset containing the Carbon Footprint BigQuery export | `string` | n/a | yes |
-| <a name="input_carbon_data_export_project_id"></a> [carbon\_data\_export\_project\_id](#input\_carbon\_data\_export\_project\_id) | GCP Project where the BiqQuery table resides that holds the Carbon Footprint export to BigQuery. See https://cloud.google.com/billing/docs/how-to/export-data-bigquery | `string` | n/a | yes |
-| <a name="input_carbon_dataset_region"></a> [carbon\_dataset\_region](#input\_carbon\_dataset\_region) | The location of the BigQuery dataset for carbon data exports. | `string` | n/a | yes |
+| <a name="input_cloud_carbon_export_dataset_id"></a> [cloud\_carbon\_export\_dataset\_id](#input\_cloud\_carbon\_export\_dataset\_id) | GCP BigQuery dataset containing the Carbon Footprint BigQuery export | `string` | n/a | yes |
+| <a name="input_cloud_carbon_export_project_id"></a> [cloud\_carbon\_export\_project\_id](#input\_cloud\_carbon\_export\_project\_id) | GCP Project where the BiqQuery table resides that holds the Carbon Footprint export to BigQuery. See https://cloud.google.com/billing/docs/how-to/export-data-bigquery | `string` | n/a | yes |
+| <a name="input_kraken_sa_email"></a> [kraken\_sa\_email](#input\_kraken\_sa\_email) | Kraken Service account email address. | `string` | n/a | yes |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
+| <a name="output_carbon_footprint_export_manual_setup"></a> [carbon\_footprint\_export\_manual\_setup](#output\_carbon\_footprint\_export\_manual\_setup) | GCP Cloud Carbon Footprint BigQuery export manual setup information. |
 | <a name="output_carbon_footprint_export_table_name"></a> [carbon\_footprint\_export\_table\_name](#output\_carbon\_footprint\_export\_table\_name) | The BigQuery table name containing the GCP Carbon Footprint BigQuery export. |
 <!-- END_TF_DOCS -->
diff --git a/modules/meshcloud-carbon-export/module.tf b/modules/meshcloud-carbon-export/module.tf
@@ -1,31 +1,10 @@
-resource "google_project_service" "bigquery_api" {
-  project            = var.carbon_data_export_project_id
-  service            = "bigquery.googleapis.com"
-  disable_on_destroy = false
-}
+# similar to the cost export, we expect that the dataset already exists and was setup externally
+# this is a bit of a bummer as we can _almost_ automate it, see https://github.com/meshcloud/terraform-gcp-meshplatform/issues/10
 
-resource "google_project_service" "bigquerydatatransfer_api" {
-  project            = var.carbon_data_export_project_id
-  service            = "bigquerydatatransfer.googleapis.com"
-  disable_on_destroy = false
-}
+resource "google_bigquery_dataset_iam_member" "read_carbon_export" {
+  project    = var.cloud_carbon_export_project_id
+  dataset_id = var.cloud_carbon_export_dataset_id
 
-resource "google_bigquery_dataset" "carbon_data_export_dataset" {
-  dataset_id    = var.carbon_data_export_dataset_id
-  friendly_name = "carbon_data_export_tf"
-  description   = "This dataset holds the carbon footprint data."
-  location      = var.carbon_dataset_region
-  project       = var.carbon_data_export_project_id
-}
-
-resource "google_bigquery_data_transfer_config" "carbon_footprint_transfer_config" {
-  display_name           = "carbon-footprint-export-tf"
-  location               = var.carbon_dataset_region
-  data_source_id         = "61cede5a-0000-2440-ad42-883d24f8f7b8"
-  schedule               = "every day 00:00"
-  destination_dataset_id = google_bigquery_dataset.carbon_data_export_dataset.dataset_id
-  project                = var.carbon_data_export_project_id
-  params = {
-    billing_accounts = var.billing_account_id
-  }
+  member = "serviceAccount:${var.kraken_sa_email}"
+  role   = "roles/bigquery.dataViewer"
 }
diff --git a/modules/meshcloud-carbon-export/outputs.tf b/modules/meshcloud-carbon-export/outputs.tf
@@ -3,5 +3,10 @@ output "carbon_footprint_export_table_name" {
 
   # note carbon_footprint is the default table name that google's data transfer config creates
   # we can't control this via terraform right now
-  value = "${var.carbon_data_export_project_id}.${var.carbon_data_export_dataset_id}.carbon_footprint"
-}
+  value = "${var.cloud_carbon_export_project_id}.${var.cloud_carbon_export_dataset_id}.carbon_footprint"
+}
+
+output "carbon_footprint_export_manual_setup" {
+  description = "GCP Cloud Carbon Footprint BigQuery export manual setup information."
+  value       = "Attention. You need to manually configure the carbon footprint export transfer config before exceuting this module. See https://docs.meshcloud.io/docs/meshstack.how-to.integrate-meshplatform-gcp-manually.html#optional-enable-gcp-cloud-carbon-footprint-export for instructions."
+}
diff --git a/modules/meshcloud-carbon-export/variables.tf b/modules/meshcloud-carbon-export/variables.tf
@@ -1,19 +1,15 @@
-variable "carbon_dataset_region" {
+variable "cloud_carbon_export_project_id" {
   type        = string
-  description = "The location of the BigQuery dataset for carbon data exports."
+  description = "GCP Project where the BiqQuery table resides that holds the Carbon Footprint export to BigQuery. See https://cloud.google.com/billing/docs/how-to/export-data-bigquery"
 }
 
-variable "billing_account_id" {
+variable "cloud_carbon_export_dataset_id" {
   type        = string
-  description = "The GCP Billing Account in your organization."
+  description = "GCP BigQuery dataset containing the Carbon Footprint BigQuery export"
 }
 
-variable "carbon_data_export_project_id" {
+variable "kraken_sa_email" {
   type        = string
-  description = "GCP Project where the BiqQuery table resides that holds the Carbon Footprint export to BigQuery. See https://cloud.google.com/billing/docs/how-to/export-data-bigquery"
-}
+  description = "Kraken Service account email address."
 
-variable "carbon_data_export_dataset_id" {
-  type        = string
-  description = "GCP BigQuery dataset containing the Carbon Footprint BigQuery export"
 }
diff --git a/modules/meshcloud-kraken-service-account/README.md b/modules/meshcloud-kraken-service-account/README.md
@@ -19,10 +19,10 @@ No modules.
 
 | Name | Type |
 |------|------|
-| [google_folder_iam_member.replicator_service](https://registry.terraform.io/providers/hashicorp/google/4.11.0/docs/resources/folder_iam_member) | resource |
+| [google_bigquery_dataset_iam_member.read_billing_export](https://registry.terraform.io/providers/hashicorp/google/4.11.0/docs/resources/bigquery_dataset_iam_member) | resource |
+| [google_folder_iam_member.kraken_service](https://registry.terraform.io/providers/hashicorp/google/4.11.0/docs/resources/folder_iam_member) | resource |
 | [google_organization_iam_custom_role.meshcloud_kraken_sa](https://registry.terraform.io/providers/hashicorp/google/4.11.0/docs/resources/organization_iam_custom_role) | resource |
 | [google_project_iam_member.bigquery_jobuser](https://registry.terraform.io/providers/hashicorp/google/4.11.0/docs/resources/project_iam_member) | resource |
-| [google_project_iam_member.biquery_dataViewer](https://registry.terraform.io/providers/hashicorp/google/4.11.0/docs/resources/project_iam_member) | resource |
 | [google_project_service.bigquery_api](https://registry.terraform.io/providers/hashicorp/google/4.11.0/docs/resources/project_service) | resource |
 | [google_service_account.meshcloud_kraken_sa](https://registry.terraform.io/providers/hashicorp/google/4.11.0/docs/resources/service_account) | resource |
 | [google_service_account_key.sa_key](https://registry.terraform.io/providers/hashicorp/google/4.11.0/docs/resources/service_account_key) | resource |
@@ -31,6 +31,7 @@ No modules.
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| <a name="input_cloud_billing_export_dataset_id"></a> [cloud\_billing\_export\_dataset\_id](#input\_cloud\_billing\_export\_dataset\_id) | GCP BigQuery dataset containing the Cloud Billing BigQuery export. | `string` | n/a | yes |
 | <a name="input_cloud_billing_export_project_id"></a> [cloud\_billing\_export\_project\_id](#input\_cloud\_billing\_export\_project\_id) | GCP Project where the BigQuery table resides that holds the Cloud Billing export to BigQuery. | `string` | n/a | yes |
 | <a name="input_landing_zone_folder_ids"></a> [landing\_zone\_folder\_ids](#input\_landing\_zone\_folder\_ids) | GCP Folders that make up the Landing Zone. The service account will only receive permissions on these folders. | `set(string)` | n/a | yes |
 | <a name="input_meshstack_root_project_id"></a> [meshstack\_root\_project\_id](#input\_meshstack\_root\_project\_id) | GCP Project ID where to create the service account. This is typically a 'meshstack-root' project. | `string` | n/a | yes |
@@ -41,6 +42,7 @@ No modules.
 
 | Name | Description |
 |------|-------------|
+| <a name="output_billing_export_manual_setup"></a> [billing\_export\_manual\_setup](#output\_billing\_export\_manual\_setup) | GCP Cloud Billing BigQuery export manual setup information. |
 | <a name="output_sa_email"></a> [sa\_email](#output\_sa\_email) | Service account email. |
 | <a name="output_sa_key"></a> [sa\_key](#output\_sa\_key) | Service account key (base64 encoded credential.json). |
 <!-- END_TF_DOCS -->
diff --git a/modules/meshcloud-kraken-service-account/module.tf b/modules/meshcloud-kraken-service-account/module.tf
@@ -18,11 +18,12 @@ resource "google_project_iam_member" "bigquery_jobuser" {
   member = "serviceAccount:${google_service_account.meshcloud_kraken_sa.email}"
 }
 
-resource "google_project_iam_member" "biquery_dataViewer" {
-  project = var.cloud_billing_export_project_id
-  role    = "roles/bigquery.dataViewer"
+resource "google_bigquery_dataset_iam_member" "read_billing_export" {
+  project    = var.cloud_billing_export_project_id
+  dataset_id = var.cloud_billing_export_dataset_id
 
   member = "serviceAccount:${google_service_account.meshcloud_kraken_sa.email}"
+  role   = "roles/bigquery.dataViewer"
 }
 
 resource "google_organization_iam_custom_role" "meshcloud_kraken_sa" {
@@ -39,9 +40,9 @@ resource "google_organization_iam_custom_role" "meshcloud_kraken_sa" {
 }
 
 # We apply a hardened security configuration, i.e. we assign permissions only on LZ folders instead of the organization
-# root
+# root - this allows kraken to read projects
 
-resource "google_folder_iam_member" "replicator_service" {
+resource "google_folder_iam_member" "kraken_service" {
   for_each = var.landing_zone_folder_ids
 
   folder = each.value
diff --git a/modules/meshcloud-kraken-service-account/outputs.tf b/modules/meshcloud-kraken-service-account/outputs.tf
@@ -8,3 +8,8 @@ output "sa_email" {
   value       = google_service_account.meshcloud_kraken_sa.email
   description = "Service account email."
 }
+
+output "billing_export_manual_setup" {
+  description = "GCP Cloud Billing BigQuery export manual setup information."
+  value       = "Attention. You need to manually configure the billing account big query export before exceuting this module. See https://docs.meshcloud.io/docs/meshstack.how-to.integrate-meshplatform-gcp-manually.html#set-up-gcp-billing-data-export for instructions."
+}
diff --git a/modules/meshcloud-kraken-service-account/variables.tf b/modules/meshcloud-kraken-service-account/variables.tf
@@ -18,6 +18,11 @@ variable "cloud_billing_export_project_id" {
   description = "GCP Project where the BigQuery table resides that holds the Cloud Billing export to BigQuery."
 }
 
+variable "cloud_billing_export_dataset_id" {
+  type        = string
+  description = "GCP BigQuery dataset containing the Cloud Billing BigQuery export."
+}
+
 variable "landing_zone_folder_ids" {
   type        = set(string)
   description = "GCP Folders that make up the Landing Zone. The service account will only receive permissions on these folders."
diff --git a/outputs.tf b/outputs.tf
diff --git a/variables.tf b/variables.tf

Original file line number	Diff line number	Diff line change
`@@ -8,3 +8,8 @@ output "sa_email" {`
`8`	`8`	`value = google_service_account.meshcloud_kraken_sa.email`
`9`	`9`	`description = "Service account email."`
`10`	`10`	`}`
	`11`	`+`
	`12`	`+output "billing_export_manual_setup" {`
	`13`	`+ description = "GCP Cloud Billing BigQuery export manual setup information."`
	`14`	`+ value = "Attention. You need to manually configure the billing account big query export before exceuting this module. See https://docs.meshcloud.io/docs/meshstack.how-to.integrate-meshplatform-gcp-manually.html#set-up-gcp-billing-data-export for instructions."`
	`15`	`+}`