Add dags and plugins to staging composer

.github/workflows/composer-apply-files.yml

@@ -0,0 +1,28 @@
+name: Apply Airflow DAGs
+
+on:
+  pull_request:
+    paths:
+      - 'airflow/**'
+
+jobs:
+  staging:
+    name: Staging
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - uses: 'google-github-actions/auth@v2'
+        with:
+          create_credentials_file: 'true'
+          project_id: cal-itp-data-infra-staging
+          workload_identity_provider: 'projects/473674835135/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider'
+          service_account: 'github-actions-terraform@cal-itp-data-infra-staging.iam.gserviceaccount.com'
+
+      - name: Terraform Apply
+        uses: dflook/terraform-apply@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          path: iac/cal-itp-data-infra-staging/airflow/us

.github/workflows/composer-plan-files.yml

@@ -0,0 +1,29 @@
+name: Plan Airflow DAGs
+
+on:
+  pull_request:
+    paths:
+      - 'airflow/**'
+
+jobs:
+  staging:
+    name: Staging
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - uses: 'google-github-actions/auth@v2'
+        with:
+          create_credentials_file: 'true'
+          project_id: cal-itp-data-infra-staging
+          workload_identity_provider: 'projects/473674835135/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider'
+          service_account: 'github-actions-terraform@cal-itp-data-infra-staging.iam.gserviceaccount.com'
+
+      - name: Terraform Plan
+        uses: dflook/terraform-plan@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          add_github_comment: changes-only
+          path: iac/cal-itp-data-infra-staging/airflow/us

.github/workflows/terraform-apply.yml

@@ -43,9 +43,8 @@ jobs:
 
   staging:
     name: Staging
-
-    needs: targets
-
+    needs: [targets]
+    if: ${{ needs.targets.outputs.staging != '[]' && needs.targets.outputs.staging != '' }}
     runs-on: ubuntu-latest
 
     permissions:
@@ -79,9 +78,8 @@ jobs:
 
   production:
     name: Production
-
-    needs: targets
-
+    needs: [targets]
+    if: ${{ needs.targets.outputs.production != '[]' && needs.targets.outputs.production != '' }}
     runs-on: ubuntu-latest
 
     permissions:

.github/workflows/terraform-plan.yml

@@ -8,7 +8,6 @@ on:
 jobs:
   targets:
     name: Find targets
-
     runs-on: ubuntu-latest
 
     outputs:
@@ -37,9 +36,8 @@ jobs:
 
   staging:
     name: Staging
-
-    needs: targets
-
+    needs: [targets]
+    if: ${{ needs.targets.outputs.staging != '[]' && needs.targets.outputs.staging != '' }}
     runs-on: ubuntu-latest
 
     permissions:
@@ -85,9 +83,8 @@ jobs:
 
   production:
     name: Production
-
-    needs: targets
-
+    needs: [targets]
+    if: ${{ needs.targets.outputs.production != '[]' && needs.targets.outputs.production != '' }}
     runs-on: ubuntu-latest
 
     permissions:

iac/cal-itp-data-infra-staging/airflow/us/provider.tf

@@ -0,0 +1,16 @@
+provider "google" {
+  project = "cal-itp-data-infra-staging"
+}
+
+terraform {
+  required_providers {
+    google = {
+      version = "~> 4.59.0"
+    }
+  }
+
+  backend "gcs" {
+    bucket = "calitp-staging-gcp-components-tfstate"
+    prefix = "cal-itp-data-infra-staging/airflow"
+  }
+}

iac/cal-itp-data-infra-staging/airflow/us/storage_bucket_object.tf

@@ -0,0 +1,15 @@
+locals {
+  files = setunion(
+    fileset("../../../../airflow", "dags/**/*.py"),
+    fileset("../../../../airflow", "dags/**/*.yml"),
+    fileset("../../../../airflow", "dags/**/*.md"),
+    fileset("../../../../airflow", "plugins/**/*.py")
+  )
+}
+
+resource "google_storage_bucket_object" "calitp-staging-composer" {
+  for_each = local.files
+  name     = each.value
+  source   = each.value
+  bucket   = data.terraform_remote_state.gcs.outputs.google_storage_bucket_calitp-staging-composer_id
+}

iac/cal-itp-data-infra-staging/airflow/us/variables.tf

@@ -0,0 +1,17 @@
+data "terraform_remote_state" "networks" {
+  backend = "gcs"
+
+  config = {
+    bucket = "calitp-staging-gcp-components-tfstate"
+    prefix = "cal-itp-data-infra-staging/networks"
+  }
+}
+
+data "terraform_remote_state" "gcs" {
+  backend = "gcs"
+
+  config = {
+    bucket = "calitp-staging-gcp-components-tfstate"
+    prefix = "cal-itp-data-infra-staging/gcs"
+  }
+}

iac/cal-itp-data-infra-staging/gcs/us/outputs.tf

@@ -126,6 +126,10 @@ output "google_storage_bucket_calitp-staging-composer_name" {
   value = google_storage_bucket.calitp-staging-composer.name
 }
 
+output "google_storage_bucket_calitp-staging-composer_id" {
+  value = google_storage_bucket.calitp-staging-composer.id
+}
+
 output "google_storage_default_object_acl_tfer--calitp-staging-data-analyses-portfolio-draft_id" {
   value = google_storage_default_object_acl.tfer--calitp-staging-data-analyses-portfolio-draft.id
 }

iac/cal-itp-data-infra-staging/gcs/us/provider.tf

@@ -5,7 +5,7 @@ provider "google" {
 terraform {
   required_providers {
     google = {
-      version = "~> 4.59.0"
+      version = "~> 6.29.0"
     }
   }
 

iac/cal-itp-data-infra-staging/gcs/us/storage_bucket.tf

@@ -143,4 +143,8 @@ resource "google_storage_bucket" "calitp-staging-composer" {
   requester_pays              = "false"
   storage_class               = "STANDARD"
   uniform_bucket_level_access = "true"
+
+  lifecycle {
+    ignore_changes = [labels]
+  }
 }

iac/cal-itp-data-infra-staging/gcs/us/storage_bucket_iam_member.tf

@@ -42,6 +42,6 @@ resource "google_storage_bucket_iam_member" "calitp-staging-dbt-docs" {
 
 resource "google_storage_bucket_iam_member" "calitp-staging-composer" {
   bucket = google_storage_bucket.calitp-staging-composer.name
-  member = "projectEditor:cal-itp-data-infra"
+  member = "projectEditor:cal-itp-data-infra-staging"
   role   = "roles/storage.legacyBucketOwner"
 }

iac/cal-itp-data-infra/gcs/us/provider.tf

@@ -5,7 +5,7 @@ provider "google" {
 terraform {
   required_providers {
     google = {
-      version = "~> 4.59.0"
+      version = "~> 6.29.0"
     }
   }