v1.2.6

Author: srastatter

CHANGELOG.md

@@ -1,6 +1,25 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## [1.2.6] - 2023-10-05
+
+### Added
+
+### Changed
+
+- Updated default to not disable api services
+- Changed required_permissions and recommend_roles from set to list
+- Updated required_permissions and recommend_roles lists to be dynamic based on user tooling selection
+- Changed google_project_iam_member block to use a local list for setting iam roles
+- Moved csr creation to only occur if use_ci=True
+- Updated github_actions config to use source_repo_branch parameter into of default `automlops` branch
+
+### Fixed
+
+- Fixed run local issue regarding dockerfile pathing
+- Fixed cloudbuild trigger terraform bug with ignored_files
+
+
 ## [1.2.5] - 2023-10-05
 
 ### Added
@@ -11,6 +30,7 @@ All notable changes to this project will be documented in this file.
 
 - Fixed bug with generating .github/workflows directories
 
+
 ## [1.2.4] - 2023-10-03
 
 ### Added
@@ -23,6 +43,7 @@ All notable changes to this project will be documented in this file.
 
 - Pinned the python docker container step for the cloudbuild yaml to python:3.10, this address some dependency issues with the latest python docker image
 
+
 ## [1.2.3] - 2023-09-29
 
 ### Added

examples/inferencing/00_batch_prediction_example.ipynb

@@ -415,8 +415,8 @@
     "        \"\"\"Generates BQ Query to read data.\n",
     "\n",
     "        Args:\n",
-    "        bq_input_table: The full name of the bq input table to be read into\n",
-    "        the dataframe (e.g. <project>.<dataset>.<table>)\n",
+    "            bq_input_table: The full name of the bq input table to be read into\n",
+    "                the dataframe (e.g. <project>.<dataset>.<table>)\n",
     "        Returns: A BQ query string.\n",
     "        \"\"\"\n",
     "        return f'''\n",
@@ -427,10 +427,10 @@
     "    def load_bq_data(query: str, client: bigquery.Client) -> pd.DataFrame:\n",
     "        \"\"\"Loads data from bq into a Pandas Dataframe for EDA.\n",
     "        Args:\n",
-    "        query: BQ Query to generate data.\n",
-    "        client: BQ Client used to execute query.\n",
+    "            query: BQ Query to generate data.\n",
+    "            client: BQ Client used to execute query.\n",
     "        Returns:\n",
-    "        pd.DataFrame: A dataframe with the requested data.\n",
+    "            pd.DataFrame: A dataframe with the requested data.\n",
     "        \"\"\"\n",
     "        df = client.query(query).to_dataframe()\n",
     "        return df\n",

examples/training/00_introduction_training_example.ipynb

@@ -327,7 +327,8 @@ def generate(
         logging.info(f'Writing README.md to {BASE_DIR}README.md')
         logging.info(f'Writing kubeflow pipelines code to {BASE_DIR}pipelines, {BASE_DIR}components')
         logging.info(f'Writing scripts to {BASE_DIR}scripts')
-        logging.info(f'Writing submission service code to {BASE_DIR}services')
+        if use_ci:
+            logging.info(f'Writing submission service code to {BASE_DIR}services')
         KfpBuilder.build(KfpConfig(
             base_image=base_image,
             custom_training_job_specs=derived_custom_training_job_specs,
@@ -414,6 +415,7 @@ def generate(
                 project_id=project_id,
                 project_number=project_number,
                 pubsub_topic_name=derived_pubsub_topic_name,
+                source_repo_branch=source_repo_branch,
                 use_ci=use_ci,
                 workload_identity_pool=workload_identity_pool,
                 workload_identity_provider=workload_identity_provider,

google_cloud_automlops/AutoMLOps.py

@@ -23,6 +23,6 @@
 series of directories to support the creation of Vertex Pipelines.
 """
 # pylint: disable=invalid-name
-__version__ = '1.2.5'
+__version__ = '1.2.6'
 __author__ = 'Sean Rastatter'
 __credits__ = 'Google'

google_cloud_automlops/__init__.py

@@ -26,6 +26,7 @@
 
 from google_cloud_automlops.utils.utils import write_file
 from google_cloud_automlops.utils.constants import (
+    BASE_DIR,
     CLOUDBUILD_TEMPLATES_PATH,
     GENERATED_CLOUDBUILD_FILE,
     COMPONENT_BASE_RELATIVE_PATH,
@@ -76,13 +77,14 @@ def create_cloudbuild_jinja(
     Returns:
         str: Contents of cloudbuild.yaml.
     """
+    component_base_relative_path = COMPONENT_BASE_RELATIVE_PATH if use_ci else f'{BASE_DIR}{COMPONENT_BASE_RELATIVE_PATH}'
     template_file = import_files(CLOUDBUILD_TEMPLATES_PATH) / 'cloudbuild.yaml.j2'
     with template_file.open('r', encoding='utf-8') as f:
         template = Template(f.read())
         return template.render(
             artifact_repo_location=artifact_repo_location,
             artifact_repo_name=artifact_repo_name,
-            component_base_relative_path=COMPONENT_BASE_RELATIVE_PATH,
+            component_base_relative_path=component_base_relative_path,
             generated_license=GENERATED_LICENSE,
             generated_parameter_values_path=GENERATED_PARAMETER_VALUES_PATH,
             naming_prefix=naming_prefix,

google_cloud_automlops/deployments/cloudbuild/builder.py

@@ -48,6 +48,7 @@ class GitHubActionsConfig(BaseModel):
         project_id: The project ID.
         project_number: The project number.
         pubsub_topic_name: The name of the pubsub topic to publish to.
+        source_repo_branch: The branch to use in the source repository.
         use_ci: Flag that determines whether to use Cloud CI/CD.
         workload_identity_pool: Pool for workload identity federation. 
         workload_identity_provider: Provider for workload identity federation.
@@ -59,6 +60,7 @@ class GitHubActionsConfig(BaseModel):
     project_id: str
     project_number: str #TODO: Check if there's any other way to pass this, could use a util with the GCP client library. See https://github.com/GoogleCloudPlatform/java-docs-samples/blob/main/content-warehouse/src/main/java/contentwarehouse/v1/CreateDocument.java#L125-L135
     pubsub_topic_name: str
+    source_repo_branch: str
     use_ci: bool
     workload_identity_pool: str
     workload_identity_provider: str

google_cloud_automlops/deployments/configs.py

@@ -26,7 +26,6 @@
 
 from google_cloud_automlops.utils.utils import write_file
 from google_cloud_automlops.utils.constants import (
-    DEFAULT_SOURCE_REPO_BRANCH,
     GENERATED_GITHUB_ACTIONS_FILE,
     COMPONENT_BASE_RELATIVE_PATH,
     GENERATED_LICENSE,
@@ -46,6 +45,7 @@ def build(config: GitHubActionsConfig):
         config.project_id: The project ID.
         config.project_number: The project number.
         config.pubsub_topic_name: The name of the pubsub topic to publish to.
+        config.source_repo_branch: The branch to use in the source repository.
         config.use_ci: Flag that determines whether to use Cloud CI/CD.
         config.workload_identity_pool: Pool for workload identity federation. 
         config.workload_identity_provider: Provider for workload identity federation.
@@ -59,6 +59,7 @@ def build(config: GitHubActionsConfig):
         config.project_id,
         config.project_number,
         config.pubsub_topic_name,
+        config.source_repo_branch,
         config.use_ci,
         config.workload_identity_pool,
         config.workload_identity_provider,
@@ -71,6 +72,7 @@ def create_github_actions_jinja(
         project_id: str,
         project_number: str,
         pubsub_topic_name: str,
+        source_repo_branch: str,
         use_ci: bool,
         workload_identity_pool: str,
         workload_identity_provider: str,
@@ -85,6 +87,7 @@ def create_github_actions_jinja(
         project_id: The project ID.
         project_number: The project number.
         pubsub_topic_name: The name of the pubsub topic to publish to.
+        source_repo_branch: The branch to use in the source repository.
         use_ci: Flag that determines whether to use Cloud CI/CD.
         workload_identity_pool: Pool for workload identity federation. 
         workload_identity_provider: Provider for workload identity federation.
@@ -106,7 +109,7 @@ def create_github_actions_jinja(
             project_id=project_id,
             project_number=project_number,
             pubsub_topic_name=pubsub_topic_name,
-            source_repo_branch=DEFAULT_SOURCE_REPO_BRANCH,
+            source_repo_branch=source_repo_branch,
             use_ci=use_ci,
             workload_identity_pool=workload_identity_pool,
             workload_identity_provider=workload_identity_provider,

google_cloud_automlops/deployments/github_actions/builder.py

@@ -73,7 +73,7 @@ def build(
         config.vpc_connector: The name of the vpc connector to use.
     """
     defaults = read_yaml_file(GENERATED_DEFAULTS_FILE)
-    required_apis = list(get_required_apis(defaults))
+    required_apis = get_required_apis(defaults)
     # create provision_resources.sh
     write_and_chmod(GENERATED_RESOURCES_SH_FILE, provision_resources_script_jinja(
         artifact_repo_location=config.artifact_repo_location,

google_cloud_automlops/provisioning/gcloud/builder.py

@@ -74,6 +74,7 @@ gcloud projects add-iam-policy-binding $PROJECT_ID \
     --member="serviceAccount:$PIPELINE_JOB_RUNNER_SERVICE_ACCOUNT_LONG" \
     --role="{{role}}" \
     --no-user-output-enabled{% endfor %}
+{% if use_ci %}
 {% if source_repo_type == 'cloud-source-repositories' %}
 echo -e "$GREEN Setting up Cloud Source Repository in project $PROJECT_ID $NC"
 if ! (gcloud source repos list --project="$PROJECT_ID" | grep -E "(^|[[:blank:]])$SOURCE_REPO_NAME($|[[:blank:]])"); then
@@ -85,8 +86,8 @@ else
 
   echo "Cloud Source Repository: ${SOURCE_REPO_NAME} already exists in project $PROJECT_ID"
 
-fi{% endif %}
-{% if use_ci %}
+fi
+{% endif %}
 # Create Pub/Sub Topic
 echo -e "$GREEN Setting up Queueing Service in project $PROJECT_ID $NC"
 if ! (gcloud pubsub topics list | grep -E "(^|[[:blank:]])projects/${PROJECT_ID}/topics/${PUBSUB_TOPIC_NAME}($|[[:blank:]])"); then

google_cloud_automlops/provisioning/gcloud/templates/provision_resources.sh.j2

@@ -80,7 +80,7 @@ def build(
         config.vpc_connector: The name of the vpc connector to use.
     """
     defaults = read_yaml_file(GENERATED_DEFAULTS_FILE)
-    required_apis = list(get_required_apis(defaults))
+    required_apis = get_required_apis(defaults)
     # create environment/data.tf
     write_file(f'{BASE_DIR}provision/environment/data.tf', create_environment_data_tf_jinja(
         required_apis=required_apis,
@@ -184,6 +184,7 @@ def create_environment_data_tf_jinja(
         return template.render(
             generated_license=GENERATED_LICENSE,
             required_apis=required_apis,
+            required_iam_roles=IAM_ROLES_RUNNER_SA,
             use_ci=use_ci)
 
 
@@ -196,9 +197,7 @@ def create_environment_iam_tf_jinja() -> str:
     template_file = import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'iam.tf.j2'
     with template_file.open('r', encoding='utf-8') as f:
         template = Template(f.read())
-        return template.render(
-            generated_license=GENERATED_LICENSE,
-            required_iam_roles=IAM_ROLES_RUNNER_SA)
+        return template.render(generated_license=GENERATED_LICENSE)
 
 
 def create_environment_main_tf_jinja(

google_cloud_automlops/provisioning/terraform/builder.py

@@ -25,5 +25,9 @@ locals {
     enable_apis = [{% for api in required_apis %}
       "{{api}}",{% endfor %}
     ]
+
+    pipeline_runner_service_account_iam_list = [{% for role in required_iam_roles %}
+      "{{role}}",{% endfor %}
+    ]
   }
 }

google_cloud_automlops/provisioning/terraform/templates/environment/data.tf.j2

@@ -12,12 +12,11 @@ resource "google_service_account" "pipeline_job_runner_service_account" {
   depends_on = [module.google_project_service,time_sleep.wait_60_seconds]
 }
 
-{% for role in required_iam_roles %}resource "google_project_iam_member" "pipeline_job_runner_service_account_{{role.replace('/', '_').replace('.', '_')}}" {
+resource "google_project_iam_member" "pipeline_job_runner_service_account_iam" {
+  for_each = toset(local.org_project.pipeline_runner_service_account_iam_list)
   project = var.project_id
-  role    = "{{role}}"
+  role = each.key
   member  = "serviceAccount:${google_service_account.pipeline_job_runner_service_account.email}"
 
   depends_on = [google_service_account.pipeline_job_runner_service_account]
 }
-
-{% endfor %}

google_cloud_automlops/provisioning/terraform/templates/environment/iam.tf.j2

@@ -1,10 +1,11 @@
 {{generated_license}}
 # Enable Google Cloud APIs
 module "google_project_service" {
-  source                  = "terraform-google-modules/project-factory/google//modules/project_services"
-  version                 = "14.1.0"
-  project_id              = var.project_id
-  activate_apis           = local.org_project.enable_apis
+  source                      = "terraform-google-modules/project-factory/google//modules/project_services"
+  version                     = "14.1.0"
+  project_id                  = var.project_id
+  activate_apis               = local.org_project.enable_apis
+  disable_services_on_destroy = false
 }
 resource "time_sleep" "wait_60_seconds" {
   depends_on = [module.google_project_service]
@@ -34,6 +35,7 @@ resource "google_storage_bucket" "storage_bucket" {
   }
   depends_on              = [module.google_project_service,time_sleep.wait_60_seconds]
 }
+{% if use_ci %}
 {% if source_repo_type == 'cloud-source-repositories' %}
 # Create cloud source repository
 resource "google_sourcerepo_repository" "source_repo" {
@@ -42,7 +44,7 @@ resource "google_sourcerepo_repository" "source_repo" {
 
   depends_on              = [module.google_project_service,time_sleep.wait_60_seconds]
 }{% endif %}
-{% if use_ci %}{% if pipeline_job_submission_service_type == 'cloud-run' %}
+{% if pipeline_job_submission_service_type == 'cloud-run' %}
 # Build and Push Submission Service image
 resource "null_resource" "build_and_push_submission_service" {
   provisioner "local-exec" {
@@ -132,7 +134,7 @@ resource "google_cloudbuild_trigger" "cloudbuild_trigger" {
   project                 = var.project_id
   name                    = var.build_trigger_name
   location                = var.build_trigger_location
-  ignored-files           = [".gitignore"]
+  ignored_files           = [".gitignore"]
   trigger_template {
     branch_name           = var.source_repo_branch
     project_id            = var.project_id

google_cloud_automlops/provisioning/terraform/templates/environment/main.tf.j2

@@ -127,12 +127,12 @@
 GITOPS_TEMPLATES_PATH = 'google_cloud_automlops.deployments.gitops.templates'
 
 # Required IAM Roles for pipeline runner service account
-IAM_ROLES_RUNNER_SA = set([
+IAM_ROLES_RUNNER_SA = [
     'roles/aiplatform.user',
     'roles/artifactregistry.reader',
+    'roles/cloudfunctions.admin',
     'roles/bigquery.user',
     'roles/bigquery.dataEditor',
     'roles/iam.serviceAccountUser',
-    'roles/storage.admin',
-    'roles/cloudfunctions.admin'
-])
+    'roles/storage.admin'
+]