Skip to content

Commit 7dc75c6

Browse files
srastattersrastatter
committed
v1.2.4
1 parent 1bb7dbc commit 7dc75c6

File tree

7 files changed

+91
-49
lines changed

7 files changed

+91
-49
lines changed

‎CHANGELOG.md

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,18 @@
11
# Change Log
22
All notable changes to this project will be documented in this file.
33

4+
## [1.2.4] - 2023-10-03
5+
6+
### Added
7+
8+
### Changed
9+
10+
- Only will create .github/ directory if using github-actions
11+
12+
### Fixed
13+
14+
- Pinned the python docker container step for the cloudbuild yaml to python:3.10, this address some dependency issues with the latest python docker image
15+
416
## [1.2.3] - 2023-09-29
517

618
### Added

‎examples/training/00_introduction_training_example.ipynb

Lines changed: 72 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -697,49 +697,49 @@
697697
"output_type": "stream",
698698
"text": [
699699
"WARNING: Provisioning requires these permissions:\n",
700+
"-pubsub.subscriptions.create\n",
701+
"-source.repos.list\n",
702+
"-resourcemanager.projects.setIamPolicy\n",
703+
"-cloudfunctions.functions.get\n",
704+
"-iam.serviceAccounts.list\n",
700705
"-cloudbuild.builds.create\n",
701-
"-pubsub.topics.list\n",
702-
"-serviceusage.services.enable\n",
703-
"-storage.buckets.get\n",
704706
"-iam.serviceAccounts.actAs\n",
705-
"-cloudscheduler.jobs.create\n",
706707
"-pubsub.subscriptions.list\n",
707-
"-pubsub.topics.create\n",
708-
"-cloudfunctions.functions.get\n",
709-
"-artifactregistry.repositories.create\n",
708+
"-source.repos.create\n",
709+
"-cloudscheduler.jobs.create\n",
710710
"-cloudscheduler.jobs.list\n",
711-
"-pubsub.subscriptions.create\n",
711+
"-serviceusage.services.enable\n",
712+
"-cloudbuild.builds.list\n",
713+
"-storage.buckets.get\n",
714+
"-artifactregistry.repositories.list\n",
715+
"-artifactregistry.repositories.create\n",
716+
"-serviceusage.services.use\n",
712717
"-iam.serviceAccounts.create\n",
713-
"-resourcemanager.projects.setIamPolicy\n",
714-
"-iam.serviceAccounts.list\n",
718+
"-pubsub.topics.create\n",
715719
"-cloudfunctions.functions.create\n",
716-
"-source.repos.list\n",
717-
"-source.repos.create\n",
718-
"-artifactregistry.repositories.list\n",
720+
"-pubsub.topics.list\n",
719721
"-storage.buckets.create\n",
720-
"-serviceusage.services.use\n",
721-
"-cloudbuild.builds.list\n",
722722
"\n",
723723
"You are currently using: srastatter@google.com. Please check your account permissions.\n",
724724
"The following are the recommended roles for provisioning:\n",
725-
"-roles/cloudbuild.builds.editor\n",
725+
"-roles/source.admin\n",
726726
"-roles/resourcemanager.projectIamAdmin\n",
727727
"-roles/pubsub.editor\n",
728-
"-roles/source.admin\n",
729-
"-roles/iam.serviceAccountUser\n",
730728
"-roles/serviceusage.serviceUsageAdmin\n",
729+
"-roles/cloudscheduler.admin\n",
731730
"-roles/cloudfunctions.admin\n",
732-
"-roles/storage.admin\n",
731+
"-roles/cloudbuild.builds.editor\n",
732+
"-roles/iam.serviceAccountUser\n",
733733
"-roles/artifactregistry.admin\n",
734-
"-roles/cloudscheduler.admin\n",
735734
"-roles/iam.serviceAccountAdmin\n",
735+
"-roles/storage.admin\n",
736736
"\n",
737737
"\u001b[0;32m Setting up API services in project automlops-sandbox \u001b[0m\n",
738-
"Operation \"operations/acat.p2-45373616427-64a80155-46ca-4ebb-abc4-66904d81e9ea\" finished successfully.\n",
738+
"Operation \"operations/acat.p2-45373616427-e2d3ac08-b15b-4cac-a8cc-c14b53524c4a\" finished successfully.\n",
739739
"\u001b[0;32m Setting up Artifact Registry in project automlops-sandbox \u001b[0m\n",
740740
"Listing items under project automlops-sandbox, location us-central1.\n",
741741
"\n",
742-
"dry-beans-dt-artifact-registry DOCKER STANDARD_REPOSITORY Artifact Registry dry-beans-dt-artifact-registry in us-central1. us-central1 Google-managed key 2023-09-05T11:25:48 2023-09-28T23:31:56 11640.147\n",
742+
"dry-beans-dt-artifact-registry DOCKER STANDARD_REPOSITORY Artifact Registry dry-beans-dt-artifact-registry in us-central1. us-central1 Google-managed key 2023-09-05T10:25:48 2023-10-03T13:57:46 14103.439\n",
743743
"Artifact Registry: dry-beans-dt-artifact-registry already exists in project automlops-sandbox\n",
744744
"\u001b[0;32m Setting up Storage Bucket in project automlops-sandbox \u001b[0m\n",
745745
"gs://automlops-sandbox-dry-beans-dt-bucket/\n",
@@ -757,11 +757,11 @@
757757
"\u001b[0;32m Deploying Cloud Functions: dry-beans-dt-job-submission-svc in project automlops-sandbox \u001b[0m\n",
758758
"Deploying function (may take a while - up to 2 minutes)...\n",
759759
"..\n",
760-
"For Cloud Build Logs, visit: https://console.cloud.google.com/cloud-build/builds;region=us-central1/2023b741-7039-4edc-8e70-61da23715920?project=45373616427\n",
761-
".............................done.\n",
760+
"For Cloud Build Logs, visit: https://console.cloud.google.com/cloud-build/builds;region=us-central1/21d26680-f4ca-4982-8ed7-5fa21b1c5d64?project=45373616427\n",
761+
"........done.\n",
762762
"availableMemoryMb: 512\n",
763-
"buildId: 2023b741-7039-4edc-8e70-61da23715920\n",
764-
"buildName: projects/45373616427/locations/us-central1/builds/2023b741-7039-4edc-8e70-61da23715920\n",
763+
"buildId: 21d26680-f4ca-4982-8ed7-5fa21b1c5d64\n",
764+
"buildName: projects/45373616427/locations/us-central1/builds/21d26680-f4ca-4982-8ed7-5fa21b1c5d64\n",
765765
"dockerRegistry: ARTIFACT_REGISTRY\n",
766766
"dockerRepository: projects/automlops-sandbox/locations/us-central1/repositories/dry-beans-dt-artifact-registry\n",
767767
"entryPoint: process_request\n",
@@ -777,11 +777,11 @@
777777
"name: projects/automlops-sandbox/locations/us-central1/functions/dry-beans-dt-job-submission-svc\n",
778778
"runtime: python39\n",
779779
"serviceAccountEmail: vertex-pipelines@automlops-sandbox.iam.gserviceaccount.com\n",
780-
"sourceUploadUrl: https://storage.googleapis.com/uploads-961973632599.us-central1.cloudfunctions.appspot.com/7e2fe33a-4a5b-4103-8a9c-e81855325ba4.zip\n",
780+
"sourceUploadUrl: https://storage.googleapis.com/uploads-961973632599.us-central1.cloudfunctions.appspot.com/2c19d400-96a6-44c3-bd8e-aeb2f608ca80.zip\n",
781781
"status: ACTIVE\n",
782782
"timeout: 540s\n",
783-
"updateTime: '2023-09-29T03:41:11.731Z'\n",
784-
"versionId: '2'\n",
783+
"updateTime: '2023-10-03T20:11:45.613Z'\n",
784+
"versionId: '4'\n",
785785
"\u001b[0;32m Setting up Cloud Build Trigger in project automlops-sandbox \u001b[0m\n",
786786
"name: dry-beans-dt-build-trigger\n",
787787
"Cloudbuild Trigger already exists in project automlops-sandbox for repo dry-beans-dt-repository\n",
@@ -816,28 +816,28 @@
816816
"output_type": "stream",
817817
"text": [
818818
"WARNING: Running precheck for deploying requires these permissions:\n",
819-
"-artifactregistry.repositories.get\n",
820-
"-iam.serviceAccounts.get\n",
821-
"-source.repos.update\n",
822819
"-resourcemanager.projects.getIamPolicy\n",
823-
"-serviceusage.services.get\n",
820+
"-artifactregistry.repositories.get\n",
824821
"-pubsub.topics.get\n",
825-
"-cloudbuild.builds.get\n",
826822
"-cloudfunctions.functions.get\n",
827-
"-storage.buckets.update\n",
823+
"-cloudbuild.builds.get\n",
828824
"-pubsub.subscriptions.get\n",
825+
"-serviceusage.services.get\n",
826+
"-storage.buckets.update\n",
827+
"-iam.serviceAccounts.get\n",
828+
"-source.repos.update\n",
829829
"\n",
830830
"You are currently using: srastatter@google.com. Please check your account permissions.\n",
831831
"The following are the recommended roles for deploying with precheck:\n",
832+
"-roles/cloudfunctions.viewer\n",
833+
"-roles/source.writer\n",
834+
"-roles/artifactregistry.reader\n",
835+
"-roles/pubsub.viewer\n",
832836
"-roles/serviceusage.serviceUsageViewer\n",
837+
"-roles/iam.roleViewer\n",
833838
"-roles/cloudbuild.builds.editor\n",
834839
"-roles/iam.serviceAccountUser\n",
835-
"-roles/pubsub.viewer\n",
836-
"-roles/cloudfunctions.viewer\n",
837840
"-roles/storage.admin\n",
838-
"-roles/artifactregistry.reader\n",
839-
"-roles/source.writer\n",
840-
"-roles/iam.roleViewer\n",
841841
"\n",
842842
"Checking for required API services in project automlops-sandbox...\n",
843843
"Checking for Artifact Registry in project automlops-sandbox...\n",
@@ -851,11 +851,39 @@
851851
"Checking for Cloud Build Trigger in project automlops-sandbox...\n",
852852
"Precheck successfully completed, continuing to deployment.\n",
853853
"\n",
854-
"[automlops d425a62] Run AutoMLOps\n",
855-
" 3 files changed, 18 insertions(+), 18 deletions(-)\n",
856-
"remote: Waiting for private key checker: 3/3 objects left \n",
854+
"Initialized empty Git repository in /Users/srastatter/Documents/2023/MLOps-graduation/AutoMLOps-github/examples/training/AutoMLOps/.git/\n",
855+
"Switched to a new branch 'automlops'\n",
856+
"[automlops (root-commit) f07e5b1] Run AutoMLOps\n",
857+
" 26 files changed, 1390 insertions(+)\n",
858+
" create mode 100644 .gitignore\n",
859+
" create mode 100644 README.md\n",
860+
" create mode 100644 cloudbuild.yaml\n",
861+
" create mode 100644 components/component_base/Dockerfile\n",
862+
" create mode 100644 components/component_base/requirements.txt\n",
863+
" create mode 100644 components/component_base/src/create_dataset.py\n",
864+
" create mode 100644 components/component_base/src/deploy_model.py\n",
865+
" create mode 100644 components/component_base/src/train_model.py\n",
866+
" create mode 100644 components/create_dataset/component.yaml\n",
867+
" create mode 100644 components/deploy_model/component.yaml\n",
868+
" create mode 100644 components/train_model/component.yaml\n",
869+
" create mode 100644 configs/defaults.yaml\n",
870+
" create mode 100644 pipelines/pipeline.py\n",
871+
" create mode 100644 pipelines/pipeline_runner.py\n",
872+
" create mode 100644 pipelines/requirements.txt\n",
873+
" create mode 100644 pipelines/runtime_parameters/pipeline_parameter_values.json\n",
874+
" create mode 100755 provision/provision_resources.sh\n",
875+
" create mode 100755 scripts/build_components.sh\n",
876+
" create mode 100755 scripts/build_pipeline_spec.sh\n",
877+
" create mode 100644 scripts/pipeline_spec/.gitkeep\n",
878+
" create mode 100755 scripts/publish_to_topic.sh\n",
879+
" create mode 100755 scripts/run_all.sh\n",
880+
" create mode 100755 scripts/run_pipeline.sh\n",
881+
" create mode 100644 services/submission_service/Dockerfile\n",
882+
" create mode 100644 services/submission_service/main.py\n",
883+
" create mode 100644 services/submission_service/requirements.txt\n",
884+
"remote: Waiting for private key checker: 11/26 objects left \n",
857885
"To https://source.developers.google.com/p/automlops-sandbox/r/dry-beans-dt-repository\n",
858-
" 33e4088..d425a62 automlops -> automlops\n",
886+
" + 8a844ee...f07e5b1 automlops -> automlops (forced update)\n",
859887
"Pushing code to automlops branch, triggering build...\n",
860888
"Cloud Build job running at: https://console.cloud.google.com/cloud-build/builds;region=us-central1\n",
861889
"Please wait for this build job to complete.\n",

‎google_cloud_automlops/AutoMLOps.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
GENERATED_RESOURCES_SH_FILE,
4242
GENERATED_SERVICES_DIRS,
4343
GENERATED_TERRAFORM_DIRS,
44+
GITHUB_DIR,
4445
OUTPUT_DIR
4546
)
4647
from google_cloud_automlops.utils.utils import (
@@ -276,6 +277,8 @@ def generate(
276277
make_dirs(GENERATED_SERVICES_DIRS)
277278
if provisioning_framework == Provisioner.TERRAFORM.value:
278279
make_dirs(GENERATED_TERRAFORM_DIRS)
280+
if deployment_framework == Deployer.GITHUB_ACTIONS.value:
281+
make_dirs(GITHUB_DIR)
279282

280283
# Set derived vars if none were given for certain variables
281284
derived_artifact_repo_name = f'{naming_prefix}-artifact-registry' if artifact_repo_name is None else artifact_repo_name

‎google_cloud_automlops/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,6 @@
2323
series of directories to support the creation of Vertex Pipelines.
2424
"""
2525
# pylint: disable=invalid-name
26-
__version__ = '1.2.3'
26+
__version__ = '1.2.4'
2727
__author__ = 'Sean Rastatter'
2828
__credits__ = 'Google'

‎google_cloud_automlops/deployments/cloudbuild/templates/cloudbuild.yaml.j2

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,14 @@ steps:
2424
waitFor: ["build_component_base"]
2525

2626
# Install dependencies
27-
- name: python
27+
- name: python:3.10
2828
entrypoint: pip
2929
args: ["install", "-r", "pipelines/requirements.txt", "--user"]
3030
id: "install_pipelines_deps"
3131
waitFor: ["push_component_base"]
3232

3333
# Build pipeline spec
34-
- name: python
34+
- name: python:3.10
3535
entrypoint: python
3636
args: ["-m", "pipelines.pipeline", "--config", "configs/defaults.yaml"]
3737
id: "build_pipeline_spec"

‎google_cloud_automlops/utils/constants.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,6 @@
7878
GENERATED_PIPELINE_JOB_SPEC_PATH = 'scripts/pipeline_spec/pipeline_job.json'
7979
GENERATED_DIRS = [
8080
BASE_DIR,
81-
GITHUB_DIR,
8281
BASE_DIR + 'components',
8382
BASE_DIR + 'components/component_base',
8483
BASE_DIR + 'components/component_base/src',

‎setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020

2121
setup(
2222
name='google-cloud-automlops',
23-
version='1.2.3',
23+
version='1.2.4',
2424
description='Build MLOps Pipelines in Minutes.',
2525
long_description=readme_contents,
2626
long_description_content_type='text/markdown',

0 commit comments

Comments
 (0)