Skip to content

Commit 339749f

Browse files
srastattersrastatter
committed
updated documentation
1 parent 62768da commit 339749f

9 files changed

+43
-143
lines changed

‎README.md

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,6 @@ Based on the above user selection, AutoMLOps will enable up to the following API
165165
- [cloudfunctions.googleapis.com](https://cloud.google.com/functions/docs/reference/rest)
166166
- [cloudresourcemanager.googleapis.com](https://cloud.google.com/resource-manager/reference/rest)
167167
- [cloudscheduler.googleapis.com](https://cloud.google.com/scheduler/docs/reference/rest)
168-
- [cloudtasks.googleapis.com](https://cloud.google.com/tasks/docs/reference/rest)
169168
- [compute.googleapis.com](https://cloud.google.com/compute/docs/reference/rest/v1)
170169
- [iam.googleapis.com](https://cloud.google.com/iam/docs/reference/rest)
171170
- [iamcredentials.googleapis.com](https://cloud.google.com/iam/docs/reference/credentials/rest)

‎examples/inferencing/00_batch_prediction_example.ipynb

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,6 @@
100100
"- [cloudfunctions.googleapis.com](https://cloud.google.com/functions/docs/reference/rest)\n",
101101
"- [cloudresourcemanager.googleapis.com](https://cloud.google.com/resource-manager/reference/rest)\n",
102102
"- [cloudscheduler.googleapis.com](https://cloud.google.com/scheduler/docs/reference/rest)\n",
103-
"- [cloudtasks.googleapis.com](https://cloud.google.com/tasks/docs/reference/rest)\n",
104103
"- [compute.googleapis.com](https://cloud.google.com/compute/docs/reference/rest/v1)\n",
105104
"- [iam.googleapis.com](https://cloud.google.com/iam/docs/reference/rest)\n",
106105
"- [iamcredentials.googleapis.com](https://cloud.google.com/iam/docs/reference/credentials/rest)\n",

‎examples/inferencing/01_customer_churn_model_monitoring_example.ipynb

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,6 @@
130130
"- [cloudfunctions.googleapis.com](https://cloud.google.com/functions/docs/reference/rest)\n",
131131
"- [cloudresourcemanager.googleapis.com](https://cloud.google.com/resource-manager/reference/rest)\n",
132132
"- [cloudscheduler.googleapis.com](https://cloud.google.com/scheduler/docs/reference/rest)\n",
133-
"- [cloudtasks.googleapis.com](https://cloud.google.com/tasks/docs/reference/rest)\n",
134133
"- [compute.googleapis.com](https://cloud.google.com/compute/docs/reference/rest/v1)\n",
135134
"- [iam.googleapis.com](https://cloud.google.com/iam/docs/reference/rest)\n",
136135
"- [iamcredentials.googleapis.com](https://cloud.google.com/iam/docs/reference/credentials/rest)\n",

‎examples/llmops/finetuning-flan-t5/00_llmops_finetuning_flan_t5_example.ipynb

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,6 @@
102102
"- [cloudfunctions.googleapis.com](https://cloud.google.com/functions/docs/reference/rest)\n",
103103
"- [cloudresourcemanager.googleapis.com](https://cloud.google.com/resource-manager/reference/rest)\n",
104104
"- [cloudscheduler.googleapis.com](https://cloud.google.com/scheduler/docs/reference/rest)\n",
105-
"- [cloudtasks.googleapis.com](https://cloud.google.com/tasks/docs/reference/rest)\n",
106105
"- [compute.googleapis.com](https://cloud.google.com/compute/docs/reference/rest/v1)\n",
107106
"- [iam.googleapis.com](https://cloud.google.com/iam/docs/reference/rest)\n",
108107
"- [iamcredentials.googleapis.com](https://cloud.google.com/iam/docs/reference/credentials/rest)\n",

‎examples/training/00_introduction_training_example.ipynb

Lines changed: 41 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,6 @@
9999
"- [cloudfunctions.googleapis.com](https://cloud.google.com/functions/docs/reference/rest)\n",
100100
"- [cloudresourcemanager.googleapis.com](https://cloud.google.com/resource-manager/reference/rest)\n",
101101
"- [cloudscheduler.googleapis.com](https://cloud.google.com/scheduler/docs/reference/rest)\n",
102-
"- [cloudtasks.googleapis.com](https://cloud.google.com/tasks/docs/reference/rest)\n",
103102
"- [compute.googleapis.com](https://cloud.google.com/compute/docs/reference/rest/v1)\n",
104103
"- [iam.googleapis.com](https://cloud.google.com/iam/docs/reference/rest)\n",
105104
"- [iamcredentials.googleapis.com](https://cloud.google.com/iam/docs/reference/credentials/rest)\n",
@@ -698,45 +697,45 @@
698697
"output_type": "stream",
699698
"text": [
700699
"WARNING: Provisioning requires these permissions:\n",
700+
"-iam.serviceAccounts.create\n",
701+
"-serviceusage.services.enable\n",
702+
"-cloudfunctions.functions.create\n",
701703
"-iam.serviceAccounts.list\n",
704+
"-source.repos.list\n",
705+
"-pubsub.topics.create\n",
702706
"-storage.buckets.create\n",
707+
"-source.repos.create\n",
703708
"-cloudscheduler.jobs.create\n",
704-
"-serviceusage.services.use\n",
705-
"-pubsub.subscriptions.create\n",
706-
"-cloudbuild.builds.create\n",
707-
"-pubsub.topics.create\n",
708-
"-cloudfunctions.functions.create\n",
709709
"-resourcemanager.projects.setIamPolicy\n",
710-
"-storage.buckets.get\n",
711-
"-iam.serviceAccounts.actAs\n",
712-
"-iam.serviceAccounts.create\n",
713-
"-source.repos.create\n",
714710
"-pubsub.subscriptions.list\n",
715-
"-cloudscheduler.jobs.list\n",
716-
"-artifactregistry.repositories.create\n",
717-
"-serviceusage.services.enable\n",
711+
"-serviceusage.services.use\n",
712+
"-storage.buckets.get\n",
713+
"-cloudbuild.builds.create\n",
714+
"-pubsub.subscriptions.create\n",
715+
"-artifactregistry.repositories.list\n",
718716
"-cloudfunctions.functions.get\n",
719-
"-source.repos.list\n",
720-
"-pubsub.topics.list\n",
721717
"-cloudbuild.builds.list\n",
722-
"-artifactregistry.repositories.list\n",
718+
"-artifactregistry.repositories.create\n",
719+
"-cloudscheduler.jobs.list\n",
720+
"-iam.serviceAccounts.actAs\n",
721+
"-pubsub.topics.list\n",
723722
"\n",
724723
"You are currently using: srastatter@google.com. Please check your account permissions.\n",
725724
"The following are the recommended roles for provisioning:\n",
726-
"-roles/iam.serviceAccountUser\n",
727725
"-roles/serviceusage.serviceUsageAdmin\n",
728-
"-roles/cloudfunctions.admin\n",
729-
"-roles/storage.admin\n",
730-
"-roles/source.admin\n",
731-
"-roles/iam.serviceAccountAdmin\n",
732-
"-roles/cloudscheduler.admin\n",
733726
"-roles/artifactregistry.admin\n",
734-
"-roles/pubsub.editor\n",
735727
"-roles/cloudbuild.builds.editor\n",
736728
"-roles/resourcemanager.projectIamAdmin\n",
729+
"-roles/cloudscheduler.admin\n",
730+
"-roles/iam.serviceAccountUser\n",
731+
"-roles/iam.serviceAccountAdmin\n",
732+
"-roles/storage.admin\n",
733+
"-roles/cloudfunctions.admin\n",
734+
"-roles/pubsub.editor\n",
735+
"-roles/source.admin\n",
737736
"\n",
738737
"\u001b[0;32m Setting up API services in project automlops-sandbox \u001b[0m\n",
739-
"Operation \"operations/acat.p2-45373616427-a14841f0-e60d-4c6c-ace7-073e858c1888\" finished successfully.\n",
738+
"Operation \"operations/acat.p2-45373616427-774ab104-8889-497b-a35a-5dea7ab8bb32\" finished successfully.\n",
740739
"\u001b[0;32m Setting up Artifact Registry in project automlops-sandbox \u001b[0m\n",
741740
"Listing items under project automlops-sandbox, location us-central1.\n",
742741
"\n",
@@ -758,11 +757,11 @@
758757
"\u001b[0;32m Deploying Cloud Functions: dry-beans-dt-job-submission-svc in project automlops-sandbox \u001b[0m\n",
759758
"Deploying function (may take a while - up to 2 minutes)...\n",
760759
"..\n",
761-
"For Cloud Build Logs, visit: https://console.cloud.google.com/cloud-build/builds;region=us-central1/71111d1b-0448-4f22-862b-d040e37cedf2?project=45373616427\n",
762-
"................................................................done.\n",
760+
"For Cloud Build Logs, visit: https://console.cloud.google.com/cloud-build/builds;region=us-central1/09aeff6a-d087-44fc-a700-c854f10388b2?project=45373616427\n",
761+
"...............................................................done.\n",
763762
"availableMemoryMb: 512\n",
764-
"buildId: 71111d1b-0448-4f22-862b-d040e37cedf2\n",
765-
"buildName: projects/45373616427/locations/us-central1/builds/71111d1b-0448-4f22-862b-d040e37cedf2\n",
763+
"buildId: 09aeff6a-d087-44fc-a700-c854f10388b2\n",
764+
"buildName: projects/45373616427/locations/us-central1/builds/09aeff6a-d087-44fc-a700-c854f10388b2\n",
766765
"dockerRegistry: ARTIFACT_REGISTRY\n",
767766
"entryPoint: process_request\n",
768767
"eventTrigger:\n",
@@ -777,11 +776,11 @@
777776
"name: projects/automlops-sandbox/locations/us-central1/functions/dry-beans-dt-job-submission-svc\n",
778777
"runtime: python39\n",
779778
"serviceAccountEmail: vertex-pipelines@automlops-sandbox.iam.gserviceaccount.com\n",
780-
"sourceUploadUrl: https://storage.googleapis.com/uploads-961973632599.us-central1.cloudfunctions.appspot.com/de8b497b-dff0-4ba2-b88c-d9b59f868941.zip\n",
779+
"sourceUploadUrl: https://storage.googleapis.com/uploads-961973632599.us-central1.cloudfunctions.appspot.com/fc8a1e99-593a-4b71-b489-46db398c8194.zip\n",
781780
"status: ACTIVE\n",
782781
"timeout: 540s\n",
783-
"updateTime: '2023-09-25T16:24:11.187Z'\n",
784-
"versionId: '5'\n",
782+
"updateTime: '2023-09-25T17:12:41.289Z'\n",
783+
"versionId: '6'\n",
785784
"\u001b[0;32m Setting up Cloud Build Trigger in project automlops-sandbox \u001b[0m\n",
786785
"name: dry-beans-dt-build-trigger\n",
787786
"Cloudbuild Trigger already exists in project automlops-sandbox for repo dry-beans-dt-repository\n",
@@ -816,26 +815,26 @@
816815
"output_type": "stream",
817816
"text": [
818817
"WARNING: Running precheck for deploying requires these permissions:\n",
819-
"-resourcemanager.projects.getIamPolicy\n",
820-
"-pubsub.subscriptions.get\n",
821818
"-cloudbuild.builds.get\n",
822819
"-storage.buckets.update\n",
823-
"-iam.serviceAccounts.get\n",
824-
"-artifactregistry.repositories.get\n",
825820
"-cloudfunctions.functions.get\n",
821+
"-artifactregistry.repositories.get\n",
822+
"-iam.serviceAccounts.get\n",
826823
"-pubsub.topics.get\n",
827-
"-source.repos.update\n",
828824
"-serviceusage.services.get\n",
825+
"-source.repos.update\n",
826+
"-pubsub.subscriptions.get\n",
827+
"-resourcemanager.projects.getIamPolicy\n",
829828
"\n",
830829
"You are currently using: srastatter@google.com. Please check your account permissions.\n",
831830
"The following are the recommended roles for deploying with precheck:\n",
832-
"-roles/pubsub.viewer\n",
833831
"-roles/serviceusage.serviceUsageViewer\n",
834832
"-roles/cloudbuild.builds.editor\n",
835833
"-roles/source.writer\n",
836834
"-roles/iam.serviceAccountUser\n",
837-
"-roles/storage.admin\n",
838835
"-roles/iam.roleViewer\n",
836+
"-roles/pubsub.viewer\n",
837+
"-roles/storage.admin\n",
839838
"-roles/cloudfunctions.viewer\n",
840839
"-roles/artifactregistry.reader\n",
841840
"\n",
@@ -851,38 +850,11 @@
851850
"Checking for Cloud Build Trigger in project automlops-sandbox...\n",
852851
"Precheck successfully completed, continuing to deployment.\n",
853852
"\n",
854-
"Initialized empty Git repository in /Users/srastatter/Documents/2023/MLOps-graduation/AutoMLOps-github/examples/training/.git/\n",
855-
"Switched to a new branch 'automlops'\n",
856-
"[automlops (root-commit) 4c3cee5] Run AutoMLOps\n",
857-
" 25 files changed, 1237 insertions(+)\n",
858-
" create mode 100644 AutoMLOps/README.md\n",
859-
" create mode 100644 AutoMLOps/cloudbuild.yaml\n",
860-
" create mode 100644 AutoMLOps/components/component_base/Dockerfile\n",
861-
" create mode 100644 AutoMLOps/components/component_base/requirements.txt\n",
862-
" create mode 100644 AutoMLOps/components/component_base/src/create_dataset.py\n",
863-
" create mode 100644 AutoMLOps/components/component_base/src/deploy_model.py\n",
864-
" create mode 100644 AutoMLOps/components/component_base/src/train_model.py\n",
865-
" create mode 100644 AutoMLOps/components/create_dataset/component.yaml\n",
866-
" create mode 100644 AutoMLOps/components/deploy_model/component.yaml\n",
867-
" create mode 100644 AutoMLOps/components/train_model/component.yaml\n",
868-
" create mode 100644 AutoMLOps/configs/defaults.yaml\n",
869-
" create mode 100644 AutoMLOps/pipelines/pipeline.py\n",
870-
" create mode 100644 AutoMLOps/pipelines/pipeline_runner.py\n",
871-
" create mode 100644 AutoMLOps/pipelines/requirements.txt\n",
872-
" create mode 100644 AutoMLOps/pipelines/runtime_parameters/pipeline_parameter_values.json\n",
873-
" create mode 100755 AutoMLOps/provision/provision_resources.sh\n",
874-
" create mode 100755 AutoMLOps/scripts/build_components.sh\n",
875-
" create mode 100755 AutoMLOps/scripts/build_pipeline_spec.sh\n",
876-
" create mode 100644 AutoMLOps/scripts/pipeline_spec/.gitkeep\n",
877-
" create mode 100755 AutoMLOps/scripts/publish_to_topic.sh\n",
878-
" create mode 100755 AutoMLOps/scripts/run_all.sh\n",
879-
" create mode 100755 AutoMLOps/scripts/run_pipeline.sh\n",
880-
" create mode 100644 AutoMLOps/services/submission_service/Dockerfile\n",
881-
" create mode 100644 AutoMLOps/services/submission_service/main.py\n",
882-
" create mode 100644 AutoMLOps/services/submission_service/requirements.txt\n",
883-
"remote: Waiting for private key checker: 12/25 objects left \n",
853+
"[automlops 21e4739] Run AutoMLOps\n",
854+
" 2 files changed, 14 insertions(+), 16 deletions(-)\n",
855+
"remote: Waiting for private key checker: 2/2 objects left \n",
884856
"To https://source.developers.google.com/p/automlops-sandbox/r/dry-beans-dt-repository\n",
885-
" + 81a1cd1...4c3cee5 automlops -> automlops (forced update)\n",
857+
" 4c3cee5..21e4739 automlops -> automlops\n",
886858
"Pushing code to automlops branch, triggering build...\n",
887859
"Cloud Build job running at: https://console.cloud.google.com/cloud-build/builds;region=us-central1\n",
888860
"Please wait for this build job to complete.\n",

‎examples/training/01_clustering_example.ipynb

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,6 @@
9898
"- [cloudfunctions.googleapis.com](https://cloud.google.com/functions/docs/reference/rest)\n",
9999
"- [cloudresourcemanager.googleapis.com](https://cloud.google.com/resource-manager/reference/rest)\n",
100100
"- [cloudscheduler.googleapis.com](https://cloud.google.com/scheduler/docs/reference/rest)\n",
101-
"- [cloudtasks.googleapis.com](https://cloud.google.com/tasks/docs/reference/rest)\n",
102101
"- [compute.googleapis.com](https://cloud.google.com/compute/docs/reference/rest/v1)\n",
103102
"- [iam.googleapis.com](https://cloud.google.com/iam/docs/reference/rest)\n",
104103
"- [iamcredentials.googleapis.com](https://cloud.google.com/iam/docs/reference/credentials/rest)\n",

‎examples/training/02_tensorflow_transfer_learning_gpu_example.ipynb

Lines changed: 2 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,6 @@
9999
"- [cloudfunctions.googleapis.com](https://cloud.google.com/functions/docs/reference/rest)\n",
100100
"- [cloudresourcemanager.googleapis.com](https://cloud.google.com/resource-manager/reference/rest)\n",
101101
"- [cloudscheduler.googleapis.com](https://cloud.google.com/scheduler/docs/reference/rest)\n",
102-
"- [cloudtasks.googleapis.com](https://cloud.google.com/tasks/docs/reference/rest)\n",
103102
"- [compute.googleapis.com](https://cloud.google.com/compute/docs/reference/rest/v1)\n",
104103
"- [iam.googleapis.com](https://cloud.google.com/iam/docs/reference/rest)\n",
105104
"- [iamcredentials.googleapis.com](https://cloud.google.com/iam/docs/reference/credentials/rest)\n",
@@ -610,74 +609,10 @@
610609
},
611610
{
612611
"cell_type": "code",
613-
"execution_count": 9,
612+
"execution_count": null,
614613
"id": "25cfeff0-c5e9-4861-83f0-844dac5e153a",
615614
"metadata": {},
616-
"outputs": [
617-
{
618-
"name": "stdout",
619-
"output_type": "stream",
620-
"text": [
621-
"INFO: Successfully saved requirements file in AutoMLOps/components/component_base/requirements.txt\n",
622-
"\u001b[0;32m Updating required API services in project automlops-sandbox \u001b[0m\n",
623-
"Operation \"operations/acat.p2-45373616427-8bbeca0a-e95a-48c7-8e7e-47fed2d515bd\" finished successfully.\n",
624-
"\u001b[0;32m Checking for Artifact Registry: vertex-mlops-af in project automlops-sandbox \u001b[0m\n",
625-
"Listing items under project automlops-sandbox, location us-central1.\n",
626-
"\n",
627-
"vertex-mlops-af DOCKER STANDARD_REPOSITORY Artifact Registry vertex-mlops-af in us-central1. us-central1 Google-managed key 2023-01-11T17:12:26 2023-07-07T11:04:11 71299.646\n",
628-
"Artifact Registry: vertex-mlops-af already exists in project automlops-sandbox\n",
629-
"\u001b[0;32m Checking for GS Bucket: automlops-sandbox-bucket in project automlops-sandbox \u001b[0m\n",
630-
"gs://automlops-sandbox-bucket/\n",
631-
"GS Bucket: automlops-sandbox-bucket already exists in project automlops-sandbox\n",
632-
"\u001b[0;32m Checking for Service Account: vertex-pipelines in project automlops-sandbox \u001b[0m\n",
633-
"Pipeline Runner Service Account vertex-pipelines@automlops-sandbox.iam.gserviceaccount.com False\n",
634-
"Service Account: vertex-pipelines already exists in project automlops-sandbox\n",
635-
"\u001b[0;32m Updating required IAM roles in project automlops-sandbox \u001b[0m\n",
636-
"\u001b[0;32m Checking for Cloud Source Repository: AutoMLOps-repo in project automlops-sandbox \u001b[0m\n",
637-
"AutoMLOps-repo automlops-sandbox https://source.developers.google.com/p/automlops-sandbox/r/AutoMLOps-repo\n",
638-
"Cloud Source Repository: AutoMLOps-repo already exists in project automlops-sandbox\n",
639-
"\u001b[0;32m Checking for Cloud Tasks Queue: queueing-svc in project automlops-sandbox \u001b[0m\n",
640-
"queueing-svc RUNNING 1000 500.0 100\n",
641-
"Cloud Tasks Queue: queueing-svc already exists in project automlops-sandbox\n",
642-
"\u001b[0;32m Checking for Cloudbuild Trigger: automlops-trigger in project automlops-sandbox \u001b[0m\n",
643-
"name: automlops-trigger\n",
644-
"Cloudbuild Trigger already exists in project automlops-sandbox for repo AutoMLOps-repo\n",
645-
"[automlops 94a6245] Run AutoMLOps\n",
646-
" 18 files changed, 497 insertions(+), 428 deletions(-)\n",
647-
" delete mode 100644 .AutoMLOps-cache/create_dataset.yaml\n",
648-
" create mode 100644 .AutoMLOps-cache/custom_train_model.yaml\n",
649-
" delete mode 100644 .AutoMLOps-cache/deploy_model.yaml\n",
650-
" delete mode 100644 .AutoMLOps-cache/train_model.yaml\n",
651-
" create mode 100644 AutoMLOps/components/component_base/src/custom_train_model.py\n",
652-
" create mode 100644 AutoMLOps/components/custom_train_model/component.yaml\n",
653-
"remote: Waiting for private key checker: 8/13 objects left \n",
654-
"To https://source.developers.google.com/p/automlops-sandbox/r/AutoMLOps-repo\n",
655-
" 8b405ef..94a6245 automlops -> automlops\n",
656-
"Pushing code to automlops branch, triggering cloudbuild...\n",
657-
"Cloudbuild job running at: https://console.cloud.google.com/cloud-build/builds;region=us-central1\n",
658-
"\n",
659-
"#################################################################\n",
660-
"# #\n",
661-
"# RESOURCES MANIFEST #\n",
662-
"#---------------------------------------------------------------#\n",
663-
"# Generated resources can be found at the following urls #\n",
664-
"# #\n",
665-
"#################################################################\n",
666-
"\n",
667-
"Google Cloud Storage Bucket: https://console.cloud.google.com/storage/automlops-sandbox-bucket\n",
668-
"Artifact Registry: https://console.cloud.google.com/artifacts/docker/automlops-sandbox/us-central1/vertex-mlops-af\n",
669-
"Service Accounts: https://console.cloud.google.com/iam-admin/serviceaccounts?project=automlops-sandbox\n",
670-
"APIs: https://console.cloud.google.com/apis\n",
671-
"Cloud Source Repository: https://source.cloud.google.com/automlops-sandbox/AutoMLOps-repo/+/automlops:\n",
672-
"Cloud Build Jobs: https://console.cloud.google.com/cloud-build/builds;region=us-central1\n",
673-
"Vertex AI Pipeline Runs: https://console.cloud.google.com/vertex-ai/pipelines/runs\n",
674-
"Cloud Build Trigger: https://console.cloud.google.com/cloud-build/triggers;region=us-central1\n",
675-
"Cloud Run Service: https://console.cloud.google.com/run/detail/us-central1/run-pipeline\n",
676-
"Cloud Tasks Queue: https://console.cloud.google.com/cloudtasks/queue/us-central1/queueing-svc/tasks\n",
677-
"Cloud Scheduler Job: https://console.cloud.google.com/cloudscheduler\n"
678-
]
679-
}
680-
],
615+
"outputs": [],
681616
"source": [
682617
"AutoMLOps.launchAll(project_id=PROJECT_ID, \n",
683618
" pipeline_params=pipeline_params, \n",

0 commit comments

Comments
 (0)