Merge branch 'develop' into ODSC-41445/ads_dataset_pandas

Author: lu-ohai

.github/workflows/publish-to-readthedocs.yml

@@ -0,0 +1,40 @@
+name: "Publish Docs"
+
+on:
+  # Auto-trigger this workflow on tag creation
+  push:
+    tags:
+      - 'v*.*.*'
+  # Auto-trigger this workflow on merge to main changes in docs/** folder
+  pull_request_target:
+    types:
+      - closed
+    branches:
+      - main
+    paths:
+      - 'docs/**'
+
+env:
+  RTDS_ADS_PROJECT: https://readthedocs.org/api/v3/projects/accelerated-data-science
+  RTDS_ADS_TOKEN: ${{ secrets.RTDS_ADS_TOKEN }}
+
+jobs:
+  build-n-publish:
+    name: Build and publish Docs 📖 to Readthedocs
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: When PR ✅ merged - Trigger Readthedocs build
+        if: github.event_name == 'pull_request_target' && github.event.pull_request.merged == true
+        run: |
+          curl \
+            -X POST \
+            -H "Authorization: Token $RTDS_ADS_TOKEN" $RTDS_ADS_PROJECT/versions/latest/builds/
+      - name: When tag 🏷️ pushed - Trigger Readthedocs build
+        if: github.event_name == 'push' && startsWith(github.ref_name, 'v')
+        run: |
+          # add 10 minutes wait time for readthedocs see freshly created tag
+          sleep 10m
+          curl \
+            -X POST \
+            -H "Authorization: Token $RTDS_ADS_TOKEN" $RTDS_ADS_PROJECT/versions/${{ github.ref_name }}/builds/

.github/workflows/run-unittests-default_setup.yml

@@ -1,18 +1,17 @@
 name: tests/unitary/default_setup/**
 
 on:
-  push:
+  pull_request:
     branches:
       - main
-      - 'release/**'
+      - "release/**"
       - develop
     paths:
-      - 'ads/**'
+      - "ads/**"
       - setup.py
-      - '**requirements.txt'
-      - '!docs/**'
-
-  pull_request:
+      - "**requirements.txt"
+      - .github/workflows/run-unittests.yml
+      - .github/workflows/run-unittests-default_setup.yml
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -77,7 +76,6 @@ jobs:
         timeout-minutes: 15
         shell: bash
         env:
-          NB_SESSION_COMPARTMENT_OCID: ocid1.compartment.oc1.<unique_ocid>
           NoDependency: True
         run: |
           set -x # print commands that are executed

.github/workflows/run-unittests.yml

@@ -1,18 +1,17 @@
 name: tests/unitary/**
 
 on:
-  push:
+  pull_request:
     branches:
       - main
-      - 'release/**'
+      - "release/**"
       - develop
     paths:
-      - 'ads/**'
+      - "ads/**"
       - setup.py
-      - '**requirements.txt'
-      - '!docs/**'
-
-  pull_request:
+      - "**requirements.txt"
+      - .github/workflows/run-unittests.yml
+      - .github/workflows/run-unittests-default_setup.yml
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -88,22 +87,21 @@ jobs:
         shell: bash
         run: |
           set -x # print commands that are executed
-          
+
           sudo apt-get install libkrb5-dev graphviz
           $CONDA/bin/conda init
           source /home/runner/.bashrc
-          
+
           pip install -r dev-requirements.txt
 
       - name: "Run unitary tests folder with maximum ADS dependencies"
         timeout-minutes: 60
         shell: bash
         env:
-          NB_SESSION_COMPARTMENT_OCID: ocid1.compartment.oc1.<unique_ocid>
           CONDA_PREFIX: /usr/share/miniconda
         run: |
           set -x # print commands that are executed
-          
+
           # Setup project and tests folder for cov reports to not be overwritten by another parallel step
           if [[ ! -z "${{ matrix.cov-reports }}" ]]; then
             mkdir -p cov-${{ matrix.name }}
@@ -112,7 +110,7 @@ jobs:
             ln -s ../ads ads
             ln -s ../.coveragerc .coveragerc
           fi
-          
+
           # Run tests
           python -m pytest -v -p no:warnings --durations=5 \
           -n auto --dist loadfile ${{ matrix.cov-reports }} \
@@ -148,41 +146,41 @@ jobs:
       - name: "Calculate overall coverage"
         run: |
           set -x # print commands that are executed
-          
+
           # Prepare default cov body text 
           COV_BODY_INTRO="📌 Overall coverage:\n\n"
           echo COV_BODY="$COV_BODY_INTRO No success to gather report. 😿" >> $GITHUB_ENV
-          
+
           # Combine coverage files
           pip install coverage
           coverage combine cov-reports-unitary/.coverage cov-reports-model/.coverage
-          
+
           # Make html report
           coverage html
-          
+
           # Calculate overall coverage and update body message
           COV=$(grep -E 'pc_cov' htmlcov/index.html | cut -d'>' -f 2 | cut -d'%' -f 1)
           if [[ ! -z $COV ]]; then 
-            if [[ $COV < 50 ]]; then COLOR=red; elif [[ $COV < 80 ]]; then COLOR=yellow; else COLOR=green; fi
+            if [[ $COV -lt 50 ]]; then COLOR=red; elif [[ $COV -lt 80 ]]; then COLOR=yellow; else COLOR=green; fi
             echo COV_BODY="$COV_BODY_INTRO ![Coverage-$COV%](https://img.shields.io/badge/coverage-$COV%25-$COLOR)" >> $GITHUB_ENV
           fi
 
       - name: "Calculate coverage diff"
         if: always()
         run: |
           set -x # print commands that are executed
-          
+
           # Prepare default diff body text 
           DIFF_BODY_INTRO="📌 Cov diff with **${{ env.COMPARE_BRANCH }}**:\n\n"
-          echo DIFF_BODY="$BODY_INTRO No success to gather report. 😿" >> $GITHUB_ENV
-          
+          echo DIFF_BODY="$DIFF_BODY_INTRO No success to gather report. 😿" >> $GITHUB_ENV
+
           # Prepare file paths to coverage xml files
           # Filenames taken from job.test last step with name - "Save coverage files"
           FILE1="cov-reports-unitary/coverage.xml"; [[ ! -f $FILE1 ]] && FILE1="" 
           FILE2="cov-reports-model/coverage.xml"; [[ ! -f $FILE2 ]] && FILE2=""
           echo "FILE1=$FILE1" >> $GITHUB_ENV
           echo "FILE2=$FILE2" >> $GITHUB_ENV
-          
+
           # Calculate coverage diff and update body message
           pip install diff_cover
           diff-cover $FILE1 $FILE2 --compare-branch=origin/${{ env.COMPARE_BRANCH }} 
@@ -193,7 +191,7 @@ jobs:
               --compare-branch=origin/${{ env.COMPARE_BRANCH }}  | grep "No lines");
             echo DIFF_BODY="$DIFF_BODY_INTRO $DIFF_INFO">> $GITHUB_ENV
           else
-            if [[ $DIFF < 50 ]]; then COLOR=red; elif [[ $DIFF < 80 ]]; then COLOR=yellow; else COLOR=green; fi
+            if [[ $DIFF -lt 50 ]]; then COLOR=red; elif [[ $DIFF -lt 80 ]]; then COLOR=yellow; else COLOR=green; fi
             echo DIFF_BODY="$DIFF_BODY_INTRO ![Coverage-$DIFF%](https://img.shields.io/badge/coverage-$DIFF%25-$COLOR)" >> $GITHUB_ENV
           fi
 

ads/ads

@@ -1,2 +1,2 @@
 export OCI_PYTHON_SDK_NO_SERVICE_IMPORTS=1
-python -m ads.cli "$@"
+python3 -m ads.cli "$@"

ads/jobs/ads_job.py

@@ -74,6 +74,9 @@ class Job(Builder):
                 .with_python_path("other_packages")
                 # Copy files in "code_dir/output" to object storage after job finishes.
                 .with_output("output", "oci://bucket_name@namespace/path/to/dir")
+                # Tags
+                .with_freeform_tag(my_tag="my_value")
+                .with_defined_tag(**{"Operations": {"CostCenter": "42"}})
             )
         )
         # Create and Run the job

ads/jobs/builders/infrastructure/base.py

@@ -61,6 +61,7 @@ def run(
         args: str = None,
         env_var: dict = None,
         freeform_tags: dict = None,
+        defined_tags: dict = None,
         wait: bool = False,
     ):
         """Runs a job on the infrastructure.
@@ -72,9 +73,11 @@ def run(
         args : str, optional
             Command line arguments for the job run, by default None.
         env_var : dict, optional
-            Environment variable for the job run, by default None
+            Environment variable for the job run, by default None.
         freeform_tags : dict, optional
-            Freeform tags for the job run, by default None
+            Freeform tags for the job run, by default None.
+        defined_tags : dict, optional
+            Defined tags for the job run, by default None.
         wait : bool, optional
             Indicate if this method should wait for the run to finish before it returns, by default False.
         """