Merge branch 'feature/feature-store-1.0.4' of github.com:oracle/accelerated-data-science into ODSC-47451-streaming-dataframe-support

backmerged the code

Author: yogesh266

.github/workflows/add-3plicense-warning.yml

@@ -3,7 +3,7 @@ name: "Add 3P License Warning to PR"
 on:
   pull_request:
     paths:
-      - setup.py
+      - pyproject.toml
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -23,9 +23,9 @@ jobs:
     steps:
       - run: |
           BODY_MSG=$(cat << EOF
-          ⚠️ This PR changed **setup.py** file. ⚠️ 
-            - PR Creator must update 📃 THIRD_PARTY_LICENSES.txt, if any 📚 library added/removed in **setup.py**.
-            - PR Approver must confirm 📃 THIRD_PARTY_LICENSES.txt updated, if any 📚 library added/removed in **setup.py**.
+          ⚠️ This PR changed **pyproject.toml** file. ⚠️
+            - PR Creator must update 📃 THIRD_PARTY_LICENSES.txt, if any 📚 library added/removed in **pyproject.toml**.
+            - PR Approver must confirm 📃 THIRD_PARTY_LICENSES.txt updated, if any 📚 library added/removed in **pyproject.toml**.
           EOF
           )
           echo "BODY_MSG<<EOF" >> $GITHUB_ENV

.github/workflows/create-more-space/action.yml

@@ -0,0 +1,14 @@
+# We are running out space on the disk, solution described here:
+# https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+
+name: "Create more disk space"
+description: "Removing some folders to create more disk space"
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      run: |
+        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf /opt/ghc
+        sudo rm -rf "/usr/local/share/boost"
+        sudo rm -rf "$AGENT_TOOLSDIRECTORY"

.github/workflows/publish-to-pypi.yml

@@ -16,15 +16,15 @@ jobs:
           python-version: "3.x"
       - name: Build distribution 📦
         run: |
-          pip install wheel
+          pip install build
           make dist
       - name: Validate
         run: |
           pip install dist/*.whl
           python -c "import ads;"
-## To run publish to test PyPI secret with token needs to be added,
+## To run publish to test PyPI a secret with token needs to be added,
 ##    this one GH_ADS_TESTPYPI_TOKEN - removed after initial test.
-## Project name also needed to be updated in setup.py - setup(name="test_oracle_ads", ...),
+## Project name also needed to be updated in pyproject.toml - name = "test_oracle_ads" in [project] section
 ##    regular name is occupied by former developer and can't be used for testing
 #      - name: Publish distribution 📦 to Test PyPI
 #        env:

.github/workflows/run-unittests-default_setup.yml

@@ -1,19 +1,13 @@
-name: tests/unitary/default_setup/**
+name: "[Py3.8][Py3.9][Py3.10] tests/unitary/default_setup/**"
 
 on:
   workflow_dispatch:
   pull_request:
-    branches:
-      - main
-      - "release/**"
-      - develop
     paths:
       - "ads/**"
-      - "!ads/ads_version.json"
-      - setup.py
+      - pyproject.toml
       - "**requirements.txt"
-      - .github/workflows/run-unittests.yml
-      - .github/workflows/run-unittests-default_setup.yml
+      - ".github/workflows/run-unittests*.yml"
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -42,37 +36,16 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      # Caching python libraries installed with pip
-      # https://github.com/actions/cache/blob/main/examples.md#python---pip
-      - uses: actions/cache@v3
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/test-requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+          cache: "pip"
+          cache-dependency-path: |
+            pyproject.toml
+            "**requirements.txt"
 
-      - name: "Test config setup"
-        shell: bash
-        env:
-          HOME_RUNNER_DIR: /home/runner
-        run: |
-          set -x # print commands that are executed
-          mkdir -p "$HOME_RUNNER_DIR"/.oci
-          openssl genrsa -out $HOME_RUNNER_DIR/.oci/oci_ads_user.pem 2048
-          cat <<EOT >> "$HOME_RUNNER_DIR/.oci/config"
-          [DEFAULT]
-          user=ocid1.user.oc1..xxx
-          fingerprint=00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00
-          tenancy=ocid1.tenancy.oc1..xxx
-          region=test_region
-          key_file=$HOME_RUNNER_DIR/.oci/oci_ads_user.pem
-          EOT
-          ls -lha "$HOME_RUNNER_DIR"/.oci
-          echo "Test config file:"
-          cat $HOME_RUNNER_DIR/.oci/config
+      - uses: ./.github/workflows/set-dummy-conf
+        name: "Test config setup"
 
       - name: "Run default_setup tests folder ONLY with minimum ADS dependencies"
         timeout-minutes: 15

.github/workflows/run-unittests.yml renamed to .github/workflows/run-unittests-py38-cov-report.yml

@@ -1,19 +1,13 @@
-name: tests/unitary/**
+name: "[Py3.8][COV REPORT] tests/unitary/**"
 
 on:
   workflow_dispatch:
   pull_request:
-    branches:
-      - main
-      - "release/**"
-      - develop
     paths:
       - "ads/**"
-      - "!ads/ads_version.json"
-      - setup.py
+      - pyproject.toml
       - "**requirements.txt"
-      - .github/workflows/run-unittests.yml
-      - .github/workflows/run-unittests-default_setup.yml
+      - ".github/workflows/run-unittests*.yml"
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -31,58 +25,37 @@ env:
 
 jobs:
   test:
-    name: python ${{ matrix.python-version }}, ${{ matrix.name }}
+    name: python 3.8, ${{ matrix.name }}
     runs-on: ubuntu-latest
     timeout-minutes: 90
 
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
-        test-path: ["tests/unitary", "tests/unitary/with_extras/model"]
+        test-path: ["tests/unitary/with_extras tests/unitary/default_setup", "tests/unitary/with_extras/model"]
         include:
-          - test-path: "tests/unitary"
+          - test-path: "tests/unitary/with_extras tests/unitary/default_setup"
             ignore-path: "--ignore tests/unitary/with_extras/model --ignore tests/unitary/with_extras/feature_store"
             name: "unitary"
           - test-path: "tests/unitary/with_extras/model"
             name: "model"
-          - python-version: "3.8"
-            cov-reports: --cov=ads --cov-report=xml --cov-report=html
 
     steps:
       - uses: actions/checkout@v3
 
-      # Caching python libraries installed with pip
-      # https://github.com/actions/cache/blob/main/examples.md#python---pip
-      - uses: actions/cache@v3
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/dev-requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
+      - uses: ./.github/workflows/create-more-space
+        name: "Create more disk space"
+
       - uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: "3.8"
+          cache: "pip"
+          cache-dependency-path: |
+            pyproject.toml
+            "**requirements.txt"
 
-      - name: "Test config setup"
-        shell: bash
-        env:
-          HOME_RUNNER_DIR: /home/runner
-        run: |
-          set -x # print commands that are executed
-          mkdir -p "$HOME_RUNNER_DIR"/.oci
-          openssl genrsa -out $HOME_RUNNER_DIR/.oci/oci_ads_user.pem 2048
-          cat <<EOT >> "$HOME_RUNNER_DIR/.oci/config"
-          [DEFAULT]
-          user=ocid1.user.oc1..xxx
-          fingerprint=00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00
-          tenancy=ocid1.tenancy.oc1..xxx
-          region=test_region
-          key_file=$HOME_RUNNER_DIR/.oci/oci_ads_user.pem
-          EOT
-          ls -lha "$HOME_RUNNER_DIR"/.oci
-          echo "Test config file:"
-          cat $HOME_RUNNER_DIR/.oci/config
+      - uses: ./.github/workflows/set-dummy-conf
+        name: "Test config setup"
 
       - name: "Test env setup"
         timeout-minutes: 20
@@ -105,22 +78,19 @@ jobs:
           set -x # print commands that are executed
 
           # Setup project and tests folder for cov reports to not be overwritten by another parallel step
-          if [[ ! -z "${{ matrix.cov-reports }}" ]]; then
-            mkdir -p cov-${{ matrix.name }}
-            cd cov-${{ matrix.name }}
-            ln -s ../tests tests
-            ln -s ../ads ads
-            ln -s ../.coveragerc .coveragerc
-          fi
+          mkdir -p cov-${{ matrix.name }}
+          cd cov-${{ matrix.name }}
+          ln -s ../tests tests
+          ln -s ../ads ads
+          ln -s ../.coveragerc .coveragerc
 
           # Run tests
           python -m pytest -v -p no:warnings --durations=5 \
-          -n auto --dist loadfile ${{ matrix.cov-reports }} \
+          -n auto --dist loadfile --cov=ads --cov-report=xml --cov-report=html \
           ${{ matrix.test-path }} ${{ matrix.ignore-path }}
 
       - name: "Save coverage files"
         uses: actions/upload-artifact@v3
-        if: ${{ matrix.cov-reports }}
         with:
           name: cov-reports-${{ matrix.name }}
           path: |
@@ -133,7 +103,7 @@ jobs:
     runs-on: ubuntu-latest
     continue-on-error: true
     needs: test
-    if: ${{ success() }} && ${{ github.event.issue.pull_request }}
+    if: ${{ success() }} && && github.event_name == 'pull_request'
     env:
       COMPARE_BRANCH: main
 
@@ -148,15 +118,15 @@ jobs:
         run: |
           set -x # print commands that are executed
 
-          # Prepare default cov body text 
+          # Prepare default cov body text
           COV_BODY_INTRO="📌 Overall coverage:\n\n"
           echo COV_BODY="$COV_BODY_INTRO No success to gather report. 😿" >> $GITHUB_ENV
 
           # Prepare file paths to .coverage files
           # Filenames taken from job.test last step with name - "Save coverage files"
-          FILE_UNITARY="cov-reports-unitary/.coverage"; [[ ! -f $FILE_UNITARY ]] && FILE_UNITARY="" 
+          FILE_UNITARY="cov-reports-unitary/.coverage"; [[ ! -f $FILE_UNITARY ]] && FILE_UNITARY=""
           FILE_MODEL="cov-reports-model/.coverage"; [[ ! -f $FILE_MODEL ]] && FILE_MODEL=""
-          
+
           # Combine coverage files
           pip install coverage
           coverage combine $FILE_UNITARY $FILE_MODEL
@@ -166,7 +136,7 @@ jobs:
 
           # Calculate overall coverage and update body message
           COV=$(grep -E 'pc_cov' htmlcov/index.html | cut -d'>' -f 2 | cut -d'%' -f 1)
-          if [[ ! -z $COV ]]; then 
+          if [[ ! -z $COV ]]; then
             if [[ $COV -lt 50 ]]; then COLOR=red; elif [[ $COV -lt 80 ]]; then COLOR=yellow; else COLOR=green; fi
             echo COV_BODY="$COV_BODY_INTRO ![Coverage-$COV%](https://img.shields.io/badge/coverage-$COV%25-$COLOR)" >> $GITHUB_ENV
           fi
@@ -176,23 +146,23 @@ jobs:
         run: |
           set -x # print commands that are executed
 
-          # Prepare default diff body text 
+          # Prepare default diff body text
           DIFF_BODY_INTRO="📌 Cov diff with **${{ env.COMPARE_BRANCH }}**:\n\n"
           echo DIFF_BODY="$DIFF_BODY_INTRO No success to gather report. 😿" >> $GITHUB_ENV
 
           # Prepare file paths to coverage xml files
           # Filenames taken from job.test last step with name - "Save coverage files"
-          FILE1="cov-reports-unitary/coverage.xml"; [[ ! -f $FILE1 ]] && FILE1="" 
+          FILE1="cov-reports-unitary/coverage.xml"; [[ ! -f $FILE1 ]] && FILE1=""
           FILE2="cov-reports-model/coverage.xml"; [[ ! -f $FILE2 ]] && FILE2=""
           echo "FILE1=$FILE1" >> $GITHUB_ENV
           echo "FILE2=$FILE2" >> $GITHUB_ENV
 
           # Calculate coverage diff and update body message
           pip install diff_cover
-          diff-cover $FILE1 $FILE2 --compare-branch=origin/${{ env.COMPARE_BRANCH }} 
+          diff-cover $FILE1 $FILE2 --compare-branch=origin/${{ env.COMPARE_BRANCH }}
           DIFF=$(diff-cover $FILE1 $FILE2 \
             --compare-branch=origin/${{ env.COMPARE_BRANCH }} | grep Coverage: | cut -d' ' -f 2 | cut -d'%' -f 1)
-          if [[ -z $DIFF ]]; then 
+          if [[ -z $DIFF ]]; then
             DIFF_INFO=$(diff-cover $FILE1 $FILE2 \
               --compare-branch=origin/${{ env.COMPARE_BRANCH }}  | grep "No lines");
             echo DIFF_BODY="$DIFF_BODY_INTRO $DIFF_INFO">> $GITHUB_ENV

.github/workflows/run-unittests-py39-py310.yml

@@ -0,0 +1,84 @@
+name: "[Py3.9][Py3.10] - tests/unitary/**"
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - "ads/**"
+      - pyproject.toml
+      - "**requirements.txt"
+      - ".github/workflows/run-unittests*.yml"
+
+# Cancel in progress workflows on pull_requests.
+# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  pull-requests: write
+
+# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359
+env:
+  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
+
+jobs:
+  test:
+    name: python ${{ matrix.python-version }}, ${{ matrix.name }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10"]
+        test-path: ["tests/unitary/with_extras tests/unitary/default_setup", "tests/unitary/with_extras/model"]
+        include:
+          - test-path: "tests/unitary/with_extras tests/unitary/default_setup"
+            ignore-path: "--ignore tests/unitary/with_extras/model --ignore tests/unitary/with_extras/feature_store"
+            name: "unitary"
+          - test-path: "tests/unitary/with_extras/model"
+            name: "model"
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: ./.github/workflows/create-more-space
+        name: "Create more disk space"
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+          cache-dependency-path: |
+            pyproject.toml
+            "**requirements.txt"
+
+      - uses: ./.github/workflows/set-dummy-conf
+        name: "Test config setup"
+
+      - name: "Test env setup"
+        timeout-minutes: 20
+        shell: bash
+        run: |
+          set -x # print commands that are executed
+
+          sudo apt-get install libkrb5-dev graphviz
+          $CONDA/bin/conda init
+          source /home/runner/.bashrc
+
+          pip install -r dev-requirements.txt
+
+      - name: "Run unitary tests folder with maximum ADS dependencies"
+        timeout-minutes: 60
+        shell: bash
+        env:
+          CONDA_PREFIX: /usr/share/miniconda
+        run: |
+          set -x # print commands that are executed
+
+          # Run tests
+          python -m pytest -v -p no:warnings --durations=5 \
+          -n auto --dist loadfile \
+          ${{ matrix.test-path }} ${{ matrix.ignore-path }}