Skip to content

Commit 8207481

Browse files
lu-ohailu-ohai
committed
Merge branch 'main' of https://github.com/oracle/accelerated-data-science into ODSC-38837/validate_load_balancer
2 parents dcd8aba + a4a3d9a commit 8207481

File tree

174 files changed

+6402
-1978
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

174 files changed

+6402
-1978
lines changed

‎.github/workflows/add-3plicense-warning.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: "Add 3P License Warning to PR"
33
on:
44
pull_request:
55
paths:
6-
- setup.py
6+
- pyproject.toml
77

88
# Cancel in progress workflows on pull_requests.
99
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -23,9 +23,9 @@ jobs:
2323
steps:
2424
- run: |
2525
BODY_MSG=$(cat << EOF
26-
⚠️ This PR changed **setup.py** file. ⚠️
27-
- PR Creator must update 📃 THIRD_PARTY_LICENSES.txt, if any 📚 library added/removed in **setup.py**.
28-
- PR Approver must confirm 📃 THIRD_PARTY_LICENSES.txt updated, if any 📚 library added/removed in **setup.py**.
26+
⚠️ This PR changed **pyproject.toml** file. ⚠️
27+
- PR Creator must update 📃 THIRD_PARTY_LICENSES.txt, if any 📚 library added/removed in **pyproject.toml**.
28+
- PR Approver must confirm 📃 THIRD_PARTY_LICENSES.txt updated, if any 📚 library added/removed in **pyproject.toml**.
2929
EOF
3030
)
3131
echo "BODY_MSG<<EOF" >> $GITHUB_ENV

‎.github/workflows/create-more-space/action.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# We are running out space on the disk, solution described here:
2+
# https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
3+
4+
name: "Create more disk space"
5+
description: "Removing some folders to create more disk space"
6+
runs:
7+
using: composite
8+
steps:
9+
- shell: bash
10+
run: |
11+
sudo rm -rf /usr/share/dotnet
12+
sudo rm -rf /opt/ghc
13+
sudo rm -rf "/usr/local/share/boost"
14+
sudo rm -rf "$AGENT_TOOLSDIRECTORY"

‎.github/workflows/publish-to-pypi.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,15 @@ jobs:
1616
python-version: "3.x"
1717
- name: Build distribution 📦
1818
run: |
19-
pip install wheel
19+
pip install build
2020
make dist
2121
- name: Validate
2222
run: |
2323
pip install dist/*.whl
2424
python -c "import ads;"
25-
## To run publish to test PyPI secret with token needs to be added,
25+
## To run publish to test PyPI a secret with token needs to be added,
2626
## this one GH_ADS_TESTPYPI_TOKEN - removed after initial test.
27-
## Project name also needed to be updated in setup.py - setup(name="test_oracle_ads", ...),
27+
## Project name also needed to be updated in pyproject.toml - name = "test_oracle_ads" in [project] section
2828
## regular name is occupied by former developer and can't be used for testing
2929
# - name: Publish distribution 📦 to Test PyPI
3030
# env:

‎.github/workflows/publish-to-readthedocs.yml

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
name: "Publish Docs"
22

33
# To run this workflow manually from the Actions tab
4-
on: workflow_dispatch
4+
on:
5+
# Auto-trigger this workflow on tag creation
6+
push:
7+
tags:
8+
- 'v*.*.*'
59

610
env:
711
RTDS_ADS_PROJECT: https://readthedocs.org/api/v3/projects/accelerated-data-science

‎.github/workflows/run-unittests-default_setup.yml

Lines changed: 10 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,13 @@
1-
name: tests/unitary/default_setup/**
1+
name: "[Py3.8][Py3.9][Py3.10] tests/unitary/default_setup/**"
22

33
on:
4+
workflow_dispatch:
45
pull_request:
5-
branches:
6-
- main
7-
- "release/**"
8-
- develop
96
paths:
107
- "ads/**"
11-
- "!ads/ads_version.json"
12-
- setup.py
8+
- pyproject.toml
139
- "**requirements.txt"
14-
- .github/workflows/run-unittests.yml
15-
- .github/workflows/run-unittests-default_setup.yml
10+
- ".github/workflows/run-unittests*.yml"
1611

1712
# Cancel in progress workflows on pull_requests.
1813
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -41,37 +36,16 @@ jobs:
4136
steps:
4237
- uses: actions/checkout@v3
4338

44-
# Caching python libraries installed with pip
45-
# https://github.com/actions/cache/blob/main/examples.md#python---pip
46-
- uses: actions/cache@v3
47-
with:
48-
path: ~/.cache/pip
49-
key: ${{ runner.os }}-pip-${{ hashFiles('**/test-requirements.txt') }}
50-
restore-keys: |
51-
${{ runner.os }}-pip-
5239
- uses: actions/setup-python@v4
5340
with:
5441
python-version: ${{ matrix.python-version }}
42+
cache: "pip"
43+
cache-dependency-path: |
44+
pyproject.toml
45+
"**requirements.txt"
5546
56-
- name: "Test config setup"
57-
shell: bash
58-
env:
59-
HOME_RUNNER_DIR: /home/runner
60-
run: |
61-
set -x # print commands that are executed
62-
mkdir -p "$HOME_RUNNER_DIR"/.oci
63-
openssl genrsa -out $HOME_RUNNER_DIR/.oci/oci_ads_user.pem 2048
64-
cat <<EOT >> "$HOME_RUNNER_DIR/.oci/config"
65-
[DEFAULT]
66-
user=ocid1.user.oc1..xxx
67-
fingerprint=00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00
68-
tenancy=ocid1.tenancy.oc1..xxx
69-
region=test_region
70-
key_file=$HOME_RUNNER_DIR/.oci/oci_ads_user.pem
71-
EOT
72-
ls -lha "$HOME_RUNNER_DIR"/.oci
73-
echo "Test config file:"
74-
cat $HOME_RUNNER_DIR/.oci/config
47+
- uses: ./.github/workflows/set-dummy-conf
48+
name: "Test config setup"
7549

7650
- name: "Run default_setup tests folder ONLY with minimum ADS dependencies"
7751
timeout-minutes: 15

‎.github/workflows/run-unittests.yml renamed to ‎.github/workflows/run-unittests-py38-cov-report.yml

Lines changed: 34 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,13 @@
1-
name: tests/unitary/**
1+
name: "[Py3.8][COV REPORT] tests/unitary/**"
22

33
on:
4+
workflow_dispatch:
45
pull_request:
5-
branches:
6-
- main
7-
- "release/**"
8-
- develop
96
paths:
107
- "ads/**"
11-
- "!ads/ads_version.json"
12-
- setup.py
8+
- pyproject.toml
139
- "**requirements.txt"
14-
- .github/workflows/run-unittests.yml
15-
- .github/workflows/run-unittests-default_setup.yml
10+
- ".github/workflows/run-unittests*.yml"
1611

1712
# Cancel in progress workflows on pull_requests.
1813
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -30,58 +25,37 @@ env:
3025

3126
jobs:
3227
test:
33-
name: python ${{ matrix.python-version }}, ${{ matrix.name }}
28+
name: python 3.8, ${{ matrix.name }}
3429
runs-on: ubuntu-latest
3530
timeout-minutes: 90
3631

3732
strategy:
3833
fail-fast: false
3934
matrix:
40-
python-version: ["3.8", "3.9", "3.10"]
41-
test-path: ["tests/unitary", "tests/unitary/with_extras/model"]
35+
test-path: ["tests/unitary/with_extras tests/unitary/default_setup", "tests/unitary/with_extras/model"]
4236
include:
43-
- test-path: "tests/unitary"
44-
ignore-path: "tests/unitary/with_extras/model"
37+
- test-path: "tests/unitary/with_extras tests/unitary/default_setup"
38+
ignore-path: "--ignore tests/unitary/with_extras/model --ignore tests/unitary/with_extras/feature_store"
4539
name: "unitary"
4640
- test-path: "tests/unitary/with_extras/model"
4741
name: "model"
48-
- python-version: "3.8"
49-
cov-reports: --cov=ads --cov-report=xml --cov-report=html
5042

5143
steps:
5244
- uses: actions/checkout@v3
5345

54-
# Caching python libraries installed with pip
55-
# https://github.com/actions/cache/blob/main/examples.md#python---pip
56-
- uses: actions/cache@v3
57-
with:
58-
path: ~/.cache/pip
59-
key: ${{ runner.os }}-pip-${{ hashFiles('**/dev-requirements.txt') }}
60-
restore-keys: |
61-
${{ runner.os }}-pip-
46+
- uses: ./.github/workflows/create-more-space
47+
name: "Create more disk space"
48+
6249
- uses: actions/setup-python@v4
6350
with:
64-
python-version: ${{ matrix.python-version }}
51+
python-version: "3.8"
52+
cache: "pip"
53+
cache-dependency-path: |
54+
pyproject.toml
55+
"**requirements.txt"
6556
66-
- name: "Test config setup"
67-
shell: bash
68-
env:
69-
HOME_RUNNER_DIR: /home/runner
70-
run: |
71-
set -x # print commands that are executed
72-
mkdir -p "$HOME_RUNNER_DIR"/.oci
73-
openssl genrsa -out $HOME_RUNNER_DIR/.oci/oci_ads_user.pem 2048
74-
cat <<EOT >> "$HOME_RUNNER_DIR/.oci/config"
75-
[DEFAULT]
76-
user=ocid1.user.oc1..xxx
77-
fingerprint=00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00
78-
tenancy=ocid1.tenancy.oc1..xxx
79-
region=test_region
80-
key_file=$HOME_RUNNER_DIR/.oci/oci_ads_user.pem
81-
EOT
82-
ls -lha "$HOME_RUNNER_DIR"/.oci
83-
echo "Test config file:"
84-
cat $HOME_RUNNER_DIR/.oci/config
57+
- uses: ./.github/workflows/set-dummy-conf
58+
name: "Test config setup"
8559

8660
- name: "Test env setup"
8761
timeout-minutes: 20
@@ -104,23 +78,19 @@ jobs:
10478
set -x # print commands that are executed
10579
10680
# Setup project and tests folder for cov reports to not be overwritten by another parallel step
107-
if [[ ! -z "${{ matrix.cov-reports }}" ]]; then
108-
mkdir -p cov-${{ matrix.name }}
109-
cd cov-${{ matrix.name }}
110-
ln -s ../tests tests
111-
ln -s ../ads ads
112-
ln -s ../.coveragerc .coveragerc
113-
fi
81+
mkdir -p cov-${{ matrix.name }}
82+
cd cov-${{ matrix.name }}
83+
ln -s ../tests tests
84+
ln -s ../ads ads
85+
ln -s ../.coveragerc .coveragerc
11486
11587
# Run tests
11688
python -m pytest -v -p no:warnings --durations=5 \
117-
-n auto --dist loadfile ${{ matrix.cov-reports }} \
118-
${{ matrix.test-path }} \
119-
--ignore "${{ matrix.ignore-path }}"
89+
-n auto --dist loadfile --cov=ads --cov-report=xml --cov-report=html \
90+
${{ matrix.test-path }} ${{ matrix.ignore-path }}
12091
12192
- name: "Save coverage files"
12293
uses: actions/upload-artifact@v3
123-
if: ${{ matrix.cov-reports }}
12494
with:
12595
name: cov-reports-${{ matrix.name }}
12696
path: |
@@ -133,7 +103,7 @@ jobs:
133103
runs-on: ubuntu-latest
134104
continue-on-error: true
135105
needs: test
136-
if: ${{ success() }} && ${{ github.event.issue.pull_request }}
106+
if: ${{ success() }} && && github.event_name == 'pull_request'
137107
env:
138108
COMPARE_BRANCH: main
139109

@@ -148,15 +118,15 @@ jobs:
148118
run: |
149119
set -x # print commands that are executed
150120
151-
# Prepare default cov body text
121+
# Prepare default cov body text
152122
COV_BODY_INTRO="📌 Overall coverage:\n\n"
153123
echo COV_BODY="$COV_BODY_INTRO No success to gather report. 😿" >> $GITHUB_ENV
154124
155125
# Prepare file paths to .coverage files
156126
# Filenames taken from job.test last step with name - "Save coverage files"
157-
FILE_UNITARY="cov-reports-unitary/.coverage"; [[ ! -f $FILE_UNITARY ]] && FILE_UNITARY=""
127+
FILE_UNITARY="cov-reports-unitary/.coverage"; [[ ! -f $FILE_UNITARY ]] && FILE_UNITARY=""
158128
FILE_MODEL="cov-reports-model/.coverage"; [[ ! -f $FILE_MODEL ]] && FILE_MODEL=""
159-
129+
160130
# Combine coverage files
161131
pip install coverage
162132
coverage combine $FILE_UNITARY $FILE_MODEL
@@ -166,7 +136,7 @@ jobs:
166136
167137
# Calculate overall coverage and update body message
168138
COV=$(grep -E 'pc_cov' htmlcov/index.html | cut -d'>' -f 2 | cut -d'%' -f 1)
169-
if [[ ! -z $COV ]]; then
139+
if [[ ! -z $COV ]]; then
170140
if [[ $COV -lt 50 ]]; then COLOR=red; elif [[ $COV -lt 80 ]]; then COLOR=yellow; else COLOR=green; fi
171141
echo COV_BODY="$COV_BODY_INTRO ![Coverage-$COV%](https://img.shields.io/badge/coverage-$COV%25-$COLOR)" >> $GITHUB_ENV
172142
fi
@@ -176,23 +146,23 @@ jobs:
176146
run: |
177147
set -x # print commands that are executed
178148
179-
# Prepare default diff body text
149+
# Prepare default diff body text
180150
DIFF_BODY_INTRO="📌 Cov diff with **${{ env.COMPARE_BRANCH }}**:\n\n"
181151
echo DIFF_BODY="$DIFF_BODY_INTRO No success to gather report. 😿" >> $GITHUB_ENV
182152
183153
# Prepare file paths to coverage xml files
184154
# Filenames taken from job.test last step with name - "Save coverage files"
185-
FILE1="cov-reports-unitary/coverage.xml"; [[ ! -f $FILE1 ]] && FILE1=""
155+
FILE1="cov-reports-unitary/coverage.xml"; [[ ! -f $FILE1 ]] && FILE1=""
186156
FILE2="cov-reports-model/coverage.xml"; [[ ! -f $FILE2 ]] && FILE2=""
187157
echo "FILE1=$FILE1" >> $GITHUB_ENV
188158
echo "FILE2=$FILE2" >> $GITHUB_ENV
189159
190160
# Calculate coverage diff and update body message
191161
pip install diff_cover
192-
diff-cover $FILE1 $FILE2 --compare-branch=origin/${{ env.COMPARE_BRANCH }}
162+
diff-cover $FILE1 $FILE2 --compare-branch=origin/${{ env.COMPARE_BRANCH }}
193163
DIFF=$(diff-cover $FILE1 $FILE2 \
194164
--compare-branch=origin/${{ env.COMPARE_BRANCH }} | grep Coverage: | cut -d' ' -f 2 | cut -d'%' -f 1)
195-
if [[ -z $DIFF ]]; then
165+
if [[ -z $DIFF ]]; then
196166
DIFF_INFO=$(diff-cover $FILE1 $FILE2 \
197167
--compare-branch=origin/${{ env.COMPARE_BRANCH }} | grep "No lines");
198168
echo DIFF_BODY="$DIFF_BODY_INTRO $DIFF_INFO">> $GITHUB_ENV

0 commit comments

Comments
 (0)