Release Python SDK v0.4.1 #114
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI (OSS) | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - "v*" | |
| # Cancel in-progress CI jobs if there is a new push | |
| # https://stackoverflow.com/a/72408109 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| check_changes: | |
| name: Check for changes | |
| runs-on: ubuntu-latest | |
| outputs: | |
| has-changes: ${{ steps.check.outputs.has-changes }} | |
| steps: | |
| - name: Check | |
| id: check | |
| uses: jiahuei/check-changes-action@v0 | |
| with: | |
| watch-dirs: "clients/python/ services/api/ docker/ .github/" | |
| sdk_tests_noop: | |
| # This job is needed so that status checks can still pass | |
| # This is because strategy matrix is evaluated after if condition | |
| name: SDK unit tests | |
| runs-on: ubuntu-latest | |
| needs: check_changes | |
| if: ${{ !(needs.check_changes.outputs.has-changes == 'true' || github.event_name == 'push') }} | |
| strategy: | |
| matrix: | |
| python-version: ["3.10"] | |
| timeout-minutes: 2 | |
| steps: | |
| - name: No-op | |
| run: echo Tests skipped !!! | |
| sdk_tests: | |
| name: SDK unit tests | |
| runs-on: ubuntu-latest-l | |
| needs: check_changes | |
| if: needs.check_changes.outputs.has-changes == 'true' || github.event_name == 'push' | |
| strategy: | |
| matrix: | |
| python-version: ["3.10"] | |
| timeout-minutes: 60 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| lfs: true | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Inspect git version | |
| run: | | |
| git --version | |
| - name: Check Docker Version | |
| run: docker version | |
| - name: Check Docker Compose Version | |
| run: docker compose version | |
| - name: Remove cloud-only modules and install Python client | |
| run: | | |
| set -e | |
| bash scripts/remove_cloud_modules.sh | |
| cd clients/python | |
| python -m pip install .[test] | |
| - name: Install ffmpeg | |
| run: | | |
| set -e | |
| sudo apt-get update -qq && sudo apt-get install ffmpeg libavcodec-extra -y | |
| - name: Authenticating to the Container registry | |
| run: echo $JH_PAT | docker login ghcr.io -u tanjiahuei@gmail.com --password-stdin | |
| env: | |
| JH_PAT: ${{ secrets.JH_PAT }} | |
| - name: Edit env file | |
| run: | | |
| set -e | |
| mv .env.example .env | |
| ORGS=$(printenv | grep API_KEY | xargs -I {} echo {} | cut -d '=' -f 1) | |
| KEYS=$(printenv | grep API_KEY | xargs -I {} echo {} | cut -d '=' -f 2-) | |
| # Convert them into arrays | |
| ORG_ARRAY=($ORGS) | |
| KEY_ARRAY=($KEYS) | |
| # Loop through the ORG_ARRAY | |
| for i in "${!ORG_ARRAY[@]}"; do | |
| # Get the org and key | |
| org="${ORG_ARRAY[$i]}" | |
| key="${KEY_ARRAY[$i]}" | |
| # Replace the org with the key in the .env file | |
| sed -i "s/$org=.*/$org=$key/g" .env | |
| done | |
| sed -i "s:EMBEDDING_MODEL=.*:EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2:g" .env | |
| sed -i "s:RERANKER_MODEL=.*:RERANKER_MODEL=cross-encoder/ms-marco-TinyBERT-L-2:g" .env | |
| echo 'OWL_MODELS_CONFIG=models_ci.json' >> .env | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} | |
| COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} | |
| HYPERBOLIC_API_KEY: ${{ secrets.HYPERBOLIC_API_KEY }} | |
| CUSTOM_API_KEY: ${{ secrets.CUSTOM_API_KEY }} | |
| - name: Launch services (OSS) | |
| id: launch_oss | |
| timeout-minutes: 20 | |
| run: | | |
| set -e | |
| docker compose -p jamai -f docker/compose.cpu.yml --profile minio --profile kopi up --quiet-pull -d --wait | |
| env: | |
| COMPOSE_DOCKER_CLI_BUILD: 1 | |
| DOCKER_BUILDKIT: 1 | |
| - name: Inspect owl Python version | |
| run: docker exec jamai-owl-1 python -V | |
| - name: Inspect owl environment | |
| run: docker exec jamai-owl-1 pip list | |
| - name: Python SDK tests (OSS) | |
| id: python_sdk_test_oss | |
| if: always() && steps.launch_oss.outcome == 'success' | |
| run: | | |
| set -e | |
| export JAMAI_API_BASE=http://localhost:6969/api | |
| python -m pytest -vv \ | |
| --timeout 300 \ | |
| --doctest-modules \ | |
| --junitxml=junit/test-results-${{ matrix.python-version }}.xml \ | |
| --cov-report=xml \ | |
| --no-flaky-report \ | |
| clients/python/tests/oss/ | |
| - name: Inspect owl logs if Python SDK tests failed | |
| if: failure() && steps.python_sdk_test_oss.outcome == 'failure' | |
| timeout-minutes: 1 | |
| run: docker exec jamai-owl-1 cat /app/api/logs/owl.log | |
| - name: Upload Pytest Test Results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pytest-results-${{ matrix.python-version }} | |
| path: junit/test-results-${{ matrix.python-version }}.xml | |
| # Always run this step to publish test results even when there are test failures | |
| if: always() | |
| - name: TS/JS SDK tests (OSS) | |
| id: ts_sdk_test_oss | |
| if: always() && steps.launch_oss.outcome == 'success' | |
| run: | | |
| cd clients/typescript | |
| echo "BASEURL=http://localhost:6969" >> __tests__/.env | |
| npm install | |
| npm run test | |
| - name: Inspect owl logs if TS/JS SDK tests failed | |
| if: failure() && steps.ts_sdk_test_oss.outcome == 'failure' | |
| timeout-minutes: 1 | |
| run: docker exec jamai-owl-1 cat /app/api/logs/owl.log | |
| - name: Update owl service for S3 test | |
| run: | | |
| # Update the .env file to include the new environment variable | |
| echo 'OWL_FILE_DIR=s3://file' >> .env | |
| echo 'S3_ENDPOINT=http://minio:9000' >> .env | |
| echo 'S3_ACCESS_KEY_ID=minioadmin' >> .env | |
| echo 'S3_SECRET_ACCESS_KEY=minioadmin' >> .env | |
| # Restart the owl service with the updated environment | |
| docker compose -p jamai -f docker/compose.cpu.yml up --quiet-pull -d --wait --no-deps --build --force-recreate owl | |
| - name: Python SDK tests (File API, OSS) | |
| id: python_sdk_test_oss_file | |
| if: always() && steps.launch_oss.outcome == 'success' | |
| run: | | |
| set -e | |
| export JAMAI_API_BASE=http://localhost:6969/api | |
| python -m pytest -vv \ | |
| --timeout 300 \ | |
| --doctest-modules \ | |
| --junitxml=junit/test-results-${{ matrix.python-version }}.xml \ | |
| --cov-report=xml \ | |
| --no-flaky-report \ | |
| clients/python/tests/oss/test_file.py | |
| lance_tests: | |
| name: Lance tests | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.12"] | |
| timeout-minutes: 60 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Inspect git version | |
| run: | | |
| git --version | |
| - name: Install owl | |
| run: | | |
| set -e | |
| cd services/api | |
| python -m pip install .[test] | |
| - name: Run tests | |
| run: pytest services/api/tests/test_lance.py |