Optimize CI Testing: Eliminate Redundant Coverage Test Execution #197
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PR Checks | |
| on: | |
| pull_request: | |
| branches: [ main ] | |
| # Allow manual triggering for testing | |
| workflow_dispatch: | |
| # Required permissions for posting PR comments | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| jobs: | |
| lint: | |
| name: Lint and Type Check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python 3.13 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| allow-prereleases: true | |
| - name: Install UV | |
| run: | | |
| pip install uv | |
| uv --version | |
| - name: Setup environment | |
| run: | | |
| # UV will automatically create and manage the virtual environment | |
| # Just sync the project dependencies | |
| uv sync --extra dev | |
| - name: Run linting | |
| run: uv run ruff check . | |
| - name: Run formatting check | |
| run: uv run ruff format --check . | |
| - name: Run type checking | |
| run: uv run mypy src | |
| e2e-fast-tests: | |
| name: E2E Tests (Direct Tool Integration) | |
| runs-on: ubuntu-latest | |
| # Run in parallel with lint and unit tests (fast feedback) | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python 3.13 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| allow-prereleases: true | |
| - name: Install UV | |
| run: | | |
| pip install uv | |
| uv --version | |
| - name: Setup environment | |
| run: | | |
| # UV will automatically create and manage the virtual environment | |
| # Just sync the project dependencies | |
| uv sync --extra dev | |
| - name: Install sbctl | |
| run: | | |
| mkdir -p /tmp/sbctl && cd /tmp/sbctl | |
| curl -L -o sbctl.tar.gz "https://github.com/replicatedhq/sbctl/releases/latest/download/sbctl_linux_amd64.tar.gz" | |
| tar xzf sbctl.tar.gz | |
| chmod +x sbctl | |
| sudo mv sbctl /usr/local/bin/ | |
| cd / && rm -rf /tmp/sbctl | |
| sbctl --help | |
| - name: Run fast E2E tests (direct tool integration) | |
| run: uv run pytest tests/e2e/test_direct_tool_integration.py -v | |
| all-tests-coverage: | |
| name: All Tests with Coverage (Optimized) | |
| runs-on: ubuntu-latest | |
| # Run after lint passes, parallel with container tests (optimized for speed) | |
| # This job runs ALL tests once with coverage enabled: | |
| # - Eliminates duplicate test execution (functional tests no longer run separately) | |
| # - Fast-fail enabled (-x flag) to stop immediately on failure | |
| # - Saves ~5.5 minutes per PR compared to previous approach | |
| needs: [lint] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python 3.13 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| allow-prereleases: true | |
| - name: Install UV | |
| run: | | |
| pip install uv | |
| uv --version | |
| - name: Setup environment | |
| run: | | |
| uv sync --extra dev | |
| - name: Install sbctl | |
| run: | | |
| mkdir -p /tmp/sbctl && cd /tmp/sbctl | |
| curl -L -o sbctl.tar.gz "https://github.com/replicatedhq/sbctl/releases/latest/download/sbctl_linux_amd64.tar.gz" | |
| tar xzf sbctl.tar.gz | |
| chmod +x sbctl | |
| sudo mv sbctl /usr/local/bin/ | |
| cd / && rm -rf /tmp/sbctl | |
| sbctl --help | |
| - name: Run all tests with coverage (single execution) | |
| run: | | |
| # Run ALL tests together in one execution to get accurate coverage | |
| # -x flag: Exit immediately on first test failure (fast-fail behavior) | |
| uv run pytest tests/unit/ tests/integration/ tests/functional/ -x --cov=src --cov-report=xml:coverage-all.xml --cov-report=term -v | |
| - name: Generate individual coverage reports for comparison | |
| run: | | |
| # Generate unit-only coverage for comparison (reusing existing coverage data) | |
| uv run coverage report --precision=0 > coverage-all-report.txt | |
| # Extract total coverage | |
| all_coverage=$(grep TOTAL coverage-all-report.txt | awk '{print $4}' | sed 's/%//') | |
| echo "ALL_TESTS_COVERAGE=${all_coverage}" >> $GITHUB_ENV | |
| echo "$all_coverage" > all_coverage.txt | |
| # For individual metrics, we'll estimate based on the full run | |
| # This is more accurate than running tests separately | |
| echo "## 📊 Test Coverage Results" >> $GITHUB_STEP_SUMMARY | |
| echo "**All Tests Combined: ${all_coverage}%**" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "\`\`\`" >> $GITHUB_STEP_SUMMARY | |
| cat coverage-all-report.txt >> $GITHUB_STEP_SUMMARY | |
| echo "\`\`\`" >> $GITHUB_STEP_SUMMARY | |
| # Check coverage threshold (60% minimum for all tests) | |
| if [ "$all_coverage" -lt 60 ]; then | |
| echo "❌ Overall test coverage ${all_coverage}% is below minimum threshold of 60%" >> $GITHUB_STEP_SUMMARY | |
| echo "::error::Overall test coverage ${all_coverage}% is below minimum threshold of 60%" | |
| exit 1 | |
| else | |
| echo "✅ Overall test coverage ${all_coverage}% meets minimum threshold of 60%" >> $GITHUB_STEP_SUMMARY | |
| echo "::notice::Overall test coverage: ${all_coverage}%" | |
| fi | |
| - name: Upload coverage results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: all-tests-coverage | |
| path: | | |
| coverage-all.xml | |
| all_coverage.txt | |
| coverage-all-report.txt | |
| coverage-report: | |
| name: Coverage Report | |
| runs-on: ubuntu-latest | |
| needs: [all-tests-coverage] | |
| if: always() && needs.all-tests-coverage.result == 'success' | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python 3.13 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| allow-prereleases: true | |
| - name: Install UV | |
| run: | | |
| pip install uv | |
| uv --version | |
| - name: Setup environment | |
| run: | | |
| # UV will automatically create and manage the virtual environment | |
| # Just sync the project dependencies | |
| uv sync --extra dev | |
| - name: Download all tests coverage results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: all-tests-coverage | |
| path: ./ | |
| - name: Generate coverage summary | |
| run: | | |
| # Read the coverage value from the all-tests job | |
| all_tests_cov="N/A" | |
| if [ -f all_coverage.txt ]; then | |
| all_tests_cov=$(cat all_coverage.txt) | |
| fi | |
| # Generate simple, clear summary | |
| echo "## 📊 Test Coverage Report" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**Overall Coverage: ${all_tests_cov}%**" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "This represents the percentage of code covered by running all tests together." >> $GITHUB_STEP_SUMMARY | |
| echo "Threshold: ≥60%" >> $GITHUB_STEP_SUMMARY | |
| if [ "$all_tests_cov" != "N/A" ] && [ "$all_tests_cov" -ge 60 ]; then | |
| echo "Status: ✅ PASSING" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "Status: ❌ BELOW THRESHOLD" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| # Set environment variable for PR comment | |
| echo "ALL_TESTS_COVERAGE=${all_tests_cov}" >> $GITHUB_ENV | |
| - name: Post coverage comment on PR | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| # Get the overall coverage | |
| all_cov="${ALL_TESTS_COVERAGE:-N/A}" | |
| # Determine status | |
| coverage_status="❌" | |
| if [ "$all_cov" != "N/A" ] && [ "$all_cov" -ge 60 ]; then | |
| coverage_status="✅" | |
| fi | |
| # Create simple, clear coverage comment | |
| cat > coverage_comment.md << EOF | |
| ## 📊 Test Coverage Report | |
| **Overall Coverage: ${all_cov}%** ${coverage_status} | |
| This represents the percentage of source code covered when running all tests together. | |
| ### Coverage Details | |
| \`\`\` | |
| EOF | |
| # Add detailed coverage report if available | |
| if [ -f coverage-all-report.txt ]; then | |
| cat coverage-all-report.txt >> coverage_comment.md | |
| else | |
| echo "Overall coverage: ${all_cov}%" >> coverage_comment.md | |
| fi | |
| cat >> coverage_comment.md << EOF | |
| \`\`\` | |
| EOF | |
| # Post comment using GitHub CLI | |
| gh pr comment ${{ github.event.number }} --body-file coverage_comment.md | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| container-tests: | |
| name: Container Tests (Slow) | |
| runs-on: ubuntu-latest | |
| needs: [lint] # Run after lint, parallel with all-tests-coverage (optimized) | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python 3.13 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| allow-prereleases: true | |
| - name: Install UV | |
| run: | | |
| pip install uv | |
| uv --version | |
| - name: Setup environment | |
| run: | | |
| # UV will automatically create and manage the virtual environment | |
| # Just sync the project dependencies | |
| uv sync --extra dev | |
| - name: Install sbctl | |
| run: | | |
| mkdir -p /tmp/sbctl && cd /tmp/sbctl | |
| curl -L -o sbctl.tar.gz "https://github.com/replicatedhq/sbctl/releases/latest/download/sbctl_linux_amd64.tar.gz" | |
| tar xzf sbctl.tar.gz | |
| chmod +x sbctl | |
| sudo mv sbctl /usr/local/bin/ | |
| cd / && rm -rf /tmp/sbctl | |
| sbctl --help | |
| - name: Check Podman version | |
| run: podman --version | |
| - name: Verify melange/apko availability via containers | |
| run: | | |
| # Test that melange and apko work via containers (as used in our build system) | |
| # We run them in containers rather than installing binaries directly | |
| podman run --rm cgr.dev/chainguard/melange:latest version | |
| podman run --rm cgr.dev/chainguard/apko:latest version | |
| - name: Set file permissions | |
| run: | | |
| chmod +x scripts/build.sh | |
| chmod +x scripts/run.sh | |
| chmod +x scripts/run_tests.sh | |
| chmod +x scripts/setup_env.sh | |
| chmod +x scripts/generate_test_keys.sh | |
| - name: Verify melange and apko configs exist | |
| run: | | |
| if [ ! -f .melange.yaml ]; then | |
| echo "ERROR: .melange.yaml not found" | |
| exit 1 | |
| fi | |
| if [ ! -f apko.yaml ]; then | |
| echo "ERROR: apko.yaml not found" | |
| exit 1 | |
| fi | |
| echo "Configuration files found:" | |
| ls -la .melange.yaml apko.yaml | |
| - name: Run container tests (melange/apko) | |
| env: | |
| MELANGE_TEST_BUILD: "true" | |
| run: uv run pytest tests/e2e/ -m container -v |