diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3e66f4fb0..848c1e6dc 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -25,10 +25,12 @@ permissions: jobs: build: runs-on: ubuntu-latest + outputs: + hashes: ${{ steps.hash.outputs.hashes }} steps: - uses: actions/checkout@v4 with: - token: ${{ secrets.ACTIONS_ACCESS_TOKEN }} + # token: ${{ secrets.ACTIONS_ACCESS_TOKEN }} ref: ${{ inputs.tag }} - name: Install the latest version of rye uses: eifinger/setup-rye@v2 @@ -43,113 +45,152 @@ jobs: run: | rye sync rye build + - name: "Generate hashes" + id: hash + run: | + cd dist && echo "hashes=$(sha256sum * | base64 -w0)" >> $GITHUB_OUTPUT - uses: actions/upload-artifact@v4 with: name: build path: ./dist - test-build: - if: ${{ !inputs.skip-tests }} - needs: ['build'] + provenance_python: + needs: [build] + permissions: + actions: read + contents: write + id-token: write # Needed to access the workflow's OIDC identity. + uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v2.0.0 + with: + base64-subjects: "${{ needs.build.outputs.hashes }}" + upload-assets: true # upload to a new release + upload-tag-name: ${{ inputs.tag }} # Tag from the initiation of the workflow + + # test-build: + # if: ${{ !inputs.skip-tests }} + # needs: ['build'] + # runs-on: ubuntu-latest + # strategy: + # fail-fast: false + # matrix: + # include: + # - python-version: 3.8 + # prod-key: PROD_LABELBOX_API_KEY_2 + # da-test-key: DA_GCP_LABELBOX_API_KEY + # - python-version: 3.9 + # prod-key: PROD_LABELBOX_API_KEY_3 + # da-test-key: DA_GCP_LABELBOX_API_KEY + # - python-version: "3.10" + # prod-key: PROD_LABELBOX_API_KEY_4 + # da-test-key: DA_GCP_LABELBOX_API_KEY + # - python-version: 3.11 + # prod-key: LABELBOX_API_KEY + # da-test-key: DA_GCP_LABELBOX_API_KEY + # - python-version: 3.12 + # prod-key: PROD_LABELBOX_API_KEY_5 + # da-test-key: DA_GCP_LABELBOX_API_KEY + # steps: + # - uses: actions/checkout@v4 + # with: + # # token: ${{ secrets.ACTIONS_ACCESS_TOKEN }} + # ref: ${{ inputs.tag }} + # - name: Install the latest version of rye + # uses: eifinger/setup-rye@v2 + # with: + # version: ${{ vars.RYE_VERSION }} + # enable-cache: true + # - name: Rye Setup + # run: | + # rye config --set-bool behavior.use-uv=true + # - name: Python setup + # run: rye pin ${{ matrix.python-version }} + # - uses: actions/download-artifact@v4 + # with: + # name: build + # path: ./dist + # - name: Prepare package and environment + # run: | + # rye sync -f --update-all + # rye run toml unset --toml-path pyproject.toml tool.rye.workspace + # rye sync -f --update-all + # - name: Integration Testing + # env: + # PYTEST_XDIST_AUTO_NUM_WORKERS: 32 + # LABELBOX_TEST_API_KEY: ${{ secrets[matrix.prod-key] }} + # DA_GCP_LABELBOX_API_KEY: ${{ secrets[matrix.da-test-key] }} + # LABELBOX_TEST_ENVIRON: prod + # run: | + # rye add labelbox --path ./$(find ./dist/ -name *.tar.gz) --sync --absolute + # cd libs/labelbox + # rm pyproject.toml + # rye run pytest tests/integration + # - name: Data Testing + # env: + # PYTEST_XDIST_AUTO_NUM_WORKERS: 32 + # LABELBOX_TEST_API_KEY: ${{ secrets[matrix.prod-key] }} + # DA_GCP_LABELBOX_API_KEY: ${{ secrets[matrix.da-test-key] }} + # LABELBOX_TEST_ENVIRON: prod + # run: | + # rye add labelbox --path ./$(find ./dist/ -name *.tar.gz) --sync --absolute --features data + # cd libs/labelbox + # rye run pytest tests/data + publish-python-package-to-release: runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - python-version: 3.8 - prod-key: PROD_LABELBOX_API_KEY_2 - da-test-key: DA_GCP_LABELBOX_API_KEY - - python-version: 3.9 - prod-key: PROD_LABELBOX_API_KEY_3 - da-test-key: DA_GCP_LABELBOX_API_KEY - - python-version: "3.10" - prod-key: PROD_LABELBOX_API_KEY_4 - da-test-key: DA_GCP_LABELBOX_API_KEY - - python-version: 3.11 - prod-key: LABELBOX_API_KEY - da-test-key: DA_GCP_LABELBOX_API_KEY - - python-version: 3.12 - prod-key: PROD_LABELBOX_API_KEY_5 - da-test-key: DA_GCP_LABELBOX_API_KEY + needs: ['build'] + permissions: + contents: write steps: - uses: actions/checkout@v4 with: - token: ${{ secrets.ACTIONS_ACCESS_TOKEN }} ref: ${{ inputs.tag }} - - name: Install the latest version of rye - uses: eifinger/setup-rye@v2 - with: - version: ${{ vars.RYE_VERSION }} - enable-cache: true - - name: Rye Setup - run: | - rye config --set-bool behavior.use-uv=true - - name: Python setup - run: rye pin ${{ matrix.python-version }} - uses: actions/download-artifact@v4 with: name: build - path: ./dist - - name: Prepare package and environment - run: | - rye sync -f --update-all - rye run toml unset --toml-path pyproject.toml tool.rye.workspace - rye sync -f --update-all - - name: Integration Testing - env: - PYTEST_XDIST_AUTO_NUM_WORKERS: 32 - LABELBOX_TEST_API_KEY: ${{ secrets[matrix.prod-key] }} - DA_GCP_LABELBOX_API_KEY: ${{ secrets[matrix.da-test-key] }} - LABELBOX_TEST_ENVIRON: prod + path: ./artifact + - name: Upload dist to release run: | - rye add labelbox --path ./$(find ./dist/ -name *.tar.gz) --sync --absolute - cd libs/labelbox - rm pyproject.toml - rye run pytest tests/integration - - name: Data Testing + gh release upload ${{ inputs.tag }} ./artifact/* env: - PYTEST_XDIST_AUTO_NUM_WORKERS: 32 - LABELBOX_TEST_API_KEY: ${{ secrets[matrix.prod-key] }} - DA_GCP_LABELBOX_API_KEY: ${{ secrets[matrix.da-test-key] }} - LABELBOX_TEST_ENVIRON: prod - run: | - rye add labelbox --path ./$(find ./dist/ -name *.tar.gz) --sync --absolute --features data - cd libs/labelbox - rye run pytest tests/data - pypi-publish: - runs-on: ubuntu-latest - needs: ['build', 'test-build'] - if: | - always() && - (needs.test-build.result == 'success' || needs.test-build.result == 'skipped') && github.event.inputs.tag - environment: - name: publish - url: 'https://pypi.org/project/labelbox/' - permissions: - # IMPORTANT: this permission is mandatory for trusted publishing - id-token: write - steps: - - uses: actions/download-artifact@v4 - with: - name: build - path: ./artifact - - name: Publish package distributions to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages-dir: artifact/ + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # pypi-publish: +# runs-on: ubuntu-latest +# needs: ['build', 'test-build'] +# if: | +# always() && +# (needs.test-build.result == 'success' || needs.test-build.result == 'skipped') && github.event.inputs.tag +# environment: +# name: publish +# url: 'https://pypi.org/project/labelbox/' +# permissions: +# # IMPORTANT: this permission is mandatory for trusted publishing +# id-token: write +# steps: +# - uses: actions/download-artifact@v4 +# with: +# name: build +# path: ./artifact +# - name: Publish package distributions to PyPI +# uses: pypa/gh-action-pypi-publish@release/v1 +# with: +# packages-dir: artifact/ container-publish: runs-on: ubuntu-latest - needs: ['build', 'test-build'] + needs: ['build'] + # needs: ['build', 'test-build'] permissions: packages: write - if: | - always() && - (needs.test-build.result == 'success' || needs.test-build.result == 'skipped') && github.event.inputs.tag + outputs: + image: ${{ steps.image.outputs.image }} + digest: ${{ steps.build_container.outputs.digest }} + # if: | + # always() && + # (needs.test-build.result == 'success' || needs.test-build.result == 'skipped') && github.event.inputs.tag env: CONTAINER_IMAGE: "ghcr.io/${{ github.repository }}" steps: - uses: actions/checkout@v4 with: - token: ${{ secrets.ACTIONS_ACCESS_TOKEN }} + # token: ${{ secrets.ACTIONS_ACCESS_TOKEN }} ref: ${{ inputs.tag }} - name: downcase CONTAINER_IMAGE @@ -168,6 +209,7 @@ jobs: - name: Build and push uses: docker/build-push-action@v5 + id: build_container with: context: . file: ./libs/labelbox/Dockerfile @@ -181,5 +223,24 @@ jobs: tags: | ${{ env.CONTAINER_IMAGE }}:latest ${{ env.CONTAINER_IMAGE }}:${{ inputs.tag }} - -# Note that the build and pypi-publish jobs are split so that the additional permissions are only granted to the pypi-publish job. \ No newline at end of file + - name: Output image + id: image + run: | + # NOTE: Set the image as an output because the `env` context is not + # available to the inputs of a reusable workflow call. + image_name="${CONTAINER_IMAGE}" + echo "image=$image_name" >> "$GITHUB_OUTPUT" + + provenance_container: + needs: [container-publish] + permissions: + actions: read # for detecting the Github Actions environment. + id-token: write # for creating OIDC tokens for signing. + packages: write # for uploading attestations. + uses: slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v2.0.0 + with: + image: ${{ needs. container-publish.outputs.image }} + digest: ${{ needs. container-publish.outputs.digest }} + registry-username: ${{ github.actor }} + secrets: + registry-password: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/python-package-develop.yml b/.github/workflows/python-package-develop.yml deleted file mode 100644 index cd5110033..000000000 --- a/.github/workflows/python-package-develop.yml +++ /dev/null @@ -1,136 +0,0 @@ -name: Labelbox Python SDK Staging (Develop) - -on: - push: - branches: [develop] - pull_request: - branches: [develop] - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -permissions: - id-token: write - -jobs: - build: - strategy: - fail-fast: false - matrix: - include: - - python-version: 3.8 - api-key: STAGING_LABELBOX_API_KEY_2 - da-test-key: DA_GCP_LABELBOX_API_KEY - - python-version: 3.9 - api-key: STAGING_LABELBOX_API_KEY_3 - da-test-key: DA_GCP_LABELBOX_API_KEY - - python-version: "3.10" - api-key: STAGING_LABELBOX_API_KEY_4 - da-test-key: DA_GCP_LABELBOX_API_KEY - - python-version: 3.11 - api-key: STAGING_LABELBOX_API_KEY - da-test-key: DA_GCP_LABELBOX_API_KEY - - python-version: 3.12 - api-key: STAGING_LABELBOX_API_KEY_5 - da-test-key: DA_GCP_LABELBOX_API_KEY - uses: ./.github/workflows/python-package-shared.yml - with: - python-version: ${{ matrix.python-version }} - api-key: ${{ matrix.api-key }} - da-test-key: ${{ matrix.da-test-key }} - fixture-profile: true - test-env: 'staging' - secrets: inherit - test-pypi: - runs-on: ubuntu-latest - needs: ['build'] - environment: - name: Test-PyPI - url: 'https://test.pypi.org/p/labelbox-test' - permissions: - # IMPORTANT: this permission is mandatory for trusted publishing - id-token: write - steps: - - uses: actions/checkout@v4 - with: - token: ${{ secrets.ACTIONS_ACCESS_TOKEN }} - ref: ${{ github.head_ref }} - - uses: ./.github/actions/python-package-shared-setup - with: - rye-version: ${{ vars.RYE_VERSION }} - python-version: '3.8' - - name: Create build - id: create-build - working-directory: libs/labelbox - run: | - VERSION=$(date +"%Y.%m.%d.%H.%M") - echo "pip install --index-url https://test.pypi.org/simple/ --extra-index-url=https://pypi.org/simple/ labelbox-test@$VERSION" >> "$GITHUB_STEP_SUMMARY" - rye version "$VERSION" - rye run toml set --toml-path pyproject.toml project.name labelbox-test - rye build - - name: Publish package distributions to Test PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages-dir: dist/ - repository-url: https://test.pypi.org/legacy/ - test-container: - runs-on: ubuntu-latest - needs: ['build'] - permissions: - # IMPORTANT: this permission is mandatory for trusted publishing - packages: write - env: - CONTAINER_IMAGE: "ghcr.io/${{ github.repository }}" - steps: - - uses: actions/checkout@v4 - with: - token: ${{ secrets.ACTIONS_ACCESS_TOKEN }} - ref: ${{ github.head_ref }} - - - name: downcase CONTAINER_IMAGE - run: | - echo "CONTAINER_IMAGE=${CONTAINER_IMAGE,,}" >> ${GITHUB_ENV} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push (Develop) - if: github.event_name == 'push' - uses: docker/build-push-action@v5 - with: - context: . - file: ./libs/labelbox/Dockerfile - github-token: ${{ secrets.GITHUB_TOKEN }} - push: true - - platforms: | - linux/amd64 - linux/arm64 - - tags: | - ${{ env.CONTAINER_IMAGE }}:develop - ${{ env.CONTAINER_IMAGE }}:${{ github.sha }} - - - name: Build and push (Pull Request) - if: github.event_name == 'pull_request' - uses: docker/build-push-action@v5 - with: - context: . - file: ./libs/labelbox/Dockerfile - github-token: ${{ secrets.GITHUB_TOKEN }} - push: true - - platforms: | - linux/amd64 - linux/arm64 - - tags: | - ${{ env.CONTAINER_IMAGE }}:${{ github.sha }} diff --git a/.github/workflows/upload_download_experiment.yml b/.github/workflows/upload_download_experiment.yml new file mode 100644 index 000000000..cef141cba --- /dev/null +++ b/.github/workflows/upload_download_experiment.yml @@ -0,0 +1,62 @@ + +on: +# pull_request: + workflow_dispatch: + inputs: + tag: + description: 'Release Tag' + required: true + +jobs: + build: + runs-on: ubuntu-latest + outputs: + hashes: ${{ steps.hash.outputs.hashes }} + steps: + - uses: actions/checkout@v4 + with: + # token: ${{ secrets.ACTIONS_ACCESS_TOKEN }} + ref: ${{ inputs.tag }} + - name: Install the latest version of rye + uses: eifinger/setup-rye@v2 + with: + version: ${{ vars.RYE_VERSION }} + enable-cache: true + - name: Rye Setup + run: | + rye config --set-bool behavior.use-uv=true + - name: Create build + working-directory: libs/labelbox + run: | + rye sync + rye build + - name: "Generate hashes" + id: hash + run: | + cd dist && echo "hashes=$(sha256sum * | base64 -w0)" >> $GITHUB_OUTPUT + - uses: actions/upload-artifact@v4 + with: + name: build + path: ./dist + + + publish-python-package-to-release: + runs-on: ubuntu-latest + needs: ['build'] + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.tag }} + - uses: actions/download-artifact@v4 + with: + name: build + path: ./artifact + - name: List artifact contents + run: ls -R ./artifact + - name: Upload dist to release + run: | + gh release upload ${{ inputs.tag }} ./artifact/* + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/README.md b/README.md index 634defa73..d9519ea01 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ [![Twitter Follow](https://img.shields.io/twitter/follow/labelbox.svg?style=social&label=Follow)](https://twitter.com/labelbox) [![LinkedIn Follow](https://img.shields.io/badge/Follow-LinkedIn-blue.svg?style=flat&logo=linkedin)](https://www.linkedin.com/company/labelbox/) [![Supported Python Versions](https://img.shields.io/pypi/pyversions/labelbox)](https://img.shields.io/pypi/pyversions/labelbox) +[![SLSA 3](https://slsa.dev/images/gh-badge-level3.svg)](https://slsa.dev) # Labelbox @@ -94,4 +95,27 @@ c.InteractiveShellApp.exec_lines = [ 'import sys; sys.path.insert(0, "")' ] ``` -4. Go to the root of your project and run `jupyter notebook` to start the server. \ No newline at end of file +4. Go to the root of your project and run `jupyter notebook` to start the server. + +## Provenance + +To enhance the software supply chain security of Labelbox's users, as of v3.73.0, every Labelbox SDK release contains a [SLSA Level 3 Provenance](https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/generic/README.md) document. +The provenance document refers to the Python package, as well as the generated Docker image. +You can use [SLSA framework's official verifier](https://github.com/slsa-framework/slsa-verifier) to verify the provenance. +Example of usage for the v3.73.0 release wheel: + +``` +pip download --no-deps labelbox==3.72.0 + +slsa-verifier verify-artifact --source-branch develop --builder-id 'https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@refs/tags/v2.0.0' --source-uri "git+https://github.com/Labelbox/labelbox-python-slsa-temp" --provenance-path multiple.intoto.jsonl ./labelbox-3.72.0-py3-none-any.whl +``` + +Example of usage for the v3.73.0 release docker image: +``` +Brew install crane +brew install slsa-verifier +IMAGE=ghcr.io/labelbox/labelbox-python-slsa-temp:6.5 +IMAGE="${IMAGE}@"$(crane digest "${IMAGE}") +slsa-verifier verify-image "$IMAGE" \ + --source-uri github.com/Labelbox/labelbox-python-slsa-temp +```