new bucket for pipeline deploys (#929)

* new bucket for pipeline deploys

* Update dataproc vep init

* add to the build

Author: bpblanken

.cloudbuild/docker.cloudbuild.yaml

@@ -7,5 +7,6 @@ steps:
   - --dockerfile=v03_pipeline/deploy/Dockerfile
   - --cache=true
   - --cache-ttl=168h
+  - --build-arg=PIPELINE_RUNNER_APP_VERSION=${COMMIT_SHA}
 
 timeout: 1800s

.github/workflows/dev-release.yml

@@ -43,6 +43,6 @@ jobs:
 
       - name: Copy files to release directory
         run: |-
-          gcloud storage rm -r gs://seqr-luigi/releases/dev/latest/ || echo 'No latest release'
-          gcloud storage cp v03_pipeline/bin/* gs://seqr-luigi/releases/dev/latest/bin/
-          gcloud storage cp dist/*.whl gs://seqr-luigi/releases/dev/latest/pyscripts.zip
+          gcloud storage rm -r gs://seqr-pipeline-runner-builds/releases/dev/latest/ || echo 'No latest release'
+          gcloud storage cp v03_pipeline/bin/* gs://seqr-pipeline-runner-builds/releases/dev/latest/bin/
+          gcloud storage cp dist/*.whl gs://seqr-pipeline-runner-builds/releases/dev/latest/pyscripts.zip

.github/workflows/prod-release.yml

@@ -32,12 +32,6 @@ jobs:
           python -m pip install build
           python -m build
 
-      - name: set tag name
-        shell: bash
-        run: |
-          tag_name=`python3 -c "import v03_pipeline;print(v03_pipeline.__version__)"`
-          echo "TAG_NAME=$tag_name" >> $GITHUB_ENV
-
       - name: authenticate to google cloud
         id: "auth"
         uses: google-github-actions/auth@v0
@@ -51,26 +45,13 @@ jobs:
       - name: Copy files to release directory
         shell: bash
         run: |-
-          gcloud storage rm -r gs://seqr-luigi/releases/prod/latest/ || echo 'No latest release'
-          gcloud storage cp v03_pipeline/bin/* gs://seqr-luigi/releases/prod/latest/bin/
-          gcloud storage cp dist/*.whl gs://seqr-luigi/releases/prod/latest/pyscripts.zip
-          gcloud storage cp v03_pipeline/bin/* gs://seqr-luigi/releases/prod/$TAG_NAME/bin/
-          gcloud storage cp dist/*.whl gs://seqr-luigi/releases/prod/$TAG_NAME/pyscripts.zip
+          gcloud storage rm -r gs://seqr-pipeline-runner-builds/releases/latest/ || echo 'No latest release'
+          gcloud storage cp v03_pipeline/bin/* gs://seqr-pipeline-runner-builds/releases/latest/bin/
+          gcloud storage cp dist/*.whl gs://seqr-pipeline-runner-builds/releases/latest/pyscripts.zip
+          gcloud storage cp v03_pipeline/bin/* gs://seqr-pipeline-runner-builds/releases/${{ github.event.workflow_run.head_sha }}/bin/
+          gcloud storage cp dist/*.whl gs://seqr-pipeline-runner-builds/releases/${{ github.event.workflow_run.head_sha }}/pyscripts.zip
           gcloud storage cp -r v03_pipeline/var/vep/* gs://seqr-reference-data/vep/
 
-      - name: Create tag
-        uses: actions/github-script@v7
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            github.rest.git.createRef({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              ref: 'refs/tags/${{ env.TAG_NAME }}',
-              sha: context.sha
-            })
-
-
   docker:
     runs-on: ubuntu-latest
     if: ${{ github.event.workflow_run.conclusion == 'success' }}

v03_pipeline/bin/dataproc_vep_init.bash

@@ -16,6 +16,7 @@ set -x
 export PROJECT="$(gcloud config get-value project)"
 export ENVIRONMENT="$(/usr/share/google/get_metadata_value attributes/ENVIRONMENT)"
 export REFERENCE_GENOME="$(/usr/share/google/get_metadata_value attributes/REFERENCE_GENOME)"
+export PIPELINE_RUNNER_APP_VERSION="$(/usr/share/google/get_metadata_value attributes/PIPELINE_RUNNER_APP_VERSION)"
 
 # Install docker
 apt-get update
@@ -52,10 +53,10 @@ EOF
 gcc -Wall -Werror -O2 /vep.c -o /vep
 chmod u+s /vep
 
-gcloud storage cp gs://seqr-luigi/releases/$ENVIRONMENT/latest/bin/download_vep_reference_data.bash /download_vep_reference_data.bash
+gcloud storage cp gs://seqr-pipeline-runner-builds/releases/$ENVIRONMENT/$PIPELINE_RUNNER_APP_VERSION/bin/download_vep_reference_data.bash /download_vep_reference_data.bash
 chmod +x /download_vep_reference_data.bash
 ./download_vep_reference_data.bash $REFERENCE_GENOME
 
-gcloud storage cp gs://seqr-luigi/releases/$ENVIRONMENT/latest/bin/vep /vep.bash
+gcloud storage cp gs://seqr-pipeline-runner-builds/releases/$ENVIRONMENT/$PIPELINE_RUNNER_APP_VERSION/bin/vep /vep.bash
 chmod +x /vep.bash
 

v03_pipeline/deploy/Dockerfile

@@ -3,6 +3,9 @@ FROM hailgenetics/hail:0.2.132-py3.10
 COPY --from=BUILD /usr/local/bin/docker /usr/local/bin/docker
 LABEL maintainer="Broad TGG"
 
+ARG PIPELINE_RUNNER_APP_VERSION
+ENV PIPELINE_RUNNER_APP_VERSION=${PIPELINE_RUNNER_APP_VERSION}
+
 RUN curl -sSL https://sdk.cloud.google.com | bash
 ENV PATH $PATH:/root/google-cloud-sdk/bin