VEP 110 docker image and dataproc init script (#758)

* Add VEP docker image

* simplify

* bump version

* Add cloudbuild

* first pass

* a bit of cleanup

* ws

* ws

* A few tweaks

* twiddle options

* Bunch of config

* working!

* Update vep-GRCh38.json

* Update vep-110-GRCh38.sh

* missing slash

* more VEP

* some vep cleanup

* Remove genesplicer

Author: bpblanken

.cloudbuild/vep-docker.cloudbuild.yaml

@@ -0,0 +1,12 @@
+# Run locally with:
+#
+# gcloud builds submit --quiet --substitutions='_VEP_VERSION=110' --config .cloudbuild/vep-docker.cloudbuild.yaml v03_pipeline/
+steps:
+- name: 'gcr.io/kaniko-project/executor:v1.3.0'
+  args:
+  - --destination=gcr.io/seqr-project/vep-docker-image:${_VEP_VERSION}
+  - --dockerfile=deploy/Dockerfile.vep
+  - --cache=true
+  - --cache-ttl=168h
+
+timeout: 1800s

pyproject.toml

@@ -11,7 +11,7 @@ version = {attr = "v03_pipeline.__version__"}
 
 [tool.setuptools.packages.find]
 include = ["v03_pipeline*"]
-exclude = ["v03_pipeline.bin", "v03_pipeline*test*"]
+exclude = ["v03_pipeline.bin", "v03_pipeline.deploy", "v03_pipeline*test*"]
 namespaces = false
 
 [tool.mypy]

v03_pipeline/bin/vep-110-GRCh38.sh

@@ -0,0 +1,90 @@
+#
+# VEP init action for dataproc
+#
+# adapted/copied from
+# https://github.com/broadinstitute/gnomad_methods/blob/main/init_scripts/vep105-init.sh
+# and gs://hail-common/hailctl/dataproc/0.2.128/vep-GRCh38.sh
+#
+
+set -x
+
+export PROJECT="$(gcloud config get-value project)"
+export VEP_CONFIG_PATH="$(/usr/share/google/get_metadata_value attributes/VEP_CONFIG_PATH)"
+export VEP_REPLICATE="$(/usr/share/google/get_metadata_value attributes/VEP_REPLICATE)"
+export ASSEMBLY=GRCh38
+export VEP_DOCKER_IMAGE=gcr.io/seqr-project/vep-docker-image
+
+mkdir -p /vep_data
+
+# Install docker
+apt-get update
+apt-get -y install \
+    apt-transport-https \
+    ca-certificates \
+    curl \
+    gnupg2 \
+    software-properties-common \
+    tabix
+curl -fsSL https://download.docker.com/linux/debian/gpg | sudo apt-key add -
+sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable"
+sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable"
+apt-get update
+apt-get install -y --allow-unauthenticated docker-ce
+
+# https://github.com/hail-is/hail/issues/12936
+sleep 60
+sudo service docker restart
+
+# Copied from the repo at v03_pipeline/var/vep_config
+gcloud storage cp --billing-project $PROJECT gs://seqr-reference-data/vep/110/vep-${ASSEMBLY}.json $VEP_CONFIG_PATH
+
+# Copied from the UTRAnnotator repo (https://github.com/ImperialCardioGenetics/UTRannotator/tree/master)
+gcloud storage cp --billing-project $PROJECT gs://seqr-reference-data/vep/110/uORF_5UTR_${ASSEMBLY}_PUBLIC.txt /vep_data/ &
+
+# Raw data files copied from the bucket (https://console.cloud.google.com/storage/browser/dm_alphamissense;tab=objects?prefix=&forceOnObjectsSortingFiltering=false)
+# Some investigation led us to want to combine the canonical and non-canonical transcript tsvs (run inside the VEP docker container):
+# cat AlphaMissense_hg38.tsv.gz | gunzip | grep -v '#' | awk 'BEGIN { OFS = "\t" };{$6=""; print $0}' > AlphaMissense_combined_hg38.tsv
+# cat AlphaMissense_isoforms_hg38.tsv.gz | gunzip | grep -v '#' >> AlphaMissense_combined_hg38.tsv
+# cat AlphaMissense_combined_hg38.tsv | sort --parallel=12 --buffer-size=20G -k1,1 -k2,2n > AlphaMissense_combined_sorted_hg38.tsv
+# cat AlphaMissense_combined_sorted_hg38.tsv | sed '1i #CHROM\tPOS\tREF\tALT\tgenome\ttranscript_id\tprotein_variant\tam_pathogenicity\tam_class' > AlphaMissense_hg38.tsv
+# bgzip AlphaMissense_hg38.tsv
+# tabix -s 1 -b 2 -e 2 -f -S 1 AlphaMissense_hg38.tsv.gz
+gcloud storage cp --billing-project $PROJECT 'gs://seqr-reference-data/vep/110/AlphaMissense_hg38.tsv.*' /vep_data/ &
+
+gcloud storage cat --billing-project $PROJECT gs://seqr-reference-data/vep_data/loftee-beta/${ASSEMBLY}.tar | tar -xf - -C /vep_data/ &
+
+# Copied from ftp://ftp.ensembl.org/pub/release-110/variation/indexed_vep_cache/homo_sapiens_merged_vep_110_${ASSEMBLY}.tar.gz
+gcloud storage cat --billing-project $PROJECT gs://seqr-reference-data/vep/110/homo_sapiens_vep_110_${ASSEMBLY}.tar.gz | tar -xzf - -C /vep_data/ &
+
+# Generated with:
+# curl -O ftp://ftp.ensembl.org/pub/release-110/fasta/homo_sapiens/dna/Homo_sapiens.${ASSEMBLY}.dna.primary_assembly.fa.gz > Homo_sapiens.${ASSEMBLY}.dna.primary_assembly.fa.gz
+# gzip -d Homo_sapiens.${ASSEMBLY}.dna.primary_assembly.fa.gz
+# bgzip Homo_sapiens.${ASSEMBLY}.dna.primary_assembly.fa
+# samtools faidx Homo_sapiens.${ASSEMBLY}.dna.primary_assembly.fa.gz
+gcloud storage cp --billing-project $PROJECT 'gs://seqr-reference-data/vep/110/Homo_sapiens.${ASSEMBLY}.dna.primary_assembly.fa.*' /vep_data/ &
+docker pull ${VEP_DOCKER_IMAGE} &
+wait
+
+cat >/vep.c <<EOF
+#include <unistd.h>
+#include <stdio.h>
+
+int
+main(int argc, char *const argv[]) {
+  if (setuid(geteuid()))
+    perror( "setuid" );
+
+  execv("/vep.sh", argv);
+  return 0;
+}
+EOF
+gcc -Wall -Werror -O2 /vep.c -o /vep
+chmod u+s /vep
+
+cat >/vep.sh <<EOF
+#!/bin/bash
+
+docker run -i -v /vep_data/:/opt/vep/.vep/:ro ${VEP_DOCKER_IMAGE} \
+  /opt/vep/src/ensembl-vep/vep "\$@"
+EOF
+chmod +x /vep.sh

v03_pipeline/deploy/Dockerfile.vep

@@ -0,0 +1,13 @@
+FROM ubuntu:18.04 as build
+
+# Adapted from https://hub.docker.com/layers/konradjk/vep95_loftee/latest/images/sha256-d5f1a155293412acb5af4811142ba6907bad1cd708ca4000528f6317b784440e?context=explore
+# and https://github.com/broadinstitute/gnomad_methods/blob/main/docker_files/Dockerfile_VEP105
+RUN apt-get update && apt-get -y install wget libncurses5-dev libncursesw5-dev libbz2-dev liblzma-dev build-essential libz-dev git
+RUN wget https://github.com/samtools/samtools/releases/download/1.7/samtools-1.7.tar.bz2 && tar xjvf samtools-1.7.tar.bz2 && cd samtools-1.7 && make && make install
+RUN git clone -b grch38 https://github.com/konradjk/loftee.git
+
+FROM ensemblorg/ensembl-vep:release_110.1 as runtime
+RUN cpanm DBD::SQLite
+COPY --from=build /usr/local/bin/samtools /usr/local/bin/samtools
+# semantics of mv vs COPY are different such that we don't need the '*' when moving files.
+COPY --from=build /loftee/ /plugins

v03_pipeline/lib/vep.py

@@ -24,4 +24,5 @@ def run_vep(
         name='vep',
         block_size=1000,
         tolerate_parse_error=True,
+        csq=False,
     )

v03_pipeline/var/vep_config/vep-GRCh38.json

@@ -0,0 +1,11 @@
+{
+  "command": [
+    "bash",
+    "-c",
+    "/vep --format vcf -json --hgvs --biotype --canonical --mane --numbers --minimal --allele_number --no_stats --cache --offline --assembly GRCh38 --fasta /opt/vep/.vep/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz --plugin LoF,loftee_path:/plugins,gerp_bigwig:/opt/vep/.vep/gerp_conservation_scores.homo_sapiens.GRCh38.bw,human_ancestor_fa:/opt/vep/.vep/human_ancestor.fa.gz,conservation_file:/opt/vep/.vep/loftee.sql --plugin UTRAnnotator,file=/opt/vep/.vep/uORF_5UTR_GRCh38_PUBLIC.txt --plugin SpliceRegion,Extended --plugin AlphaMissense,file=/opt/vep/.vep/AlphaMissense_hg38.tsv.gz --dir_plugins /plugins -o STDOUT | sed s/5utr/fiveutr/g"
+  ],
+  "env": {
+    "PERL5LIB": "/plugins"
+  },
+  "vep_json_schema": "Struct{allele_string:String,end:Int32,id:String,input:String,intergenic_consequences:Array[Struct{allele_num:Int32,ancestral:String,consequence_terms:Array[String],context:String,impact:String,variant_allele:String}],minimised:Int32,most_severe_consequence:String,seq_region_name:String,start:Int32,strand:Int32,transcript_consequences:Array[Struct{allele_num:Int32,amino_acids:String,biotype:String,canonical:Int32,cdna_end:Int32,cdna_start:Int32,cds_end:Int32,cds_start:Int32,codons:String,consequence_terms:Array[String],distance:Int32,flags:String,gene_id:String,hgvsc:String,hgvsp:String,hgvs_offset:Int32,impact:String,mane_select:String,mane_plus_clinical:String,protein_end:Int32,protein_start:Int32,strand:Int32,transcript_id:String,variant_allele:String,lof:String,lof_flags:String,lof_filter:String,lof_info:String,existing_inframe_oorfs: Int32,existing_outofframe_oorfs: Int32,existing_uorfs: Int32,fiveutr_consequence: String,fiveutr_annotation: Dict[String, Dict[String, String]],spliceregion: Array[String],am_pathogenicity: Float32,am_class: String}]}"
+}