Merge branch 'main' of https://github.com/broadinstitute/gnomad_qc in…

…to jg/v4_random_forest_annotation_ht # Conflicts: # gnomad_qc/v4/annotations/generate_variant_qc_annotations.py
broadinstitute · Aug 22, 2023 · 23e4a6a · 23e4a6a
2 parents ff13cb4 + 7c9f5df
commit 23e4a6a
Show file tree

Hide file tree

Showing 2 changed files with 111 additions and 3 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,7 +1,7 @@
 ---
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
       - id: check-yaml
       - id: end-of-file-fixer

diff --git a/gnomad_qc/v4/annotations/Dockerfile_vrs084_v4 b/gnomad_qc/v4/annotations/Dockerfile_vrs084_v4
@@ -1,6 +1,105 @@
-# Pull the VRS image we created for v3, add bgzip to it.
+############################################################################################################
+# This Dockerfile has three main changes compared to the original Dockerfile
+# (which was copied from tgg_methods repo to gnomad_qc/v3/annotations/Dockerfile_vrs084):
+# 1) Uses install GCS connector from broad.io
+# 2) Installs bgzip
+# 3) Puts gnomad & gnomad_qc dependencies inside the docker image
+############################################################################################################
+# Dockerfile for creating an image that can run the VRS Annotation Script inside of it
+# includes libpqdev and pythondev, plus psycopg2, ga4gh.vrs[extras] , and seqrepo
 
-FROM us-central1-docker.pkg.dev/broad-mpg-gnomad/ga4gh-vrs/marten_0615_vrs0_8_4
+FROM 'google/cloud-sdk:slim'
+
+RUN apt-get -y update && apt-get install -y --no-install-recommends \
+    apt-utils \
+    docker-compose \
+    g++ \
+    gawk \
+    less \
+    libbz2-dev \
+    libcurl4-openssl-dev \
+    liblzma-dev \
+    libncurses5-dev \
+    man-db \
+    pkg-config \
+    python3-venv \
+    software-properties-common \
+    unzip \
+    wget \
+    zlib1g-dev \
+    libpq-dev \
+    python-dev\
+    && \
+	# clean up apt cache
+    rm -rf /var/lib/apt/lists/*
+
+# Install Java 8 for hail
+RUN wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | apt-key add - \
+    && add-apt-repository -y https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ \
+    && apt-get update && apt-get install -y adoptopenjdk-8-hotspot
+
+RUN mkdir tools
+WORKDIR /tools
+
+
+# Install vcftools
+ENV VCFTOOLS_VERSION="0.1.16"
+RUN wget https://github.com/vcftools/vcftools/releases/download/v${VCFTOOLS_VERSION}/vcftools-${VCFTOOLS_VERSION}.tar.gz \
+    && tar xf vcftools-${VCFTOOLS_VERSION}.tar.gz \
+    && rm vcftools-${VCFTOOLS_VERSION}.tar.gz \
+	&& cd vcftools-${VCFTOOLS_VERSION} \
+	&& ./configure \
+	&& make \
+	&& make install \
+	&& make clean
+
+
+# Install python packages
+RUN apt-get update \
+	&& apt-get dist-upgrade -y \
+	&& apt-get install -y --no-install-recommends\
+    libc6-dev \
+    libffi-dev \
+    libgdbm-dev \
+    liblapack-dev \
+    liblapack3 \
+    libncursesw5-dev \
+    libopenblas-base \
+    libopenblas-dev \
+    libsqlite3-dev \
+    libssl-dev \
+    openssl \
+    python3-smbus \
+    python3 python3-pip
+
+# Upgrade pip to latest version
+RUN python3 -m pip install --upgrade pip
+
+# Install hail and other python libraries
+# Including packages needed for VRS annotation functionality
+# Also include version for ga4gh.vrs-python to be most recent and supporting their new variant schema
+# Please update the constant VRS_VERSION in vrs_annotation_batch.py accordingly when GA4GH_VRS_VERSION is changed
+ENV HAIL_VERSION="0.2.113"
+ENV GA4GH_VRS_VERSION="0.8.4"
+RUN python3 --version
+RUN python3 -m pip install \
+    wheel \
+    pypandoc \
+    hail==${HAIL_VERSION} \
+    scipy \
+    numpy \
+    pandas \
+    matplotlib \
+    seaborn \
+    ipython \
+    pybedtools \
+    dill \
+    psycopg2 \
+    ga4gh.vrs[extras]==${GA4GH_VRS_VERSION}  \
+    seqrepo \
+    click \
+    git+https://github.com/broadinstitute/gnomad_methods.git@main \
+    git+https://github.com/broadinstitute/gnomad_qc.git@main
 
 # Install bgzip
 ENV BGZIP_VERSION="1.17"
@@ -11,3 +110,12 @@ RUN wget https://github.com/samtools/htslib/releases/download/1.17/htslib-1.17.t
     && make \
     && make install \
     && export PATH=htslib-1.17/bin:$PATH
+
+# Install GCS Connector
+RUN curl -sSL broad.io/install-gcs-connector > install-gcs-connector.py && \
+    python3 install-gcs-connector.py --key-file-path /gsa-key/key.json
+
+# Copy in SeqRepo from local file structure
+COPY /usr/local/share/seqrepo/2018-11-26/ /tmp/local-seqrepo
+
+WORKDIR /home