Feature/cleanup dockerfiles (#299)

* creating multi stage dockerfile, adjusting Makefile accordingly

* properly including ccache also in docker containers

* updating top level README with docker related things

.dockerignore

@@ -0,0 +1,11 @@
+**/.idea
+.git
+cmake-build*
+Dockerfile
+metagraph/CMakeFiles
+metagraph/build*
+metagraph/ccache_docker
+metagraph/cmake-build*
+metagraph/external-libaries/caches/*
+projects
+visualization

Dockerfile

@@ -0,0 +1,113 @@
+ARG CODE_BASE="/opt/metagraph"
+
+FROM ubuntu:20.04 AS metagraph_dev_env
+
+# contains all dependencies to build metagraph. Can also be used during development by mounting the code base and
+# build dir on the host (this is done in `make build-metagraph env=docker`)
+
+RUN export DEBIAN_FRONTEND="noninteractive" && apt-get update && apt-get install -y \
+    autoconf \
+    automake \
+    binutils-dev \
+    ccache \
+    cmake \
+    curl \
+    extra-cmake-modules \
+    g++-10 \
+    git \
+    libboost-all-dev \
+    libbrotli-dev \
+    libbz2-dev \
+    libdouble-conversion-dev \
+    libevent-dev \
+    libgflags-dev \
+    libgoogle-glog-dev \
+    libiberty-dev \
+    libjemalloc-dev \
+    liblz4-dev \
+    liblzma-dev \
+    libsnappy-dev \
+    libssl-dev \
+    libtool \
+    libunwind-dev \
+    make \
+    pkg-config \
+    python3 \
+    python3-pip \
+    python3-venv \
+    vim \
+    wget \
+    zlib1g-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN pip3 install parameterized==0.7.1
+
+WORKDIR /opt
+ENV LD_LIBRARY_PATH=/usr/local/lib
+ENV CC /usr/bin/gcc-10
+ENV CXX /usr/bin/g++-10
+
+### Compile our own libcurl (compiling from source to avoid linking issues in static builds)
+RUN git clone https://github.com/curl/curl.git \
+    && mkdir curl/_build \
+    && cd curl/_build \
+    && cmake -DBUILD_SHARED_LIBS=on .. \
+    && make -j $(nproc) \
+    && make install \
+    && cmake -DBUILD_SHARED_LIBS=off .. \
+    && make -j $(nproc) \
+    && make install \
+    && cd /opt && rm -rf curl
+
+### Installing htslib (compiling from source to avoid linking issues in static builds)
+ARG HTSLIB_VERSION=1.10.2
+RUN wget https://github.com/samtools/htslib/releases/download/$HTSLIB_VERSION/htslib-$HTSLIB_VERSION.tar.bz2 \
+    && tar -vxjf htslib-$HTSLIB_VERSION.tar.bz2 \
+    && cd htslib-$HTSLIB_VERSION \
+    && make -j $(nproc) \
+    && make install \
+    && cd /opt && rm -rf htslib-$HTSLIB_VERSION htslib-${HTSLIB_VERSION}.tar.bz2
+
+RUN mkdir -p /opt/metagraph/build_docker /opt/ccache_docker
+RUN chmod o+rwx /opt/metagraph /opt/ccache_docker
+
+ENV CCACHE_DIR=/opt/ccache_docker
+
+FROM metagraph_dev_env as metagraph_bin
+ARG CODE_BASE
+
+COPY . ${CODE_BASE}
+
+WORKDIR ${CODE_BASE}
+RUN make build-sdsl-lite \
+    && make build-metagraph alphabet=Protein \
+    && make build-metagraph alphabet=DNA
+
+FROM ubuntu:20.04
+ARG CODE_BASE
+
+# the image used in production. It contains a basic runtime environment for metagraph without build tools along with
+# the metagraph binary and python API code. This image is published on dockerhub (`ratschlab/metagraph`).
+
+RUN apt-get update && apt-get install -y \
+    libatomic1 \
+    libcurl4-nss-dev \
+    libgomp1 \
+    libhts-dev \
+    libjemalloc2 \
+    python3 \
+    python3-pip \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=metagraph_bin /opt/metagraph/metagraph/build/metagraph_* /usr/local/bin/
+
+RUN ln -s /usr/local/bin/metagraph_DNA /usr/local/bin/metagraph
+
+RUN mkdir ${CODE_BASE}
+COPY metagraph/api/python ${CODE_BASE}/api/python
+
+RUN pip3 install ${CODE_BASE}/api/python
+
+USER nobody
+
+ENTRYPOINT ["metagraph"]

Makefile

@@ -0,0 +1,112 @@
+IMG_NAME := metagraph # final image name
+IMG_NAME_DEV := metagraph_dev_env # development environment image name
+
+MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
+CODE_BASE_HOST := $(abspath $(MKFILE_PATH)/..)
+BUILD_DIR_HOST := $(CODE_BASE_HOST)/metagraph/build
+BUILD_DIR_STATIC_HOST := $(CODE_BASE_HOST)/metagraph/build_static
+
+
+# build dir on host when used with the build environment provided by metagraph_dev_env
+BUILD_DIR_HOST_DOCKER := $(CODE_BASE_HOST)/metagraph/build_docker
+BUILD_DIR_STATIC_HOST_DOCKER := $(CODE_BASE_HOST)/metagraph/build_docker_static
+CCACHE_FOR_DOCKER := $(CODE_BASE_HOST)/metagraph/ccache_docker
+
+
+OS := $(shell uname -s)
+
+ifeq ($(OS), Darwin)
+DOCKER_GRP = 0
+else
+DOCKER_GRP = `stat -c '%g' /var/run/docker.sock`
+endif
+
+alphabet := $(or $(alphabet), DNA)
+
+additional_cmake_args := $(or $(cmake_args), -DBUILD_KMC=OFF)
+
+ifeq ($(env), docker)
+	CODE_BASE := /opt/metagraph
+	BUILD_DIR := /opt/metagraph_build
+	BUILD_DIR_STATIC := /opt/metagraph_build_static
+else
+	CODE_BASE := $(CODE_BASE_HOST)
+	BUILD_DIR := $(BUILD_DIR_HOST)
+	BUILD_DIR_STATIC := $(BUILD_DIR_STATIC_HOST)
+endif
+
+DATA_DIR := /mnt
+
+DOCKER_OPTS := -it -u `id -u ${USER}`:$(DOCKER_GRP) \
+			   -v $(BUILD_DIR_HOST_DOCKER):${BUILD_DIR} \
+               -v $(BUILD_DIR_STATIC_HOST_DOCKER):${BUILD_DIR_STATIC} \
+               -v $(CCACHE_FOR_DOCKER):/opt/ccache_docker \
+               -v  $(CODE_BASE_HOST):$(CODE_BASE) \
+               -v $(DATA_DIR_HOST):$(DATA_DIR)
+
+DOCKER_BASE_CMD := docker run --rm $(DOCKER_OPTS) $(IMG_NAME_DEV)
+
+ifeq ($(env), docker)
+ 	$(shell mkdir -p $(BUILD_DIR_HOST_DOCKER) $(BUILD_DIR_STATIC_HOST_DOCKER) )
+	EXEC_CMD := $(DOCKER_BASE_CMD) bash -c
+else
+	EXEC_CMD := bash -c
+endif
+
+all: update build test
+
+ifeq ($(env), docker)
+update: sync build-docker-dev-env
+else
+update: sync
+endif
+
+sync:
+	cd $(CODE_BASE_HOST) && git submodule sync && git submodule update --init --recursive
+
+## BUILDING BINARIES AND IMAGES
+
+build: build-sdsl-lite build-metagraph
+
+build-sdsl-lite:
+	$(EXEC_CMD) 'cd $(CODE_BASE)/metagraph/external-libraries/sdsl-lite && ./install.sh $${PWD}'
+
+
+
+build-metagraph:
+	[ -d $(BUILD_DIR_HOST_DOCKER) ] || mkdir -p $(BUILD_DIR_HOST_DOCKER)
+	[ -d $(CCACHE_FOR_DOCKER) ] || mkdir -p $(CCACHE_FOR_DOCKER)
+
+	$(EXEC_CMD) 'mkdir -p $(BUILD_DIR) && cd $(BUILD_DIR) \
+                   && cmake -DCMAKE_DBG_ALPHABET=$(alphabet) $(additional_cmake_args) $(CODE_BASE)/metagraph \
+                   && make metagraph -j $$(($$(getconf _NPROCESSORS_ONLN) - 1))'
+
+build-metagraph-static:
+	[ -d $(BUILD_DIR_STATIC_HOST_DOCKER) ] || mkdir -p $(BUILD_DIR_STATIC_HOST_DOCKER)
+	[ -d $(CCACHE_FOR_DOCKER) ] || mkdir -p $(CCACHE_FOR_DOCKER)
+	$(EXEC_CMD) 'mkdir -p $(BUILD_DIR_STATIC) && cd $(BUILD_DIR_STATIC) \
+                  && cmake -DCMAKE_DBG_ALPHABET=$(alphabet) -DBUILD_STATIC=ON $(additional_cmake_args) $(CODE_BASE)/metagraph \
+                  && make metagraph -j $$(($$(getconf _NPROCESSORS_ONLN) - 1))'
+
+build-docker:
+	docker build -t metagraph $(CODE_BASE_HOST)
+
+# explicitly generating intermediate stages
+build-docker-dev-env:
+	docker build --target metagraph_dev_env -t metagraph_dev_env $(CODE_BASE_HOST)
+
+build-docker-bin:
+	docker build --target metagraph_bin -t metagraph_bin $(CODE_BASE_HOST)
+
+
+# TESTING
+
+integration-tests:
+	$(EXEC_CMD) 'cd $(BUILD_DIR) && ./integration_tests'
+
+integration-tests-api:
+	$(EXEC_CMD) 'cd $(BUILD_DIR) && ./integration_tests --test_filter="test_api*"'
+
+
+
+

README.md

@@ -2,6 +2,22 @@
 
 ## Install
 
+### Docker
+
+If docker is available on your system, you can immediately get started using
+e.g.
+```
+docker run -v ${DATA_DIR_HOST}:/mnt ratschlab/metagraph build -v -k 10 /mnt/transcripts_1000.fa -o /mnt/transcripts_1000
+```
+
+where you'd need to replace `${DATA_DIR_HOST}` with a directory on the host system. This directory is then mapped 
+under `/mnt` in the container.
+
+See also section [Developing with Docker Images](#developing-with-docker-images)
+
+
+## Install From Sources
+
 ### Prerequisites
 - cmake 3.10 or higher
 - GNU GCC with C++17 (gcc-8.0.1 or higher), LLVM Clang (clang-7 or higher), or AppleClang (clang-1100.0.33.8 or higher)
@@ -229,6 +245,24 @@ Stats for both
 ./metagraph stats -a annotation.column.annodbg graph.dbg
 ```
 
+## Developer Notes
+
+### Makefile
+
+The `Makefile` in the top level source directory can be used to build and test `metagraph` more conveniently. The following
+arguments are supported:
+* `env`: environment in which to compile/run (`""`: on the host, `docker`: in a docker container)
+* `alphabet`: compile metagraph for a certain alphabet (e.g. `DNA` or `Protein`, default `DNA`)
+* `additional_cmake_args`: additional arguments to pass to cmake.
+
+Examples:
+
+```
+# compiles metagraph in a docker container for the `DNA` alphabet
+make build-metagraph env=docker alphabet=DNA
+```
+
+
 ## License
 Metagraph is distributed under the GPLv3 License (see LICENSE).
 Please find further information in the AUTHORS and COPYRIGHTS files.