From 3122b9ce6a03e28617be0c866ebc3f7a8b6e7d77 Mon Sep 17 00:00:00 2001 From: Paul Robello <probello@gmail.com> Date: Fri, 15 Mar 2024 14:56:30 -0700 Subject: [PATCH] added new env var PIP_EXTRA_REQUIREMENTS_TXT to allow for installing extra python packages into global GDC python env reordered some of the installs int GDC Dockerfile for better cache performance --- .dockerignore | 1 - Dockerfile | 95 ++++++++++++++++++++------------------- docker-compose.yml | 4 +- noop | 1 + readme.md | 1 + root/bin/requirements.txt | 5 ++- run-dev-container.sh | 18 +++++--- 7 files changed, 72 insertions(+), 53 deletions(-) create mode 100644 noop diff --git a/.dockerignore b/.dockerignore index 2b3fa05..df2fd7c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,5 @@ docker-compose.yml **/temp -**/tmp test **/*.ps1 **/.idea diff --git a/Dockerfile b/Dockerfile index 2ca562d..b198e1d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,24 +39,33 @@ COPY /etc/locale.gen /etc/locale.gen RUN LC_ALL=en_US.UTF-8 LC_CTYPE=en_US.UTF-8 LANG=en_US.UTF-8 locale-gen RUN mkdir -p /usr/local/share/.cache -# intstall python if requested -COPY /root/bin/requirements.txt /root/requirements.txt -ARG PYTHON_VERSION -RUN /bin/bash -c 'if [ -n "${PYTHON_VERSION}" ] ; then \ - apt-get install -fy python3-dev python3-openssl && \ - export PYENV_ROOT=/usr/local/pyenv && \ - curl https://pyenv.run | bash && \ - command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH" && \ - eval "$(pyenv init -)" && \ - eval "$(pyenv virtualenv-init -)" && \ - pyenv install -v $PYTHON_VERSION && \ - pyenv global $PYTHON_VERSION && \ - python -m pip install --upgrade pip && \ - pip install virtualenv pre-commit && \ - pip install -r /root/requirements.txt; \ -fi; \ -rm /root/requirements.txt; \ -' +RUN mkdir -p /usr/local/data +WORKDIR /usr/local/data + +#ARG DOCKER_VERSION +# install docker +RUN install -m 0755 -d /etc/apt/keyrings +RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg +RUN chmod a+r /etc/apt/keyrings/docker.gpg +RUN echo \ + "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ + tee /etc/apt/sources.list.d/docker.list > /dev/null +RUN apt-get update +RUN apt-get install -fy --fix-missing docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + +# Install websocat +RUN /bin/bash -c 'set -ex && \ + ARCH=`uname -m` && \ + PLATFORM=`uname -s | tr '[:upper:]' '[:lower:]'` && \ + if [ "$ARCH" = "x86_64" ]; then \ + echo "websocat x86_64" && \ + curl -L "https://github.com/vi/websocat/releases/download/v1.10.0/websocat.x86_64-unknown-linux-musl" -o /usr/local/bin/websocat;\ + else \ + echo "websocat assuming ARM" && \ + curl -L "https://github.com/vi/websocat/releases/download/v1.10.0/websocat.arm-unknown-linux-musleabi" -o /usr/local/bin/websocat;\ + fi; \ + chmod +x /usr/local/bin/websocat;' # install php if requested ARG PHP_VERSION @@ -102,34 +111,29 @@ RUN /bin/bash -c 'if [ -n "${RUST_VERSION}" ]; then \ fi; \ fi' - -RUN mkdir -p /usr/local/data -WORKDIR /usr/local/data - -#ARG DOCKER_VERSION -# install docker -RUN install -m 0755 -d /etc/apt/keyrings -RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg -RUN chmod a+r /etc/apt/keyrings/docker.gpg -RUN echo \ - "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ - "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ - tee /etc/apt/sources.list.d/docker.list > /dev/null -RUN apt-get update -RUN apt-get install -fy --fix-missing docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin - -# Install websocat -RUN /bin/bash -c 'set -ex && \ - ARCH=`uname -m` && \ - PLATFORM=`uname -s | tr '[:upper:]' '[:lower:]'` && \ - if [ "$ARCH" = "x86_64" ]; then \ - echo "websocat x86_64" && \ - curl -L "https://github.com/vi/websocat/releases/download/v1.10.0/websocat.x86_64-unknown-linux-musl" -o /usr/local/bin/websocat;\ - else \ - echo "websocat assuming ARM" && \ - curl -L "https://github.com/vi/websocat/releases/download/v1.10.0/websocat.arm-unknown-linux-musleabi" -o /usr/local/bin/websocat;\ +# intstall python if requested +RUN mkdir -p /build-tmp +COPY /root/bin/requirements.txt /build-tmp/requirements.txt +ARG PYTHON_VERSION +ARG PIP_EXTRA_REQUIREMENTS_TXT +ARG HOST_PROJECT_FOLDER_NAME +COPY tmp/$HOST_PROJECT_FOLDER_NAME-* /build-tmp/ +RUN /bin/bash -c 'if [ -n "${PYTHON_VERSION}" ] ; then \ + apt-get install -fy python3-dev python3-openssl && \ + export PYENV_ROOT=/usr/local/pyenv && \ + curl https://pyenv.run | bash && \ + command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH" && \ + eval "$(pyenv init -)" && \ + eval "$(pyenv virtualenv-init -)" && \ + pyenv install -v $PYTHON_VERSION && \ + pyenv global $PYTHON_VERSION && \ + python -m pip install --upgrade pip && \ + pip install -r /build-tmp/requirements.txt && \ + if [ -n "${PIP_EXTRA_REQUIREMENTS_TXT}" ]; then \ + pip install -r "/build-tmp/${HOST_PROJECT_FOLDER_NAME}-pip-extra-requirements.txt" || exit 1; \ fi; \ - chmod +x /usr/local/bin/websocat;' +fi; \ +' # Install AWS CLI and SSM plugin ARG USE_AWS @@ -261,6 +265,7 @@ WORKDIR /workspace ENV PHP_VERSION=$PHP_VERSION ENV USE_JAVA=$USE_JAVA ENV PYTHON_VERSION=$PYTHON_VERSION +ENV PIP_EXTRA_REQUIREMENTS_TXT=$PIP_EXTRA_REQUIREMENTS_TXT ENV GOLANG_VERSION=$GOLANG_VERSION ENV USE_DOT_NET=$USE_DOT_NET ENV USE_AWS=$USE_AWS diff --git a/docker-compose.yml b/docker-compose.yml index f505a55..b29cb27 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,7 @@ services: - USE_DOT_NET # install ubuntu:latest dotnet core packages - GOLANG_VERSION # install this golang version. blank for none. latest is 1.18 - PYTHON_VERSION # install this python version + - PIP_EXTRA_REQUIREMENTS_TXT # install extra pip requirements if python is installed - USE_AWS # install latest aws cli, ssm plugin, and ecr helper - NODE_VERSION # install this version of node. blank for none - RUST_VERSION # install this version of rust. blank for none @@ -31,6 +32,7 @@ services: # - DOCKER_COMPOSE_VERSION # install docker-compose version inside container - EXTRA_PACKAGES # any extra Ubuntu packages you want installed into the base container - AWS_VERSION # install AWS CLI version + - HOST_PROJECT_FOLDER_NAME environment: - ARCH - AWS_VERSION @@ -123,7 +125,7 @@ services: - PROXY_URL # http proxy to use. if USE_PROXY_HOST is enabled will be set to http://PROXY_CONTAINER_NAME:8080 - PROXY_URL_SSL # https proxy to use. if USE_PROXY_HOST is enabled will be set to https://PROXY_CONTAINER_NAME:8080 - PROXY_AUTO_EXPORT_ENV # auto export HTTP_PROXY and HTTPS_PROXY vars equal to PROXY_URL - - DEV_CONTAINER=1.10.12 # used to detect if running inside dev container + - DEV_CONTAINER=1.10.13 # used to detect if running inside dev container volumes: - /var/run/docker.sock:/var/run/docker.sock # allow container to interact with host docker - "${GDC_DIR}:/root/gdc-host" # mount gdc folder inside container to get access to compose files diff --git a/noop b/noop new file mode 100644 index 0000000..7eb02a7 --- /dev/null +++ b/noop @@ -0,0 +1 @@ +This file does nothing. Its purpose is to allow wildcard copy in dockerfile diff --git a/readme.md b/readme.md index 9a100af..ddc050d 100644 --- a/readme.md +++ b/readme.md @@ -143,6 +143,7 @@ These options control what packages / functionality are built into the container * USE_DOT_NET=no - installs .NET SDK v6.x. * USE_JAVA=no - installs headless openjdk v11.x. * PYTHON_VERSION=3.11 - installs Python v3.11, which is highest aws lambda supported python runtime +* PIP_EXTRA_REQUIREMENTS_TXT=<not set> - if set should be a name of a requirements pip file to be installed if python is installed. * USE_PRECOMMIT=no - installs git pre-commit hooks in repo if not already installed. Enables Python if not already enabled. * USE_AWS=yes - installs AWS CLI version specified by AWS_VERSION, SSM Plugin and EKS IAM auth helper as well as aws helper scripts and aliases. * AWS_VERSION=latest - installs specified AWS cli version if USE_AWS=yes. Defaults to latest. diff --git a/root/bin/requirements.txt b/root/bin/requirements.txt index ac75bf3..dea5e93 100644 --- a/root/bin/requirements.txt +++ b/root/bin/requirements.txt @@ -1,6 +1,9 @@ +virtualenv +pipenv +pre-commit boto3 click awscliv2 awscli-local terraform-local -pipenv + diff --git a/run-dev-container.sh b/run-dev-container.sh index fd701d3..5f30243 100755 --- a/run-dev-container.sh +++ b/run-dev-container.sh @@ -47,6 +47,12 @@ fi HOST_PROJECT_FOLDER_NAME="$(basename "$HOST_PROJECT_PATH")" export HOST_PROJECT_FOLDER_NAME +if [ ! -d "./tmp" ]; then + mkdir ./tmp +fi +rm -rf "$GDC_DIR/tmp/$HOST_PROJECT_FOLDER_NAME-*" +cp "$GDC_DIR/noop" "$GDC_DIR/tmp/$HOST_PROJECT_FOLDER_NAME-noop" + path1=$GDC_DIR path2=$HOST_PROJECT_PATH @@ -93,6 +99,13 @@ if [[ "$AWS_VERSION" = "latest" ]]; then fi fi +if [ -n "$PYTHON_VERSION" ]; then + if [ -n "$PIP_EXTRA_REQUIREMENTS_TXT" ]; then + export PIP_EXTRA_REQUIREMENTS_TXT + cp "$HOST_PROJECT_PATH/$PIP_EXTRA_REQUIREMENTS_TXT" "$GDC_DIR/tmp/$HOST_PROJECT_FOLDER_NAME-pip-extra-requirements.txt" || exit 1 + fi +fi + # Function to convert a string to a number in the range 10-200 convert_string_to_number() { local input_string=$1 @@ -211,11 +224,6 @@ if [ -n "$GDC_DNS_PRI_IP" ]; then COMPOSE_FILES="$COMPOSE_FILES -f dc-dns.yml" fi - -if [ ! -d "./tmp" ]; then - mkdir ./tmp -fi - # this is the stack name for compose COMPOSE_PROJECT_NAME="$GDC_NAME" COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME//[ -]/_}"