added new env var PIP_EXTRA_REQUIREMENTS_TXT to allow for installing …

…extra python packages into global GDC python env

reordered some of the installs int GDC Dockerfile for better cache performance

.dockerignore

@@ -1,6 +1,5 @@
 docker-compose.yml
 **/temp
-**/tmp
 test
 **/*.ps1
 **/.idea

Dockerfile

@@ -39,24 +39,33 @@ COPY /etc/locale.gen /etc/locale.gen
 RUN LC_ALL=en_US.UTF-8 LC_CTYPE=en_US.UTF-8 LANG=en_US.UTF-8 locale-gen
 RUN mkdir -p /usr/local/share/.cache
 
-# intstall python if requested
-COPY /root/bin/requirements.txt /root/requirements.txt
-ARG PYTHON_VERSION
-RUN /bin/bash -c 'if [ -n "${PYTHON_VERSION}" ] ; then \
-    apt-get install -fy python3-dev python3-openssl && \
-    export PYENV_ROOT=/usr/local/pyenv && \
-    curl https://pyenv.run | bash && \
-    command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH" && \
-    eval "$(pyenv init -)" && \
-    eval "$(pyenv virtualenv-init -)" && \
-    pyenv install -v $PYTHON_VERSION && \
-    pyenv global $PYTHON_VERSION && \
-    python -m pip install --upgrade pip && \
-    pip install virtualenv pre-commit && \
-    pip install -r /root/requirements.txt; \
-fi; \
-rm /root/requirements.txt; \
-'
+RUN mkdir -p /usr/local/data
+WORKDIR /usr/local/data
+
+#ARG DOCKER_VERSION
+# install docker
+RUN install -m 0755 -d /etc/apt/keyrings
+RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+RUN chmod a+r /etc/apt/keyrings/docker.gpg
+RUN echo \
+  "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
+  "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
+  tee /etc/apt/sources.list.d/docker.list > /dev/null
+RUN apt-get update
+RUN apt-get install -fy --fix-missing  docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+
+# Install websocat
+RUN  /bin/bash -c 'set -ex && \
+    ARCH=`uname -m` && \
+    PLATFORM=`uname -s | tr '[:upper:]' '[:lower:]'` && \
+    if [ "$ARCH" = "x86_64" ]; then \
+       echo "websocat x86_64" && \
+       curl -L "https://github.com/vi/websocat/releases/download/v1.10.0/websocat.x86_64-unknown-linux-musl" -o /usr/local/bin/websocat;\
+    else \
+       echo "websocat assuming ARM" && \
+       curl -L "https://github.com/vi/websocat/releases/download/v1.10.0/websocat.arm-unknown-linux-musleabi" -o /usr/local/bin/websocat;\
+    fi; \
+    chmod +x /usr/local/bin/websocat;'
 
 # install php if requested
 ARG PHP_VERSION
@@ -102,34 +111,29 @@ RUN  /bin/bash -c 'if [ -n "${RUST_VERSION}" ]; then \
     fi; \
 fi'
 
-
-RUN mkdir -p /usr/local/data
-WORKDIR /usr/local/data
-
-#ARG DOCKER_VERSION
-# install docker
-RUN install -m 0755 -d /etc/apt/keyrings
-RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
-RUN chmod a+r /etc/apt/keyrings/docker.gpg
-RUN echo \
-  "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
-  "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
-  tee /etc/apt/sources.list.d/docker.list > /dev/null
-RUN apt-get update
-RUN apt-get install -fy --fix-missing  docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
-
-# Install websocat
-RUN  /bin/bash -c 'set -ex && \
-    ARCH=`uname -m` && \
-    PLATFORM=`uname -s | tr '[:upper:]' '[:lower:]'` && \
-    if [ "$ARCH" = "x86_64" ]; then \
-       echo "websocat x86_64" && \
-       curl -L "https://github.com/vi/websocat/releases/download/v1.10.0/websocat.x86_64-unknown-linux-musl" -o /usr/local/bin/websocat;\
-    else \
-       echo "websocat assuming ARM" && \
-       curl -L "https://github.com/vi/websocat/releases/download/v1.10.0/websocat.arm-unknown-linux-musleabi" -o /usr/local/bin/websocat;\
+# intstall python if requested
+RUN mkdir -p /build-tmp
+COPY /root/bin/requirements.txt /build-tmp/requirements.txt
+ARG PYTHON_VERSION
+ARG PIP_EXTRA_REQUIREMENTS_TXT
+ARG HOST_PROJECT_FOLDER_NAME
+COPY tmp/$HOST_PROJECT_FOLDER_NAME-* /build-tmp/
+RUN /bin/bash -c 'if [ -n "${PYTHON_VERSION}" ] ; then \
+    apt-get install -fy python3-dev python3-openssl && \
+    export PYENV_ROOT=/usr/local/pyenv && \
+    curl https://pyenv.run | bash && \
+    command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH" && \
+    eval "$(pyenv init -)" && \
+    eval "$(pyenv virtualenv-init -)" && \
+    pyenv install -v $PYTHON_VERSION && \
+    pyenv global $PYTHON_VERSION && \
+    python -m pip install --upgrade pip && \
+    pip install -r /build-tmp/requirements.txt && \
+    if [ -n "${PIP_EXTRA_REQUIREMENTS_TXT}" ]; then \
+        pip install -r "/build-tmp/${HOST_PROJECT_FOLDER_NAME}-pip-extra-requirements.txt" || exit 1; \
     fi; \
-    chmod +x /usr/local/bin/websocat;'
+fi; \
+'
 
 # Install AWS CLI and SSM plugin
 ARG USE_AWS
@@ -261,6 +265,7 @@ WORKDIR /workspace
 ENV PHP_VERSION=$PHP_VERSION
 ENV USE_JAVA=$USE_JAVA
 ENV PYTHON_VERSION=$PYTHON_VERSION
+ENV PIP_EXTRA_REQUIREMENTS_TXT=$PIP_EXTRA_REQUIREMENTS_TXT
 ENV GOLANG_VERSION=$GOLANG_VERSION
 ENV USE_DOT_NET=$USE_DOT_NET
 ENV USE_AWS=$USE_AWS

docker-compose.yml

@@ -20,6 +20,7 @@ services:
         - USE_DOT_NET # install ubuntu:latest dotnet core packages
         - GOLANG_VERSION # install this golang version. blank for none. latest is 1.18
         - PYTHON_VERSION # install this python version
+        - PIP_EXTRA_REQUIREMENTS_TXT # install extra pip requirements if python is installed
         - USE_AWS # install latest aws cli, ssm plugin, and ecr helper
         - NODE_VERSION # install this version of node. blank for none
         - RUST_VERSION # install this version of rust. blank for none
@@ -31,6 +32,7 @@ services:
 #        - DOCKER_COMPOSE_VERSION # install docker-compose version inside container
         - EXTRA_PACKAGES # any extra Ubuntu packages you want installed into the base container
         - AWS_VERSION # install AWS CLI version
+        - HOST_PROJECT_FOLDER_NAME
     environment:
       - ARCH
       - AWS_VERSION
@@ -123,7 +125,7 @@ services:
       - PROXY_URL # http proxy to use. if USE_PROXY_HOST is enabled will be set to http://PROXY_CONTAINER_NAME:8080
       - PROXY_URL_SSL # https proxy to use. if USE_PROXY_HOST is enabled will be set to https://PROXY_CONTAINER_NAME:8080
       - PROXY_AUTO_EXPORT_ENV # auto export HTTP_PROXY and HTTPS_PROXY vars equal to PROXY_URL
-      - DEV_CONTAINER=1.10.12 # used to detect if running inside dev container
+      - DEV_CONTAINER=1.10.13 # used to detect if running inside dev container
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock # allow container to interact with host docker
       - "${GDC_DIR}:/root/gdc-host" # mount gdc folder inside container to get access to compose files

noop

@@ -0,0 +1 @@
+This file does nothing. Its purpose is to allow wildcard copy in dockerfile

readme.md

@@ -143,6 +143,7 @@ These options control what packages / functionality are built into the container
 * USE_DOT_NET=no - installs .NET SDK v6.x.
 * USE_JAVA=no - installs headless openjdk v11.x.
 * PYTHON_VERSION=3.11 - installs Python v3.11, which is highest aws lambda supported python runtime
+* PIP_EXTRA_REQUIREMENTS_TXT=<not set> - if set should be a name of a requirements pip file to be installed if python is installed.
 * USE_PRECOMMIT=no - installs git pre-commit hooks in repo if not already installed. Enables Python if not already enabled.
 * USE_AWS=yes - installs AWS CLI version specified by AWS_VERSION, SSM Plugin and EKS IAM auth helper as well as aws helper scripts and aliases.
 * AWS_VERSION=latest - installs specified AWS cli version if USE_AWS=yes. Defaults to latest.

root/bin/requirements.txt

@@ -1,6 +1,9 @@
+virtualenv
+pipenv
+pre-commit
 boto3
 click
 awscliv2
 awscli-local
 terraform-local
-pipenv
+

run-dev-container.sh

@@ -47,6 +47,12 @@ fi
 HOST_PROJECT_FOLDER_NAME="$(basename "$HOST_PROJECT_PATH")"
 export HOST_PROJECT_FOLDER_NAME
 
+if [ ! -d "./tmp" ]; then
+  mkdir ./tmp
+fi
+rm -rf "$GDC_DIR/tmp/$HOST_PROJECT_FOLDER_NAME-*"
+cp "$GDC_DIR/noop" "$GDC_DIR/tmp/$HOST_PROJECT_FOLDER_NAME-noop"
+
 path1=$GDC_DIR
 path2=$HOST_PROJECT_PATH
 
@@ -93,6 +99,13 @@ if [[ "$AWS_VERSION" = "latest" ]]; then
   fi
 fi
 
+if [ -n "$PYTHON_VERSION" ]; then
+  if [ -n "$PIP_EXTRA_REQUIREMENTS_TXT" ]; then
+    export PIP_EXTRA_REQUIREMENTS_TXT
+    cp "$HOST_PROJECT_PATH/$PIP_EXTRA_REQUIREMENTS_TXT" "$GDC_DIR/tmp/$HOST_PROJECT_FOLDER_NAME-pip-extra-requirements.txt" || exit 1
+  fi
+fi
+
 # Function to convert a string to a number in the range 10-200
 convert_string_to_number() {
     local input_string=$1
@@ -211,11 +224,6 @@ if [ -n "$GDC_DNS_PRI_IP" ]; then
   COMPOSE_FILES="$COMPOSE_FILES -f dc-dns.yml"
 fi
 
-
-if [ ! -d "./tmp" ]; then
-  mkdir ./tmp
-fi
-
 # this is the stack name for compose
 COMPOSE_PROJECT_NAME="$GDC_NAME"
 COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME//[ -]/_}"