massive update for proxy support

devxpod · Feb 8, 2024 · 2f533a1 · 2f533a1
1 parent 8323731
commit 2f533a1
Show file tree

Hide file tree

Showing 10 changed files with 78 additions and 31 deletions.
diff --git a/.env-gdc b/.env-gdc
@@ -119,10 +119,11 @@ export DEV_CONTAINER_NAME=${DEV_CONTAINER_NAME:="dev-1"} # dev container name
 
 export COPY_CMD_TO_CLIPBOARD=${COPY_CMD_TO_CLIPBOARD:=yes} # COPY GDC shell launch command to clipboard
 
-export USE_PROXY_HOST=${USE_PROXY_HOST:=no}                # no, proxy, dump, web
+export USE_PROXY=${USE_PROXY:=no}                          # no, proxy, dump, web
+export USE_PROXY_CA=${USE_PROXY_CA:=yes}                   # if yes and USE_PROXY!=no then install proxy CA cert into GDC
 export PROXY_VERSION=${PROXY_VERSION:=latest}              # container image version tag
 export PROXY_CONTAINER_NAME=${PROXY_CONTAINER_NAME:=proxy} # name of the container
+export PROXY_AUTO_EXPORT_ENV=${PROXY_AUTO_EXPORT_ENV:=no}  # auto export HTTP_PROXY and HTTPS_PROXY
+export USE_PROXY_HOST=${USE_PROXY_HOST:=no}                # no, yes
 export PROXY_HOST_PORT=${PROXY_HOST_PORT:=8080}            # port to expose to host
 export PROXY_WEB_HOST_PORT=${PROXY_WEB_HOST_PORT:=8081}    # if running in web mode expose this port
-export PROXY_VOLUME_DIR=${PROXY_VOLUME_DIR:=/tmp/mitproxy} # volume to persist certs to
-export PROXY_AUTO_EXPORT_ENV=${PROXY_AUTO_EXPORT_ENV:=no}  # auto export HTTP_PROXY and HTTPS_PROXY
diff --git a/dc-proxy-dump.yml b/dc-proxy-dump.yml
@@ -2,4 +2,4 @@ version: '3.8'
 
 services:
   proxy:
-    command: mitdump
+    command: mitmdump --set block_global=false  --set flow_detail=3
diff --git a/dc-proxy-host.yml b/dc-proxy-host.yml
@@ -2,17 +2,5 @@ version: '3.8'
 
 services:
   proxy:
-    image: "mitmproxy/mitmproxy:${PROXY_VERSION-latest}"
-    container_name: "${PROXY_CONTAINER_NAME-proxy}"
-    networks:
-      devnet:
     ports:
       - "${PROXY_HOST_PORT-8080}:8080"
-    volumes:
-      - "${PROXY_VOLUME_DIR:-/tmp/mitmproxy}:/home/mitmproxy/.mitmproxy"
-
-
-networks:
-  devnet:
-    name: ${DEVNET_NAME-devnet}
-    external: true
diff --git a/dc-proxy-web-host.yml b/dc-proxy-web-host.yml
@@ -4,4 +4,4 @@ services:
   proxy:
     ports:
       - "${PROXY_WEB_HOST_PORT-8081}:8081"
-    command: mitmweb --web-host 0.0.0.0
+    command: mitmweb --web-iface 0.0.0.0 --web-host 0.0.0.0
diff --git a/dc-proxy-web.yml b/dc-proxy-web.yml
@@ -0,0 +1,5 @@
+version: '3.8'
+
+services:
+  proxy:
+    command: mitmweb --web-iface 0.0.0.0 --web-host 0.0.0.0
diff --git a/dc-proxy.yml b/dc-proxy.yml
@@ -0,0 +1,21 @@
+version: '3.8'
+
+services:
+  proxy:
+    image: "mitmproxy/mitmproxy:${PROXY_VERSION-latest}"
+    container_name: "${PROXY_CONTAINER_NAME-proxy}"
+    stdin_open: true
+    tty: true
+    dns:
+      - ${GDC_DNS_PRI_IP}
+      - ${GDC_DNS_SEC_IP}
+    networks:
+      devnet:
+    volumes:
+      - "${PROXY_VOLUME_DIR:-/tmp/mitmproxy}:/home/mitmproxy/.mitmproxy"
+
+
+networks:
+  devnet:
+    name: ${DEVNET_NAME-devnet}
+    external: true
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -109,15 +109,18 @@ services:
       - COMPOSE_BIN # used to toggle between docker-compose and docker compose invocations
       - COPY_CMD_TO_CLIPBOARD # defaults to yes to copy gdc shell launch command to clipboard
       - HOST_PROJECT_REL_PATH # Project path on host relative to GDC repository
-      - USE_PROXY_HOST # can be no/proxy/dump/web. defaults to no
+      - USE_PROXY # can be no/proxy/dump/web. defaults to no
+      - USE_PROXY_HOST # can be no/yes. defaults to no
+      - USE_PROXY_CA # if yes and USE_PROXY is not set to no then install proxy CA in GDC
       - PROXY_VERSION # version of proxy image to start. default to latest
       - PROXY_CONTAINER_NAME # name of proxy container. defaults to proxy
       - PROXY_HOST_PORT # proxy port. defaults to 8080
       - PROXY_WEB_HOST_PORT # when running in web mode defaults to port 8081
       - PROXY_VOLUME_DIR # where to persist cert files. defaults to /tmp/mitproxy
-      - PROXY_URL # http(s) proxy to use. if USE_PROXY_HOST is enabled will be set to http://PROXY_CONTAINER_NAME:8080
+      - PROXY_URL # http proxy to use. if USE_PROXY_HOST is enabled will be set to http://PROXY_CONTAINER_NAME:8080
+      - PROXY_URL_SSL # https proxy to use. if USE_PROXY_HOST is enabled will be set to https://PROXY_CONTAINER_NAME:8080
       - PROXY_AUTO_EXPORT_ENV # auto export HTTP_PROXY and HTTPS_PROXY vars equal to PROXY_URL
-      - DEV_CONTAINER=1.10.3 # used to detect if running inside dev container
+      - DEV_CONTAINER=1.10.4 # used to detect if running inside dev container
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock # allow container to interact with host docker
       - "${GDC_DIR}:/root/gdc-host" # mount gdc folder inside container to get access to compose files

diff --git a/postStartCommand.sh b/postStartCommand.sh
@@ -68,7 +68,19 @@ fi
 
 if [[ -n "$PROXY_URL" && "$PROXY_AUTO_EXPORT_ENV" = "yes" ]]; then
   export HTTP_PROXY=$PROXY_URL
-  export HTTPS_PROXY=$PROXY_URL
+  export HTTPS_PROXY=$PROXY_URL_SSL
+fi
+
+if [[ -n "$USE_PROXY" && "$USE_PROXY" != "no" && "$USE_PROXY_CA" = "yes" ]]; then
+  if [ -r /workspace/proxy_volume/mitmproxy-ca-cert.pem ]; then
+    echo "Setting up proxy CA..."
+    cp /workspace/proxy_volume/mitmproxy-ca-cert.pem /usr/local/share/ca-certificates/mitmproxy-ca-cert.crt
+    update-ca-certificates
+    cat /usr/local/share/ca-certificates/mitmproxy-ca-cert.crt >> "$(python -m certifi)"
+    export AWS_CA_BUNDLE=/usr/local/share/ca-certificates/mitmproxy-ca-cert.crt
+  else
+    echo "Unable to locate mitmproxy-ca-cert.pem. Please ensure the proxy_volume is mounted"
+  fi
 fi
 
 

diff --git a/readme.md b/readme.md
@@ -194,13 +194,14 @@ These options control what packages / functionality are built into the container
 * NO_DEVNET_RM=<not set> # if set to yes, GDC will not remove the DEVNET.
 * COMPOSE_EX=<not set> - add specified compose yaml file to list of compose files that get loaded for solution. Note paths in specified compose.yaml are relative to generic-dev-container repo folder.
 * STARTUP_MSG=<not set> - displays this message after container has started and init script is complete.
-* COPY_CMD_TO_CLIPBOARD=yes - copy gdc shell launch command to clipboard. 
-* USE_PROXY_HOST=no - can be no/proxy/dump/web.
+* COPY_CMD_TO_CLIPBOARD=yes - copy gdc shell launch command to clipboard.
+* USE_PROXY=no - can be no/proxy/dump/web.
+* USE_PROXY_HOST=no - can be no/yes.
 * PROXY_VERSION=latest - version of proxy image to start.
 * PROXY_CONTAINER_NAME=proxy - name of proxy container.
 * PROXY_HOST_PORT=8080 - host proxy port.
 * PROXY_WEB_HOST_PORT=8081 - host port when running in web mode.
-* PROXY_VOLUME_DIR=/tmp/mitproxy - where to persist cert files.
+* PROXY_VOLUME_DIR=$HOST_PROJECT_PATH/proxy_volume - where to persist cert files.
 * PROXY_AUTO_EXPORT_ENV=no  # auto export HTTP_PROXY and HTTPS_PROXY set to PROXY_URL. PROXY_URL is auto set when USE_PROXY_HOST != no
 
 # Extra environment vars available in dev container

diff --git a/run-dev-container.sh b/run-dev-container.sh
@@ -449,17 +449,33 @@ fi
 
 
 # this will start mitm proxy server
-if [[ -n "$USE_PROXY_HOST" && "$USE_PROXY_HOST" != "no" ]]; then
-  echo "Adding compose layer dc-proxy-host.yml"
-  COMPOSE_FILES="$COMPOSE_FILES -f dc-proxy-host.yml"
-  if [[ "$USE_PROXY_HOST" = "web" ]]; then
-    echo "Adding compose layer dc-proxy-web-host.yml"
-    COMPOSE_FILES="$COMPOSE_FILES -f dc-proxy-web-host.yml"
-  elif [[ "$USE_PROXY_HOST" = "dump" ]]; then
+if [[ -n "$USE_PROXY" && "$USE_PROXY" != "no" ]]; then
+  if [ -z "$PROXY_VOLUME_DIR" ]; then
+    PROXY_VOLUME_DIR="$HOST_PROJECT_PATH/proxy_volume"
+  fi
+  export PROXY_VOLUME_DIR
+  echo "Adding compose layer dc-proxy.yml"
+  COMPOSE_FILES="$COMPOSE_FILES -f dc-proxy.yml"
+
+  if [[ -n "$USE_PROXY_HOST" && "$USE_PROXY_HOST" != "no" ]]; then
+    echo "Adding compose layer dc-proxy-host.yml"
+    COMPOSE_FILES="$COMPOSE_FILES -f dc-proxy-host.yml"
+    if [[ "$USE_PROXY" = "web" ]]; then
+      echo "Adding compose layer dc-proxy-web-host.yml"
+      COMPOSE_FILES="$COMPOSE_FILES -f dc-proxy-web-host.yml"
+    fi
+  else
+    if [[ "$USE_PROXY" = "web" ]]; then
+      echo "Adding compose layer dc-proxy-web.yml"
+      COMPOSE_FILES="$COMPOSE_FILES -f dc-proxy-web.yml"
+    fi
+  fi
+  if [[ "$USE_PROXY" = "dump" ]]; then
     echo "Adding compose layer dc-proxy-dump.yml"
     COMPOSE_FILES="$COMPOSE_FILES -f dc-proxy-dump.yml"
   fi
   export PROXY_URL=http://$PROXY_CONTAINER_NAME:8080
+  export PROXY_URL_SSL=https://$PROXY_CONTAINER_NAME:8080
 fi
 
 # forwards ssh agent socket to container