Merge pull request #126 from debricked/speedup-image-build

Build images on pull requests (#121), add cache and optimise build order

.github/workflows/docker.yml

@@ -6,10 +6,14 @@ on:
       - main
     tags:
       - 'v*'
+  pull_request:
+    paths:
+      - '**.Dockerfile'
+      - '.github/workflows/docker.yaml'
 
 jobs:
   push:
-    name: 'Push Docker images'
+    name: 'Build and (conditionally) push Docker images'
     strategy:
       matrix:
         stage: ['cli', 'scan', 'resolution']
@@ -20,28 +24,29 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           flavor: |
             latest=${{ matrix.stage == 'cli' }}
@@ -61,7 +66,7 @@ jobs:
             type=raw,value=latest-${{ matrix.stage }}
 
       - name: Build and push ${{ matrix.docker-os }} images
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: build/docker/${{ matrix.docker-os }}.Dockerfile
@@ -70,3 +75,5 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/test.yml

@@ -61,26 +61,3 @@ jobs:
         uses: golangci/golangci-lint-action@v3
         with:
           version: v1.52
-
-  docker:
-    name: Docker
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: 1.20
-
-      - name: Test dev image
-        run: bash scripts/test_docker.sh dev
-
-      - name: Test cli image
-        run: bash scripts/test_docker.sh cli
-
-      - name: Test scan image
-        run: bash scripts/test_docker.sh scan
-
-      - name: Test resolution image
-        run: bash scripts/test_docker.sh resolution

Makefile

@@ -10,10 +10,6 @@ lint:
 test:
 	bash scripts/test_cli.sh
 
-.PHONY: test-docker
-test-docker:
-	bash scripts/test_docker.sh cli
-
 .PHONY: test-e2e
 test-e2e:
 	bash scripts/test_e2e.sh

build/docker/alpine.Dockerfile

@@ -8,26 +8,19 @@ COPY . .
 RUN go build -o debricked ./cmd/debricked
 ENTRYPOINT ["debricked"]
 
-FROM alpine:latest AS cli
+FROM alpine:latest AS cli-base
 ENV DEBRICKED_TOKEN=""
 RUN apk add --no-cache git
 WORKDIR /root/
+
+# Please update resolution step accordingly when changing this
+FROM cli-base AS cli
 COPY --from=dev /cli/debricked /usr/bin/debricked
 
 FROM cli AS scan
 ENTRYPOINT [ "debricked",  "scan" ]
 
-FROM cli AS resolution
-RUN apk --no-cache --update add \
-  openjdk11-jre \
-  python3 \
-  py3-scipy \
-  py3-pip \
-  go~=1.20 \
-  nodejs \
-  yarn \
-  dotnet7-sdk
-
+FROM cli-base AS resolution
 ENV MAVEN_VERSION 3.9.2
 ENV MAVEN_HOME /usr/lib/mvn
 ENV PATH $MAVEN_HOME/bin:$PATH
@@ -41,4 +34,21 @@ ENV GRADLE_HOME /usr/lib/gradle
 ENV PATH $GRADLE_HOME/gradle-$GRADLE_VERSION/bin:$PATH
 RUN wget https://services.gradle.org/distributions/gradle-$GRADLE_VERSION-bin.zip && \
   unzip gradle-$GRADLE_VERSION-bin.zip -d $GRADLE_HOME && \
-  rm gradle-$GRADLE_VERSION-bin.zip
+  rm gradle-$GRADLE_VERSION-bin.zip
+
+RUN apk --no-cache --update add \
+  openjdk11-jre \
+  python3 \
+  py3-scipy \
+  py3-pip \
+  go~=1.20 \
+  nodejs \
+  yarn \
+  dotnet7-sdk
+
+RUN dotnet --version
+
+# Put copy at the end to speedup Docker build by caching previous RUNs and run those concurrently
+COPY --from=dev /cli/debricked /usr/bin/debricked
+
+ENTRYPOINT [ "debricked",  "scan" ]

build/docker/debian.Dockerfile

@@ -8,74 +8,79 @@ COPY . .
 RUN go build -o debricked ./cmd/debricked
 ENTRYPOINT ["debricked"]
 
-FROM debian:bullseye-slim AS cli
+FROM debian:bullseye-slim AS cli-base
 ENV DEBRICKED_TOKEN=""
 RUN apt -y update && apt -y upgrade && apt -y install git && \
     apt -y clean && rm -rf /var/lib/apt/lists/*
 WORKDIR /root/
+
+# Please update resolution step accordingly when changing this
+FROM cli-base AS cli
 COPY --from=dev /cli/debricked /usr/bin/debricked
 
 FROM cli AS scan
 ENTRYPOINT [ "debricked",  "scan" ]
 
-FROM cli AS resolution
-RUN echo "deb http://ftp.us.debian.org/debian testing-updates main" >> /etc/apt/sources.list && \
-    echo "deb http://ftp.us.debian.org/debian testing main" >> /etc/apt/sources.list && \
-    echo "Package: *" >> /etc/apt/preferences && \
-    echo "Pin: release a=testing" >> /etc/apt/preferences && \
-    echo "Pin-Priority: -2" >> /etc/apt/preferences
-
-RUN apt -y update && apt -y upgrade && apt -y install openjdk-11-jre \
-    wget \
-    unzip \
-    python3 \
-    python3-scipy \
-    ca-certificates \
-    curl \
-    gnupg \
-    python3-pip && \
-    apt -y install -t testing golang-1.20 && \
-    apt -y clean && rm -rf /var/lib/apt/lists/* && \
-    # Symlink pip3 to pip, we assume that "pip" works in CLI
-    ln -sf /usr/bin/pip3 /usr/bin/pip && \
-    # Symlink go binary to bin directory which is in path
-    ln -s /usr/lib/go-1.20/bin/go /usr/bin/go
+FROM cli-base AS resolution
+RUN apt -y update && apt -y upgrade && apt -y install curl gnupg unzip && \
+    apt -y clean && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /etc/apt/keyrings
 
 ENV MAVEN_VERSION 3.9.2
 ENV MAVEN_HOME /usr/lib/mvn
 ENV PATH $MAVEN_HOME/bin:$PATH
-RUN wget http://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz && \
+RUN curl -fsSLO http://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz && \
     tar -zxvf apache-maven-$MAVEN_VERSION-bin.tar.gz && \
     rm apache-maven-$MAVEN_VERSION-bin.tar.gz && \
     mv apache-maven-$MAVEN_VERSION $MAVEN_HOME
 
 ENV GRADLE_VERSION 8.1.1
 ENV GRADLE_HOME /usr/lib/gradle
 ENV PATH $GRADLE_HOME/gradle-$GRADLE_VERSION/bin:$PATH
-RUN wget https://services.gradle.org/distributions/gradle-$GRADLE_VERSION-bin.zip && \
+RUN curl -fsSLO https://services.gradle.org/distributions/gradle-$GRADLE_VERSION-bin.zip && \
     unzip gradle-$GRADLE_VERSION-bin.zip -d $GRADLE_HOME && \
     rm gradle-$GRADLE_VERSION-bin.zip
 
-
 ENV NODE_MAJOR 18
 RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
 RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
 RUN apt -y update && apt -y upgrade && apt -y install nodejs && \
     apt -y clean && rm -rf /var/lib/apt/lists/*
-RUN npm install -g npm@latest
-RUN npm install --global yarn
-
+RUN npm install --global npm@latest && npm install --global yarn
 
 # https://learn.microsoft.com/en-us/dotnet/core/install/linux-scripted-manual#scripted-install
 # https://learn.microsoft.com/en-us/dotnet/core/install/linux-debian
 # Package manager installs are only supported on the x64 architecture. Other architectures, such as Arm, must install .NET by some other means such as with Snap, an installer script, or through a manual binary installation.
 ENV DOTNET_ROOT /usr/lib/dotnet
 ENV DOTNET_MAJOR 7.0
-RUN wget https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh
+RUN curl -fsSLO https://dot.net/v1/dotnet-install.sh
 RUN chmod u+x ./dotnet-install.sh
 RUN ./dotnet-install.sh --channel $DOTNET_MAJOR --install-dir $DOTNET_ROOT
 RUN rm ./dotnet-install.sh
 ENV PATH $DOTNET_ROOT:$PATH
-RUN dotnet --version
+
+RUN echo "deb http://ftp.us.debian.org/debian testing-updates main" >> /etc/apt/sources.list && \
+    echo "deb http://ftp.us.debian.org/debian testing main" >> /etc/apt/sources.list && \
+    echo "Package: *" >> /etc/apt/preferences && \
+    echo "Pin: release a=testing" >> /etc/apt/preferences && \
+    echo "Pin-Priority: -2" >> /etc/apt/preferences
+
+RUN apt -y update && apt -y upgrade && apt -y install openjdk-11-jre \
+    python3 \
+    python3-scipy \
+    ca-certificates \
+    python3-pip && \
+    apt -y install -t testing golang-1.20 && \
+    apt -y clean && rm -rf /var/lib/apt/lists/* && \
+    # Symlink pip3 to pip, we assume that "pip" works in CLI
+    ln -sf /usr/bin/pip3 /usr/bin/pip && \
+    # Symlink go binary to bin directory which is in path
+    ln -s /usr/lib/go-1.20/bin/go /usr/bin/go
+
+RUN dotnet --version
+
+# Put copy at the end to speedup Docker build by caching previous RUNs and run those concurrently
+COPY --from=dev /cli/debricked /usr/bin/debricked
+
+ENTRYPOINT [ "debricked",  "scan" ]