From 2c32294ba95faf2bd8c4965fc58ff4af29e2ae64 Mon Sep 17 00:00:00 2001 From: Oscar Reimer Date: Tue, 3 Oct 2023 13:40:59 +0200 Subject: [PATCH 1/3] Build images on pull requests (#121), add cache and optimise build order --- .github/workflows/docker.yml | 16 +++++---- build/docker/alpine.Dockerfile | 36 ++++++++++++------- build/docker/debian.Dockerfile | 66 ++++++++++++++++++---------------- 3 files changed, 68 insertions(+), 50 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d30cb146..f6142165 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -6,6 +6,7 @@ on: - main tags: - 'v*' + pull_request: jobs: push: @@ -20,20 +21,21 @@ jobs: uses: actions/checkout@v3 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub if: github.event_name != 'pull_request' - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -41,7 +43,7 @@ jobs: - name: Docker meta id: meta - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: flavor: | latest=${{ matrix.stage == 'cli' }} @@ -61,7 +63,7 @@ jobs: type=raw,value=latest-${{ matrix.stage }} - name: Build and push ${{ matrix.docker-os }} images - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: build/docker/${{ matrix.docker-os }}.Dockerfile @@ -70,3 +72,5 @@ jobs: push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/build/docker/alpine.Dockerfile b/build/docker/alpine.Dockerfile index a4b85af1..71ca6f3e 100644 --- a/build/docker/alpine.Dockerfile +++ b/build/docker/alpine.Dockerfile @@ -8,26 +8,19 @@ COPY . . RUN go build -o debricked ./cmd/debricked ENTRYPOINT ["debricked"] -FROM alpine:latest AS cli +FROM alpine:latest AS cli-base ENV DEBRICKED_TOKEN="" RUN apk add --no-cache git WORKDIR /root/ + +# Please update resolution step accordingly when changing this +FROM cli-base AS cli COPY --from=dev /cli/debricked /usr/bin/debricked FROM cli AS scan ENTRYPOINT [ "debricked", "scan" ] -FROM cli AS resolution -RUN apk --no-cache --update add \ - openjdk11-jre \ - python3 \ - py3-scipy \ - py3-pip \ - go~=1.20 \ - nodejs \ - yarn \ - dotnet7-sdk - +FROM cli-base AS resolution ENV MAVEN_VERSION 3.9.2 ENV MAVEN_HOME /usr/lib/mvn ENV PATH $MAVEN_HOME/bin:$PATH @@ -41,4 +34,21 @@ ENV GRADLE_HOME /usr/lib/gradle ENV PATH $GRADLE_HOME/gradle-$GRADLE_VERSION/bin:$PATH RUN wget https://services.gradle.org/distributions/gradle-$GRADLE_VERSION-bin.zip && \ unzip gradle-$GRADLE_VERSION-bin.zip -d $GRADLE_HOME && \ - rm gradle-$GRADLE_VERSION-bin.zip \ No newline at end of file + rm gradle-$GRADLE_VERSION-bin.zip + +RUN apk --no-cache --update add \ + openjdk11-jre \ + python3 \ + py3-scipy \ + py3-pip \ + go~=1.20 \ + nodejs \ + yarn \ + dotnet7-sdk + +RUN dotnet --version + +# Put copy at the end to speedup Docker build by caching +COPY --from=dev /cli/debricked /usr/bin/debricked + +ENTRYPOINT [ "debricked", "scan" ] diff --git a/build/docker/debian.Dockerfile b/build/docker/debian.Dockerfile index 75117158..06c8ed4b 100644 --- a/build/docker/debian.Dockerfile +++ b/build/docker/debian.Dockerfile @@ -8,45 +8,29 @@ COPY . . RUN go build -o debricked ./cmd/debricked ENTRYPOINT ["debricked"] -FROM debian:bullseye-slim AS cli +FROM debian:bullseye-slim AS cli-base ENV DEBRICKED_TOKEN="" RUN apt -y update && apt -y upgrade && apt -y install git && \ apt -y clean && rm -rf /var/lib/apt/lists/* WORKDIR /root/ + +# Please update resolution step accordingly when changing this +FROM cli-base AS cli COPY --from=dev /cli/debricked /usr/bin/debricked FROM cli AS scan ENTRYPOINT [ "debricked", "scan" ] -FROM cli AS resolution -RUN echo "deb http://ftp.us.debian.org/debian testing-updates main" >> /etc/apt/sources.list && \ - echo "deb http://ftp.us.debian.org/debian testing main" >> /etc/apt/sources.list && \ - echo "Package: *" >> /etc/apt/preferences && \ - echo "Pin: release a=testing" >> /etc/apt/preferences && \ - echo "Pin-Priority: -2" >> /etc/apt/preferences - -RUN apt -y update && apt -y upgrade && apt -y install openjdk-11-jre \ - wget \ - unzip \ - python3 \ - python3-scipy \ - ca-certificates \ - curl \ - gnupg \ - python3-pip && \ - apt -y install -t testing golang-1.20 && \ - apt -y clean && rm -rf /var/lib/apt/lists/* && \ - # Symlink pip3 to pip, we assume that "pip" works in CLI - ln -sf /usr/bin/pip3 /usr/bin/pip && \ - # Symlink go binary to bin directory which is in path - ln -s /usr/lib/go-1.20/bin/go /usr/bin/go +FROM cli-base AS resolution +RUN apt -y update && apt -y upgrade && apt -y install curl gnupg unzip && \ + apt -y clean && rm -rf /var/lib/apt/lists/* RUN mkdir -p /etc/apt/keyrings ENV MAVEN_VERSION 3.9.2 ENV MAVEN_HOME /usr/lib/mvn ENV PATH $MAVEN_HOME/bin:$PATH -RUN wget http://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz && \ +RUN curl -fsSLO http://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz && \ tar -zxvf apache-maven-$MAVEN_VERSION-bin.tar.gz && \ rm apache-maven-$MAVEN_VERSION-bin.tar.gz && \ mv apache-maven-$MAVEN_VERSION $MAVEN_HOME @@ -54,28 +38,48 @@ RUN wget http://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/ap ENV GRADLE_VERSION 8.1.1 ENV GRADLE_HOME /usr/lib/gradle ENV PATH $GRADLE_HOME/gradle-$GRADLE_VERSION/bin:$PATH -RUN wget https://services.gradle.org/distributions/gradle-$GRADLE_VERSION-bin.zip && \ +RUN curl -fsSLO https://services.gradle.org/distributions/gradle-$GRADLE_VERSION-bin.zip && \ unzip gradle-$GRADLE_VERSION-bin.zip -d $GRADLE_HOME && \ rm gradle-$GRADLE_VERSION-bin.zip - ENV NODE_MAJOR 18 RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list RUN apt -y update && apt -y upgrade && apt -y install nodejs && \ apt -y clean && rm -rf /var/lib/apt/lists/* -RUN npm install -g npm@latest -RUN npm install --global yarn - +RUN npm install --global npm@latest && npm install --global yarn # https://learn.microsoft.com/en-us/dotnet/core/install/linux-scripted-manual#scripted-install # https://learn.microsoft.com/en-us/dotnet/core/install/linux-debian # Package manager installs are only supported on the x64 architecture. Other architectures, such as Arm, must install .NET by some other means such as with Snap, an installer script, or through a manual binary installation. ENV DOTNET_ROOT /usr/lib/dotnet ENV DOTNET_MAJOR 7.0 -RUN wget https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh +RUN curl -fsSLO https://dot.net/v1/dotnet-install.sh RUN chmod u+x ./dotnet-install.sh RUN ./dotnet-install.sh --channel $DOTNET_MAJOR --install-dir $DOTNET_ROOT RUN rm ./dotnet-install.sh ENV PATH $DOTNET_ROOT:$PATH -RUN dotnet --version \ No newline at end of file + +RUN echo "deb http://ftp.us.debian.org/debian testing-updates main" >> /etc/apt/sources.list && \ + echo "deb http://ftp.us.debian.org/debian testing main" >> /etc/apt/sources.list && \ + echo "Package: *" >> /etc/apt/preferences && \ + echo "Pin: release a=testing" >> /etc/apt/preferences && \ + echo "Pin-Priority: -2" >> /etc/apt/preferences + +RUN apt -y update && apt -y upgrade && apt -y install openjdk-11-jre \ + python3 \ + python3-scipy \ + ca-certificates \ + python3-pip && \ + apt -y install -t testing golang-1.20 && \ + apt -y clean && rm -rf /var/lib/apt/lists/* && \ + # Symlink pip3 to pip, we assume that "pip" works in CLI + ln -sf /usr/bin/pip3 /usr/bin/pip && \ + # Symlink go binary to bin directory which is in path + ln -s /usr/lib/go-1.20/bin/go /usr/bin/go + +RUN dotnet --version + +COPY --from=dev /cli/debricked /usr/bin/debricked + +ENTRYPOINT [ "debricked", "scan" ] From 20b7f70fe4bd54491f7ddf73f0d9eae1dc347eca Mon Sep 17 00:00:00 2001 From: Oscar Reimer Date: Tue, 3 Oct 2023 14:02:45 +0200 Subject: [PATCH 2/3] Only build images on pull requests when Dockerfiles have changed --- .github/workflows/docker.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f6142165..5ed395d9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -7,10 +7,13 @@ on: tags: - 'v*' pull_request: + paths: + - '**.Dockerfile' + - '.github/workflows/docker.yaml' jobs: push: - name: 'Push Docker images' + name: 'Build and (conditionally) push Docker images' strategy: matrix: stage: ['cli', 'scan', 'resolution'] From 83e8660a3772a3a00c58803b2a2fa5b64725aeb5 Mon Sep 17 00:00:00 2001 From: Oscar Reimer Date: Tue, 3 Oct 2023 14:07:38 +0200 Subject: [PATCH 3/3] Remove old docker test --- .github/workflows/test.yml | 23 ----------------------- Makefile | 4 ---- build/docker/alpine.Dockerfile | 2 +- build/docker/debian.Dockerfile | 1 + scripts/test_docker.sh | 30 ------------------------------ 5 files changed, 2 insertions(+), 58 deletions(-) delete mode 100755 scripts/test_docker.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fcbf1ec3..7fb5b8ed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,26 +61,3 @@ jobs: uses: golangci/golangci-lint-action@v3 with: version: v1.52 - - docker: - name: Docker - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version: 1.20 - - - name: Test dev image - run: bash scripts/test_docker.sh dev - - - name: Test cli image - run: bash scripts/test_docker.sh cli - - - name: Test scan image - run: bash scripts/test_docker.sh scan - - - name: Test resolution image - run: bash scripts/test_docker.sh resolution diff --git a/Makefile b/Makefile index a4240f0b..8386aa15 100644 --- a/Makefile +++ b/Makefile @@ -10,10 +10,6 @@ lint: test: bash scripts/test_cli.sh -.PHONY: test-docker -test-docker: - bash scripts/test_docker.sh cli - .PHONY: test-e2e test-e2e: bash scripts/test_e2e.sh diff --git a/build/docker/alpine.Dockerfile b/build/docker/alpine.Dockerfile index 71ca6f3e..9e577a97 100644 --- a/build/docker/alpine.Dockerfile +++ b/build/docker/alpine.Dockerfile @@ -48,7 +48,7 @@ RUN apk --no-cache --update add \ RUN dotnet --version -# Put copy at the end to speedup Docker build by caching +# Put copy at the end to speedup Docker build by caching previous RUNs and run those concurrently COPY --from=dev /cli/debricked /usr/bin/debricked ENTRYPOINT [ "debricked", "scan" ] diff --git a/build/docker/debian.Dockerfile b/build/docker/debian.Dockerfile index 06c8ed4b..8d40c611 100644 --- a/build/docker/debian.Dockerfile +++ b/build/docker/debian.Dockerfile @@ -80,6 +80,7 @@ RUN apt -y update && apt -y upgrade && apt -y install openjdk-11-jre \ RUN dotnet --version +# Put copy at the end to speedup Docker build by caching previous RUNs and run those concurrently COPY --from=dev /cli/debricked /usr/bin/debricked ENTRYPOINT [ "debricked", "scan" ] diff --git a/scripts/test_docker.sh b/scripts/test_docker.sh deleted file mode 100755 index 36cb9e32..00000000 --- a/scripts/test_docker.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -type="$1" - -build_command() -{ - docker build -f build/docker/alpine.Dockerfile -t debricked/cli:$1 --target $1 . - docker build -f build/docker/debian.Dockerfile -t debricked/cli:$1-debian --target $1 . -} - -case $type in - "dev") - build_command $type - ;; - "cli") - build_command $type - ;; - "scan") - build_command $type - ;; - "resolution") - build_command $type - ;; - *) - echo -e "Please use the following type dev, cli, scan. For example ./test_docker.sh dev" - exit 1 - ;; -esac - -