diff --git a/test/Dockerfile b/test/Dockerfile
index 06c99e8..75a8236 100644
--- a/test/Dockerfile
+++ b/test/Dockerfile
@@ -1,5 +1,63 @@
+ARG TOOLCHAIN_DIR=/toolchain
+ARG TOOLCHAIN_GNU_INSTALL_DIR=${TOOLCHAIN_DIR}/gnu
+
+# Build stage
+FROM ubuntu:22.04 AS builder
+
+# Needed to have the arguments in the stage
+ARG TOOLCHAIN_DIR
+ARG TOOLCHAIN_GNU_INSTALL_DIR
+
+RUN DEBIAN_FRONTEND=noninteractive \
+	apt-get update && \
+	apt-get upgrade -y && \
+	apt-get install -y --no-install-recommends \
+		autoconf \
+		automake \
+		autotools-dev \
+		bc \
+		bison \
+		build-essential \
+		ca-certificates \
+		curl \
+		flex \
+		gawk \
+		git \
+		gperf \
+		libexpat-dev \
+		libgmp-dev \
+		libmpc-dev \
+		libmpfr-dev \
+		libtool \
+		patchutils \
+		python3 \
+		texinfo \
+		zlib1g-dev \
+	&& apt-get clean && \
+	rm -rf /var/lib/apt/lists/*
+
+# Build gnu toolchain
+ENV TOOLCHAIN_GNU_INSTALL_DIR=${TOOLCHAIN_GNU_INSTALL_DIR}
+COPY docker/clone_riscv_gnu_toolchain.sh ./
+RUN ./clone_riscv_gnu_toolchain.sh
+COPY docker/build_riscv_gnu_toolchain.sh ./
+RUN ./build_riscv_gnu_toolchain.sh
+
+# Build pulp-sdk
+COPY docker/requirements-pulp-sdk.txt ./
+RUN pip3 install --no-cache-dir -r requirements-pulp-sdk.txt
+ENV PULP_RISCV_GCC_TOOLCHAIN=${TOOLCHAIN_GNU_INSTALL_DIR}
+COPY docker/get_pulp_sdk.sh ./
+RUN ./get_pulp_sdk.sh
+
+
+# Minimal image
 FROM ubuntu:22.04
 
+# Needed to have the arguments in the stage
+ARG TOOLCHAIN_DIR
+ARG TOOLCHAIN_GNU_INSTALL_DIR
+
 RUN DEBIAN_FRONTEND=noninteractive \
 	apt-get update && \
 	apt-get upgrade && \
@@ -8,10 +66,7 @@ RUN DEBIAN_FRONTEND=noninteractive \
 		ca-certificates \
 		cmake \
 		curl \
-		gcc \
 		git \
-		git-lfs \
-		g++ \
 		make \
 		python3 \
 		python3-pip \
@@ -24,10 +79,7 @@ COPY requirements-pip.txt docker/requirements-pulp-sdk.txt ./
 RUN pip3 install --no-cache-dir -r requirements-pip.txt -r requirements-pulp-sdk.txt --extra-index-url https://download.pytorch.org/whl/cpu
 
 ENV PULP_SDK_HOME="/pulp-sdk"
-ENV PULP_RISCV_GCC_TOOLCHAIN="/toolchains/pulp-riscv-gnu-toolchain"
-
-COPY docker/get_pulp_riscv_gnu_toolchain.sh ./
-RUN ./get_pulp_riscv_gnu_toolchain.sh
+ENV PULP_RISCV_GCC_TOOLCHAIN=${TOOLCHAIN_GNU_INSTALL_DIR}
 
-COPY docker/get_pulp_sdk.sh ./
-RUN ./get_pulp_sdk.sh
+COPY --from=builder ${TOOLCHAIN_GNU_INSTALL_DIR} ${TOOLCHAIN_GNU_INSTALL_DIR}
+COPY --from=builder /pulp-sdk ./
diff --git a/test/docker/build_riscv_gnu_toolchain.sh b/test/docker/build_riscv_gnu_toolchain.sh
new file mode 100755
index 0000000..c317b47
--- /dev/null
+++ b/test/docker/build_riscv_gnu_toolchain.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Build and install
+mkdir -p $TOOLCHAIN_GNU_INSTALL_DIR
+cd riscv-gnu-toolchain
+./configure --prefix=$TOOLCHAIN_GNU_INSTALL_DIR --with-arch=rv32imfcxpulpv3 --with-abi=ilp32 --enable-multilib
+make
diff --git a/test/docker/clone_riscv_gnu_toolchain.sh b/test/docker/clone_riscv_gnu_toolchain.sh
new file mode 100755
index 0000000..f53ea00
--- /dev/null
+++ b/test/docker/clone_riscv_gnu_toolchain.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Clone and get all submodules except qemu
+git clone https://github.com/pulp-platform/riscv-gnu-toolchain.git \
+	--branch=v2.6.0 --depth=1
+cd riscv-gnu-toolchain
+git submodule update --init --recursive --depth=1 --recommend-shallow riscv-*