diff --git a/.github/workflows/builder.yml b/.github/workflows/builder.yml
index 33b21850..40ee61bd 100644
--- a/.github/workflows/builder.yml
+++ b/.github/workflows/builder.yml
@@ -80,6 +80,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
       - name: Cache Docker layers
+        if: "!startsWith(github.ref, 'refs/tags/')"
         uses: actions/cache@v2
         with:
           path: /tmp/.buildx-cache
@@ -117,6 +118,7 @@ jobs:
       # https://github.com/docker/build-push-action/issues/252
       # https://github.com/moby/buildkit/issues/1896
       - name: Move cache
+        if: "!startsWith(github.ref, 'refs/tags/')"
         run: |
           rm -rf /tmp/.buildx-cache
           mv /tmp/.buildx-cache-new /tmp/.buildx-cache