diff --git a/.github/workflows/precompiled.yml b/.github/workflows/precompiled.yml
index 3ea2171..700afe3 100644
--- a/.github/workflows/precompiled.yml
+++ b/.github/workflows/precompiled.yml
@@ -58,7 +58,7 @@ jobs:
         with:
           ruby-version: "3.2"
           bundler-cache: true
-      - run: ./scripts/test-gem-build gems ruby
+      - run: ./scripts/test-gem-build gems ruby ${{github.ref_type}}
       - uses: actions/upload-artifact@v3
         with:
           name: cruby-gem
@@ -169,7 +169,7 @@ jobs:
         run: |
           docker run --rm -v "$(pwd):/re2" -w /re2 \
             ${DOCKER_IMAGE} \
-            ./scripts/test-gem-build gems ${{matrix.plat}}
+            ./scripts/test-gem-build gems ${{matrix.plat}} ${github.ref_type}}
       - uses: actions/upload-artifact@v3
         with:
           name: "cruby-${{matrix.plat}}-gem"
diff --git a/scripts/test-gem-build b/scripts/test-gem-build
index 0bdd772..5d69171 100755
--- a/scripts/test-gem-build
+++ b/scripts/test-gem-build
@@ -11,13 +11,17 @@ set -e -u
 
 OUTPUT_DIR=$1
 BUILD_NATIVE_GEM=$2
+REF_TYPE=$3
 
 test -e /etc/os-release && cat /etc/os-release
 
 set -x
 
 bundle install --local || bundle install
-bundle exec rake set-version-to-timestamp
+
+if [[ "${REF_TYPE}}" != "tag" ]]; then
+  bundle exec rake set-version-to-timestamp
+fi
 
 if [[ "${BUILD_NATIVE_GEM}" == "ruby" ]] ; then
   # TODO we're only compiling so that we retrieve tarballs, we can do better.