diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index a10e891bf..75c9b913d 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -110,7 +110,7 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest]
         arch: [x86_64, aarch64]
-        cuda-version: ['11.8.0', '12.1.1']
+        cuda-version: ${{ startsWith(github.ref, 'refs/tags') && fromJson('["11.8.0","12.1.1"]') || fromJson('["12.1.1"]') }}
         build_type: [Release]
         exclude:
           - os: windows-latest # This probably requires arm64 Windows agents