diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 27ab83138..c5958e955 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -103,7 +103,7 @@ jobs:
 
       matrix:
         os: [ubuntu-latest, windows-latest]
-        arch: [x86_64, aarch64]
+        arch: ${{ startsWith(github.ref, 'refs/tags') && fromJson('["x86_64","aarch64"]') || fromJson('["x86_64"]') }}
         cuda-version: ${{ startsWith(github.ref, 'refs/tags') && fromJson('["11.8.0","12.1.1"]') || fromJson('["12.1.1"]') }}
         build_type: [Release]
         exclude: