Enhancements to CI Workflows and Python Module Initialization with Minor Fixes

.github/workflows/cmake.yml

@@ -1,76 +1,99 @@
 name: CI for general build
-
+# This workflow handles the general build process including CMake configuration,
+# C++ build, Python wrapper compilation, and testing across multiple platforms
 on:
-  push:
-    branches: [ master ]
-    tags:
-      - 'v*'
   pull_request:
     branches: [ master ]
+    types: [opened, synchronize, reopened]
+  workflow_dispatch:
+
+# Prevent concurrent workflow runs on the same PR
+concurrency:
+  group: cmake-${{ github.event.name }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: false
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  PYTHONPATH: ${{ github.workspace }}/build/root/lib
+  LD_LIBRARY_PATH: ${{ github.workspace }}/build/root/lib
 
 permissions:
-  contents: read
+  contents: write
+  pull-requests: write
+  actions: write
+  checks: write
+  id-token: write
 
 jobs:
   build:
+    # Only run on pull requests from forks
+    if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork || github.event_name == 'workflow_dispatch'
     strategy:
+      fail-fast: false
       matrix:
-        os: [ ubuntu-latest, ubuntu-20.04, windows-latest, macOS-11 ]
+        os: [ ubuntu-latest ]
         arch: [ x64 ]
-        include:
-          - os: windows-latest
-            arch: x86
     runs-on: ${{ matrix.os }}
 
-    permissions:
-      contents: write # svenstaro/upload-release-action 
+    # Inherit permissions from workflow level
+    # Removed redundant permissions block as it's inherited from workflow level
 
     steps:
+    - name: Install Dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y cmake build-essential swig python3-dev
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      with:
+        fetch-depth: 2
     - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
       with:
         python-version: '3.x'
         architecture: ${{matrix.arch}}
 
-    - name: Config for Windows
-      if: runner.os == 'Windows'
+    - name: Configure CMake
       run: |
-        if ("${{matrix.arch}}" -eq "x64") {
-          $msbuildPlatform = "x64"
-        } else {
-          $msbuildPlatform = "Win32"
-        }
-        cmake -A $msbuildPlatform -B ${{github.workspace}}/build -DSPM_BUILD_TEST=ON -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root
-
-    - name: Config for Unix
-      if: runner.os != 'Windows'
-      run: cmake -B ${{github.workspace}}/build -DSPM_BUILD_TEST=ON -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root
-      env:
-        CMAKE_OSX_ARCHITECTURES: arm64;x86_64
+        echo "Configuring CMake build..."
+        cmake -B ${{github.workspace}}/build \
+          -DSPM_BUILD_TEST=ON \
+          -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root
 
     - name: Build
-      run: cmake --build ${{github.workspace}}/build --config Release --target install --parallel 8
+      run: |
+        echo "Building with CMake..."
+        cmake --build ${{github.workspace}}/build --config Release --target install --parallel 8
 
     - name: Test
       working-directory: ${{github.workspace}}/build
-      run: ctest -C Release --output-on-failure
+      run: |
+        echo "Running tests..."
+        ctest -C Release --output-on-failure -V
 
     - name: Package
       working-directory: ${{github.workspace}}/build
-      run: cpack
+      run: |
+        echo "Creating package..."
+        cpack -V
 
-    - name: Build Python wrapper
+    - name: Build Python wrapper (Unix)
+      if: runner.os != 'Windows'
       working-directory: ${{github.workspace}}/python
+      shell: bash
       run: |
+        python -m pip install --upgrade pip setuptools wheel
+        python -m pip install build pytest
         python -m pip install --require-hashes --no-dependencies -r ../.github/workflows/requirements/base.txt
-        python setup.py build
-        python setup.py bdist_wheel
-        python -m pytest
+        # Ensure we have the built C++ library in the Python path
+        echo "PYTHONPATH=${{github.workspace}}/build/root/lib" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=${{github.workspace}}/build/root/lib" >> $GITHUB_ENV
+        python setup.py build -v
+        python setup.py bdist_wheel -v
+        python -m pytest -v --log-cli-level=INFO
 
-    - name: Upload artifcacts
+    - name: Upload artifacts
       uses: actions/upload-artifact@v3
       with:
-        name: artifcacts
+        name: artifacts-${{ matrix.os }}-${{ matrix.arch }}
         path: ./build/*.7z
 
     - name: Upload Release Assets

.github/workflows/wheel.yml

@@ -8,8 +8,16 @@ on:
   pull_request:
     branches: [ master ]
 
+concurrency:
+  group: wheel-${{ github.event.pull_request.number || github.sha }}-${{ github.event.pull_request.head.ref || github.ref_name }}
+  cancel-in-progress: false
+
 permissions:
-  contents: read
+  contents: write
+  pull-requests: write
+  actions: write
+  checks: write
+  issues: write
 
 jobs:
   build_wheels:
@@ -18,16 +26,16 @@ jobs:
       digests-macos: ${{ steps.hash-macos.outputs.digests }}
       digests-windows: ${{ steps.hash-windows.outputs.digests }}
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macOS-11]
     runs-on: ${{ matrix.os }}
     name: Build wheels on ${{ matrix.os }}
 
-    permissions:
-      contents: write # svenstaro/upload-release-action 
-
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          fetch-depth: 0
       - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
         with:
           python-version: "3.x"
@@ -70,7 +78,8 @@ jobs:
           CIBW_ARCHS_MACOS: x86_64 universal2 arm64
           CIBW_ARCHS_WINDOWS: auto ARM64
           CIBW_SKIP: "pp* *-musllinux_*"
-          CIBW_BUILD_VERBOSITY: 1
+          CIBW_TEST_SKIP: "*-win_arm64 *_aarch64 *-macosx_arm64"
+          CIBW_BUILD_VERBOSITY: 2
 
       - name: Build sdist archive
         working-directory: ${{github.workspace}}/python
@@ -93,7 +102,7 @@ jobs:
       - name: Upload artifact
         uses: actions/upload-artifact@v3
         with:
-          name: artifacts
+          name: artifacts-${{ matrix.os }}
           path: |
             ./python/wheelhouse/*.whl
             ./python/wheelhouse/*.tar.gz
@@ -124,7 +133,7 @@ jobs:
         if: runner.os == 'Windows'
         run: echo "digests=$(sha256sum ./python/wheelhouse/* | base64 -w0)" >> $GITHUB_OUTPUT
 
-  gather-disgests:
+  gather-digests:
     needs: [build_wheels]
     outputs:
       digests: ${{ steps.hash.outputs.digests }}
@@ -138,19 +147,26 @@ jobs:
           WINDOWS_DIGESTS: "${{ needs.build_wheels.outputs.digests-windows }}"
         run: |
           set -euo pipefail
-          echo "$LINUX_DIGESTS" | base64 -d > checksums.txt
-          echo "$MACOS_DIGESTS" | base64 -d >> checksums.txt
-          echo "$WINDOWS_DIGESTS" | base64 -d >> checksums.txt
+          touch checksums.txt
+          if [ ! -z "${LINUX_DIGESTS:-}" ]; then
+            echo "$LINUX_DIGESTS" | base64 -d >> checksums.txt
+          fi
+          if [ ! -z "${MACOS_DIGESTS:-}" ]; then
+            echo "$MACOS_DIGESTS" | base64 -d >> checksums.txt
+          fi
+          if [ ! -z "${WINDOWS_DIGESTS:-}" ]; then
+            echo "$WINDOWS_DIGESTS" | base64 -d >> checksums.txt
+          fi
           echo "digests=$(cat checksums.txt | base64 -w0)" >> $GITHUB_OUTPUT
 
   provenance:
     if: startsWith(github.ref, 'refs/tags/')
-    needs: [build_wheels, gather-disgests]
+    needs: [build_wheels, gather-digests]
     permissions:
       actions: read   # To read the workflow path.
       id-token: write # To sign the provenance.
       contents: write # To add assets to a release.
     uses: slsa-framework/slsa-github-generator/.github/workflows/[email protected]
     with:
-      base64-subjects: "${{ needs.gather-disgests.outputs.digests }}"
+      base64-subjects: "${{ needs.gather-digests.outputs.digests }}"
       upload-assets: true # Optional: Upload to a new release

README.md

@@ -12,7 +12,7 @@
 SentencePiece is an unsupervised text tokenizer and detokenizer mainly for
 Neural Network-based text generation systems where the vocabulary size
 is predetermined prior to the neural model training. SentencePiece implements
-**subword units** (e.g., **byte-pair-encoding (BPE)** [[Sennrich et al.](https://www.aclweb.org/anthology/P16-1162)]) and
+**subword units** (e.g., **byte-pair-encoding (BPE)** [[Sennrich et al.](https://aclanthology.org/P16-1162)]) and
 **unigram language model** [[Kudo.](https://arxiv.org/abs/1804.10959)])
 with the extension of direct training from raw sentences. SentencePiece allows us to make a purely end-to-end system that does not depend on language-specific pre/postprocessing.
 

python/setup.py

@@ -24,18 +24,21 @@
 from setuptools.command.build_ext import build_ext as _build_ext
 from setuptools.command.build_py import build_py as _build_py
 
+# Add the source directory to the Python path
+package_root = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(package_root, 'src', 'sentencepiece'))
 sys.path.append(os.path.join('.', 'test'))
 
+# Import version directly from the package
+from _version import __version__
+
 
 def long_description():
   with codecs.open('README.md', 'r', 'utf-8') as f:
     long_description = f.read()
   return long_description
 
 
-exec(open('src/sentencepiece/_version.py').read())
-
-
 def run_pkg_config(section, pkg_config_path=None):
   try:
     cmd = 'pkg-config sentencepiece --{}'.format(section)
@@ -192,6 +195,7 @@ def get_win_arch():
     license='Apache',
     platforms='Unix',
     py_modules=[
+        'sentencepiece/_init',
         'sentencepiece/__init__',
         'sentencepiece/_version',
         'sentencepiece/sentencepiece_model_pb2',