Migrate from hyperscan to vectorscan fork

gliwka · Nov 24, 2023 · 9c49062 · 9c49062
1 parent b0d5c9a
commit 9c49062
Show file tree

Hide file tree

Showing 5 changed files with 56 additions and 64 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -18,48 +18,51 @@ jobs:
   build-native:
     name: Build native libs
     needs: create-staging-repo
-    runs-on: ${{ matrix.os }}
+    runs-on: ${{ matrix.runner }}
     defaults:
       run:
         shell: ${{ matrix.shell }}
     strategy:
+      fail-fast: false
       matrix:
         include:
-          - os: windows-latest
-            shell: msys2 {0}
-            platform: windows-x86_64
-          - os: macos-latest
+          - os: macos
+            runner: macos-12
             shell: bash
             platform: macosx-x86_64
-          - os: macos-latest
+          - os: macos
+            runner: macos-12-large
             shell: bash
             platform: macosx-arm64
-          - os: ubuntu-latest
+          - os: linux
+            runner: ubuntu-latest
             shell: bash
             platform: linux-x86_64
+          - os: linux
+            runner: buildjet-4vcpu-ubuntu-2204-arm
+            shell: bash
+            platform: linux-arm64
     env:
-      SIGN_KEY: ${{ secrets.SIGN_KEY }}
-      SIGN_KEY_PASS: ${{ secrets.SIGN_KEY_PASS }}
       STAGING_REPO: ${{ needs.create-staging-repo.outputs.staging-repo }}
       DETECTED_PLATFORM: ${{ matrix.platform }}
-      SDKROOT: /Library/Developer/CommandLineTools/SDKs/MacOSX11.1.sdk
       STAGING: ${{ startsWith(github.ref, 'refs/tags/v') }}
     steps:
       - uses: actions/checkout@v2
-      - uses: msys2/setup-msys2@v2
-        if: matrix.os == 'windows-latest'
-        with:
-          install: base-devel gcc
-          path-type: inherit
       - uses: ilammy/msvc-dev-cmd@v1
-      - name: Switch XCode-Version
-        if: matrix.os == 'macos-latest'
-        run: sudo xcode-select -switch /Applications/Xcode_12.2.app
+      - uses: maxim-lobanov/setup-xcode@v1
+        if: matrix.os == 'macos'
+        with:
+          xcode-version: latest-stable
+      - name: Set signing keys
+        if: startsWith(github.ref, 'refs/tags/v') == true
+        run: |
+          echo "SIGN_KEY=${{ secrets.SIGN_KEY }}" >> "$GITHUB_ENV"
+          echo "SIGN_KEY_PASS=${{ secrets.SIGN_KEY_PASS }}" >> "$GITHUB_ENV"
       - name: Build native binaries
-        run: bash build.sh deploy
-        if: matrix.os != 'ubuntu-latest'
+        run: ./build.sh deploy
+        if: matrix.os != 'linux'
       - name: Build native binaries in centos container
-        if: matrix.os == 'ubuntu-latest'
-        uses: docker://ghcr.io/gliwka/centos6-devtoolset7:main
+        if: matrix.os == 'linux'
+        uses: docker://ghcr.io/gliwka/centos7-toolchain:main@sha256:e6cf17223408bb4d1314be547d74b23d04bcf6480d2fe23822fb672a52775f76
         with:
-          args: build.sh deploy
+          args: ./build.sh deploy
diff --git a/.gitignore b/.gitignore
@@ -34,3 +34,5 @@ cppbuild/
 
 #test artifacts
 db
+
+.DS_Store
diff --git a/build.sh b/build.sh
@@ -4,8 +4,8 @@
 set -xeu
 set -o pipefail
 
-HYPERSCAN=5.4.0
-VECTORSCAN=af8c6d375cba32a50a9e575de83807d96a0fb503
+VERSION="5.4.11"
+SHA256="905f76ad1fa9e4ae0eb28232cac98afdb96c479666202c5a4c27871fb30a2711"
 
 detect_platform() {
   # use os-maven-plugin to detect platform
@@ -21,7 +21,7 @@ export DETECTED_PLATFORM=${DETECTED_PLATFORM:-$(detect_platform)}
 cross_platform_nproc() {
   case $DETECTED_PLATFORM in
     macosx-x86_64|macosx-arm64) echo $(sysctl -n hw.logicalcpu) ;;
-    linux-x86_64|windows-x86_64) echo $(nproc --all) ;;
+    linux-x86_64|linux-arm64) echo $(nproc --all) ;;
     *) echo Unsupported Platform: $DETECTED_PLATFORM >&2 ; exit -1 ;;
   esac
 }
@@ -31,7 +31,7 @@ cross_platform_check_sha() {
   local file=$2
   case $DETECTED_PLATFORM in
     macosx-x86_64|macosx-arm64) echo "$sha  $file" | shasum -a 256 -c ;;
-    linux-x86_64|windows-x86_64) echo "$sha  $file" | sha256sum -c ;;
+    linux-x86_64|linux-arm64) echo "$sha  $file" | sha256sum -c ;;
     *) echo Unsupported Platform: $DETECTED_PLATFORM >&2 ; exit -1 ;;
   esac
 }
@@ -43,29 +43,19 @@ mkdir -p cppbuild/bin
 mkdir -p cppbuild/include/hs
 cd cppbuild
 
-# use own vectorscan fork with clang patches for apple silicon macs
-if [ $DETECTED_PLATFORM = "macosx-arm64" ]
-then
-  curl -L -o vectorscan-$VECTORSCAN.zip https://github.com/gliwka/vectorscan/archive/$VECTORSCAN.zip
-  cross_platform_check_sha \
-    5ba9b3766b92371324a83fc2fc47d5005bbed49bc6ba601e2e3ca679d74a150e \
-    vectorscan-$VECTORSCAN.zip
-  unzip vectorscan-$VECTORSCAN.zip
-  mv vectorscan-$VECTORSCAN hyperscan-$HYPERSCAN
-else
- curl -L -o hyperscan-$HYPERSCAN.tar.gz https://github.com/intel/hyperscan/archive/v$HYPERSCAN.tar.gz
-  cross_platform_check_sha \
-    e51aba39af47e3901062852e5004d127fa7763b5dbbc16bcca4265243ffa106f \
-    hyperscan-$HYPERSCAN.tar.gz
-  tar -zxf hyperscan-$HYPERSCAN.tar.gz
-fi
+curl -L -o vectorscan-$VERSION.tar.gz https://github.com/VectorCamp/vectorscan/archive/refs/tags/vectorscan/5.4.11.tar.gz
+cross_platform_check_sha \
+  $SHA256 \
+  vectorscan-$VERSION.tar.gz
+tar -xvf vectorscan-$VERSION.tar.gz
+mv vectorscan-vectorscan-$VERSION vectorscan
 
 curl -L -o boost_1_74_0.tar.gz https://boostorg.jfrog.io/artifactory/main/release/1.74.0/source/boost_1_74_0.tar.gz
 cross_platform_check_sha \
   afff36d392885120bcac079148c177d1f6f7730ec3d47233aa51b0afa4db94a5 \
   boost_1_74_0.tar.gz
 tar -zxf boost_1_74_0.tar.gz
-mv boost_1_74_0/boost hyperscan-$HYPERSCAN/include/boost
+mv boost_1_74_0/boost vectorscan/include/boost
 
 curl -L -o ragel-6.10.tar.gz https://www.colm.net/files/ragel/ragel-6.10.tar.gz
 cross_platform_check_sha \
@@ -79,31 +69,27 @@ make -j $THREADS
 make install
 cd ..
 
-cd hyperscan-$HYPERSCAN
+cd vectorscan
+
+# Disable flakey sqlite detection - only needed to build auxillary tools anyways.
+> cmake/sqlite3.cmake
 
 case $DETECTED_PLATFORM in
 linux-x86_64)
-  CFLAGS='-O -fPIC' CC="gcc" CXX="g++ -std=c++11 -m64 -fPIC" cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$(pwd)/.." -DCMAKE_INSTALL_LIBDIR="lib" -DPCRE_SOURCE="." .
-  make -j $THREADS hs hs_runtime hs_compile
+  cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$(pwd)/.." -DCMAKE_INSTALL_LIBDIR="lib" -DPCRE_SOURCE="." -DFAT_RUNTIME=on -DBUILD_SHARED_LIBS=on -DBUILD_AVX2=yes -DBUILD_AVX512=yes -DBUILD_AVX512VBMI=yes .
+  make -j $THREADS
   make install/strip
   ;;
-macosx-x86_64)
-  cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$(pwd)/.." -DCMAKE_INSTALL_LIBDIR="lib" -DARCH_OPT_FLAGS='-Wno-error' -DPCRE_SOURCE="." .
-  make -j $THREADS hs hs_runtime hs_compile
+linux-arm64)
+  CC="clang" CXX="clang++" cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$(pwd)/.." -DCMAKE_INSTALL_LIBDIR="lib" -DPCRE_SOURCE="." -DFAT_RUNTIME=on -DBUILD_SHARED_LIBS=on .
+  make -j $THREADS
   make install/strip
   ;;
-macosx-arm64)
-  CFLAGS="-target arm64-apple-macos11" CXXFLAGS="-target arm64-apple-macos11" cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$(pwd)/.." -DCMAKE_INSTALL_LIBDIR="lib" -DARCH_OPT_FLAGS='-Wno-error' -DPCRE_SOURCE="." .
-  make -j $THREADS hs hs_runtime hs_compile
+macosx-x86_64|macosx-arm64)
+  cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$(pwd)/.." -DCMAKE_INSTALL_LIBDIR="lib" -DARCH_OPT_FLAGS='-Wno-error' -DPCRE_SOURCE="." -DBUILD_SHARED_LIBS=on .
+  make -j $THREADS
   make install/strip
   ;;
-windows-x86_64)
-  unset TEMP TMP # temp is defined in uppercase by bash and lowercase by windows, which causes problems with cmake + msbuild
-  CXXFLAGS="/Wv:17" cmake -G "Visual Studio 16 2019" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$(pwd)/.." -DCMAKE_INSTALL_LIBDIR="lib" -DARCH_OPT_FLAGS='' -DPCRE_SOURCE="." .
-  MSBuild.exe hyperscan.sln //p:Configuration=Release //p:Platform=x64
-  cp -r src/* ../include/hs/
-  cp lib/*.lib ../lib
-  ;;
 *)
   echo "Error: Arch \"$DETECTED_PLATFORM\" is not supported"
   ;;
@@ -117,4 +103,5 @@ then
   mvn -B -Dorg.bytedeco.javacpp.platform=$DETECTED_PLATFORM --settings mvnsettings.xml deploy
 else
   mvn -B -Dorg.bytedeco.javacpp.platform=$DETECTED_PLATFORM install
-fi
+fi
+
diff --git a/pom.xml b/pom.xml
@@ -100,8 +100,8 @@
                 <artifactId>maven-compiler-plugin</artifactId>
                 <version>3.8.1</version>
                 <configuration>
-                    <source>1.7</source>
-                    <target>1.7</target>
+                    <source>1.8</source>
+                    <target>1.8</target>
                 </configuration>
                 <executions>
                     <execution>

diff --git a/src/main/java/com/gliwka/hyperscan/jni/JavaCppPreset.java b/src/main/java/com/gliwka/hyperscan/jni/JavaCppPreset.java
@@ -35,7 +35,7 @@
         value = {
                 @Platform(
                         define = "NO_JNI_DETACH_THREAD", //JNI callback on same thread
-                        value = {"linux-x86_64", "macosx-x86_64", "macosx-arm64", "windows-x86_64"},
+                        value = {"linux-x86_64", "linux-arm64", "macosx-x86_64", "macosx-arm64"},
                         compiler = "cpp11",
                         include = {"hs/hs_common.h", "hs/hs_compile.h", "hs/hs_runtime.h", "hs/hs.h"},
                         link = {"hs", "hs_runtime"}
Original file line number	Diff line number	Diff line change
Expand Up		@@ -34,3 +34,5 @@ cppbuild/

		#test artifacts
		db

		.DS_Store