From e25819317e1f74757f7c3c6bfdca816e1b8cea66 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 15 Dec 2023 16:38:37 -0500 Subject: [PATCH] [Backport release-0.27] Embedded 2.18.3 (#637) * Embedded 2.18.3 (#635) * Migrate macOS CI to GitHub Actions --------- Co-authored-by: Adam Wenocur Co-authored-by: John Blischak --- .azure-pipelines.yml | 10 - .github/workflows/macos.yml | 195 ++++++++++++++++++ .../cmake/Modules/FindTileDB_EP.cmake | 20 +- 3 files changed, 205 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/macos.yml diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 9633ed273..bff284770 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -32,16 +32,6 @@ stages: BUILD_SPARK_API: OFF DOWNLOAD_TILEDB_PREBUILT: OFF ARTIFACT_NAME: linux_tiledb_source_build - macos: - imageName: 'macOS-12' - python.version: '3.7' - CXX: clang++ - BUILD_PYTHON_API: ON - BUILD_SPARK_API: ON - DOWNLOAD_TILEDB_PREBUILT: ON - #SDKROOT: '/Applications/Xcode_10.3.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk' - ARTIFACT_NAME: macos - MACOSX_DEPLOYMENT_TARGET: '11' pool: vmImage: $(imageName) steps: diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml new file mode 100644 index 000000000..3cb56558a --- /dev/null +++ b/.github/workflows/macos.yml @@ -0,0 +1,195 @@ +name: macOS +on: + push: + paths: + - '.github/workflows/macos.yml' + - 'apis/**' + - 'libtiledbvcf/**' + pull_request: + paths: + - '.github/workflows/macos.yml' + - 'apis/**' + - 'libtiledbvcf/**' + workflow_dispatch: +env: + MACOSX_DEPLOYMENT_TARGET: "11" +jobs: + libtiledbvcf: + # Stuck on macos-11 because of the htslib dependency. The htslib + # configuration step fails on macos-12 and macos-13 + runs-on: macos-11 + steps: + - uses: actions/checkout@v3 + - name: Configure libtiledbvcf + run: | + cmake -S libtiledbvcf -B $(pwd)/libtiledbvcf/build \ + -D CMAKE_BUILD_TYPE=Debug \ + -D CMAKE_INSTALL_PREFIX=$(pwd)/dist \ + -D OVERRIDE_INSTALL_PREFIX=OFF \ + -D DOWNLOAD_TILEDB_PREBUILT=ON + - name: Build libtiledbvcf + run: cmake --build $(pwd)/libtiledbvcf/build -j 2 --config Debug + - name: Upload coredump as artifact if build failed + if: failure() + uses: actions/upload-artifact@v3 + with: + name: coredump + path: libtiledbvcf/build/core + retention-days: 14 + if-no-files-found: error + - name: Install libtiledbvcf + run: cmake --build $(pwd)/libtiledbvcf/build --config Debug --target install-libtiledbvcf + - name: Upload libtiledbvcf as artifact + uses: actions/upload-artifact@v3 + with: + name: libtiledbvcf + path: dist/* + retention-days: 14 + if-no-files-found: error + - name: Confirm linking + run: otool -L dist/lib/libtiledbvcf.dylib + - name: libtiledbvcf version + run: ./dist/bin/tiledbvcf version + - name: Install bcftools + run: brew install bcftools + - name: Unit tests + run: | + make -j 2 -C libtiledbvcf/build/libtiledbvcf tiledb_vcf_unit + ./libtiledbvcf/build/libtiledbvcf/test/tiledb_vcf_unit + - name: CLI tests (require bcftools) + run: | + # USAGE: run-cli-tests.sh + libtiledbvcf/test/run-cli-tests.sh libtiledbvcf/build libtiledbvcf/test/inputs + python: + runs-on: macos-11 + needs: libtiledbvcf + env: + DYLD_LIBRARY_PATH: "${{ github.workspace }}/dist/lib" + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # fetch everything for python setuptools_scm + - name: Download libtiledbvcf artifact + uses: actions/download-artifact@v3 + with: + name: libtiledbvcf + path: dist + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Install dependencies + run: python -m pip install --prefer-binary \ + dask \ + distributed \ + 'fsspec<2023.3.0' \ + pandas \ + pyarrow==10.0.1 \ + pyarrow-hotfix \ + pybind11 \ + pytest \ + setuptools \ + setuptools_scm \ + setuptools_scm_git_archive \ + wheel + - name: Build tiledbvcf-py + env: + LIBTILEDBVCF_PATH: "${{ github.workspace }}/dist" + run: | + echo $DYLD_LIBRARY_PATH + cd apis/python + python -m pip install . + - name: Confirm linking + run: otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/libtiledbvcf.cpython-*-darwin.so + java: + runs-on: macos-11 + needs: libtiledbvcf + steps: + - uses: actions/checkout@v3 + - name: Download libtiledbvcf artifact + uses: actions/download-artifact@v3 + with: + name: libtiledbvcf + path: dist + - name: Check format + run: cd apis/java && ./gradlew checkFormat + - name: Assemble + run: cd apis/java && ./gradlew assemble + - name: Test + run: cd apis/java && ./gradlew test + spark: + runs-on: macos-11 + needs: libtiledbvcf + steps: + - uses: actions/checkout@v3 + - name: Download libtiledbvcf artifact + uses: actions/download-artifact@v3 + with: + name: libtiledbvcf + path: dist + - name: Assemble + run: cd apis/spark && ./gradlew assemble + - name: Jar + run: cd apis/spark && ./gradlew jar + - name: Test + run: cd apis/spark && ./gradlew test + - name: Check format + run: cd apis/spark && ./gradlew checkFormat + spark3: + runs-on: macos-11 + needs: libtiledbvcf + steps: + - uses: actions/checkout@v3 + - name: Download libtiledbvcf artifact + uses: actions/download-artifact@v3 + with: + name: libtiledbvcf + path: dist + - name: Assemble + run: cd apis/spark3 && ./gradlew assemble + - name: Jar + run: cd apis/spark3 && ./gradlew jar + - name: Test + run: cd apis/spark3 && ./gradlew test + - name: Check format + run: cd apis/spark3 && ./gradlew checkFormat + python-standalone: + runs-on: macos-11 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # fetch everything for python setuptools_scm + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Install dependencies + run: python -m pip install --prefer-binary \ + dask \ + distributed \ + 'fsspec<2023.3.0' \ + pandas \ + pyarrow==10.0.1 \ + pyarrow-hotfix \ + pybind11 \ + pytest \ + setuptools \ + setuptools_scm \ + setuptools_scm_git_archive \ + wheel + - name: Build tiledbvcf-py + run: cd apis/python && python -m pip install . + - name: Confirm linking + run: | + otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/libtiledbvcf.cpython-*-darwin.so + otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/libtiledbvcf.dylib + otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/libtiledb.dylib + otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/libhts.*.dylib + - name: Version + run: python -c "import tiledbvcf; print(tiledbvcf.version)" + - name: Install bcftools (for tests) + run: brew install bcftools + - name: Install tiledb-py + run: pip install tiledb + - name: Test tiledbvcf-py + run: cd apis/python && pytest diff --git a/libtiledbvcf/cmake/Modules/FindTileDB_EP.cmake b/libtiledbvcf/cmake/Modules/FindTileDB_EP.cmake index 451724790..c1833296a 100644 --- a/libtiledbvcf/cmake/Modules/FindTileDB_EP.cmake +++ b/libtiledbvcf/cmake/Modules/FindTileDB_EP.cmake @@ -52,20 +52,20 @@ else() # Try to download prebuilt artifacts unless the user specifies to build from source if(DOWNLOAD_TILEDB_PREBUILT) if (WIN32) # Windows - SET(DOWNLOAD_URL "https://github.com/TileDB-Inc/TileDB/releases/download/2.18.2/tiledb-windows-x86_64-2.18.2-9ae6e1a.zip") - SET(DOWNLOAD_SHA1 "c440403fde701e5014a7ffa046d38bb49b141e55") + SET(DOWNLOAD_URL "https://github.com/TileDB-Inc/TileDB/releases/download/2.18.3/tiledb-windows-x86_64-2.18.3-8f8b766.zip") + SET(DOWNLOAD_SHA1 "313cc237c8cab22edaf2745014471d231f185926") elseif(APPLE) # OSX if (CMAKE_OSX_ARCHITECTURES STREQUAL x86_64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(AMD64|amd64)|(^i.86$)") - SET(DOWNLOAD_URL "https://github.com/TileDB-Inc/TileDB/releases/download/2.18.2/tiledb-macos-x86_64-2.18.2-9ae6e1a.tar.gz") - SET(DOWNLOAD_SHA1 "6ba986407b2d0e25fb5df19d08ed22fefeac1c2a") + SET(DOWNLOAD_URL "https://github.com/TileDB-Inc/TileDB/releases/download/2.18.3/tiledb-macos-x86_64-2.18.3-8f8b766.tar.gz") + SET(DOWNLOAD_SHA1 "fbd7515172de977c8ebd553427b7c136a73df299") elseif (CMAKE_OSX_ARCHITECTURES STREQUAL arm64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "^arm") - SET(DOWNLOAD_URL "https://github.com/TileDB-Inc/TileDB/releases/download/2.18.2/tiledb-macos-arm64-2.18.2-9ae6e1a.tar.gz") - SET(DOWNLOAD_SHA1 "4395a283db57844a944f02dab5bd0f0001eb7e30") + SET(DOWNLOAD_URL "https://github.com/TileDB-Inc/TileDB/releases/download/2.18.3/tiledb-macos-arm64-2.18.3-8f8b766.tar.gz") + SET(DOWNLOAD_SHA1 "8109bbac66824ee5806cbea93386d66f5b874bc2") endif() else() # Linux - SET(DOWNLOAD_URL "https://github.com/TileDB-Inc/TileDB/releases/download/2.18.2/tiledb-linux-x86_64-2.18.2-9ae6e1a.tar.gz") - SET(DOWNLOAD_SHA1 "6386504a58d52330f41fa688a2b259b67824e2c8") + SET(DOWNLOAD_URL "https://github.com/TileDB-Inc/TileDB/releases/download/2.18.3/tiledb-linux-x86_64-2.18.3-8f8b766.tar.gz") + SET(DOWNLOAD_SHA1 "22ccf17c67532ca80fe2c6f06350a267bc839e81") endif() ExternalProject_Add(ep_tiledb @@ -87,8 +87,8 @@ else() else() # Build from source ExternalProject_Add(ep_tiledb PREFIX "externals" - URL "https://github.com/TileDB-Inc/TileDB/archive/2.18.2.zip" - URL_HASH SHA1=70bb2e9a7603675849830b763d1a0d17fce4bafa + URL "https://github.com/TileDB-Inc/TileDB/archive/2.18.3.zip" + URL_HASH SHA1=12edfe14b78f1d0119d2e4229daf83a06d37e440 DOWNLOAD_NAME "tiledb.zip" CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EP_INSTALL_PREFIX}