From e3013419f1e4673cb45f90d77c14cddc366c1c94 Mon Sep 17 00:00:00 2001 From: luncliff Date: Wed, 20 Dec 2023 01:54:55 +0900 Subject: [PATCH] ci: overhaul for end of 2023 * azure-pipelines: remove Windows jobs * gh-actions: update test manifests * gh-actions: update versions of applications * readme: update to later vcpkg release --- .github/workflows/build-macos.yml | 10 +- .github/workflows/build-ubuntu.yml | 10 +- .github/workflows/build-windows.yml | 12 +- .github/workflows/gh-pages.yml | 10 +- README.md | 2 +- azure-pipelines.yml | 171 ++++++++-------------------- test/self-hosted-cuda.json | 16 +-- test/vcpkg.json | 23 +++- 8 files changed, 97 insertions(+), 157 deletions(-) diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index 65829bd4..8be9f44c 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -18,21 +18,21 @@ jobs: VCPKG_OVERLAY_PORTS: "${{ github.workspace }}/ports" VCPKG_OVERLAY_TRIPLETS: ${{ github.workspace }}/triplets steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4.1.1 - name: "create cache folders" run: | mkdir -p ${VCPKG_DOWNLOADS} mkdir -p ${VCPKG_DEFAULT_BINARY_CACHE} - - uses: actions/cache@v3 + - uses: actions/cache@v3.3.2 with: - key: "v2339-${{ runner.os }}-${{ matrix.triplet }}" + key: "v2351-${{ runner.os }}-${{ matrix.triplet }}" path: | /usr/local/share/vcpkg-downloads /usr/local/share/vcpkg-archives - - uses: ConorMacBride/install-package@v1 + - uses: ConorMacBride/install-package@v1.1.0 with: brew: ninja autoconf automake libtool - - uses: lukka/run-vcpkg@v11 + - uses: lukka/run-vcpkg@v11.3 with: vcpkgDirectory: "/usr/local/share/vcpkg" # see VCPKG_INSTALLATION_ROOT vcpkgGitCommitId: "a42af01b72c28a8e1d7b48107b33e4f286a55ef6" # 2023.11.20 diff --git a/.github/workflows/build-ubuntu.yml b/.github/workflows/build-ubuntu.yml index 6445d3a8..9919834c 100644 --- a/.github/workflows/build-ubuntu.yml +++ b/.github/workflows/build-ubuntu.yml @@ -17,7 +17,7 @@ jobs: VCPKG_OVERLAY_PORTS: "${{ github.workspace }}/ports" VCPKG_OVERLAY_TRIPLETS: ${{ github.workspace }}/triplets steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4.1.1 - name: "create cache folders" run: | mkdir -p ${VCPKG_DOWNLOADS} @@ -25,16 +25,16 @@ jobs: env: VCPKG_DOWNLOADS: "${{ runner.temp }}/vcpkg-downloads" VCPKG_DEFAULT_BINARY_CACHE: "${{ runner.temp }}/vcpkg-archives" - - uses: actions/cache@v3 + - uses: actions/cache@v3.3.2 with: - key: "v2339-${{ runner.os }}-${{ matrix.triplet }}" + key: "v2351-${{ runner.os }}-${{ matrix.triplet }}" path: | ${{ runner.temp }}/vcpkg-downloads ${{ runner.temp }}/vcpkg-archives - - uses: ConorMacBride/install-package@v1 + - uses: ConorMacBride/install-package@v1.1.0 with: apt: nuget nasm libnuma-dev libopenmpi-dev libx11-dev libxi-dev libxext-dev libx11-xcb-dev - - uses: lukka/run-vcpkg@v11 + - uses: lukka/run-vcpkg@v11.3 with: vcpkgDirectory: "/usr/local/share/vcpkg" # see VCPKG_INSTALLATION_ROOT doNotUpdateVcpkg: true diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 5b46aae9..e37bb0aa 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -22,13 +22,13 @@ jobs: run: | New-Item -Type Directory -Force ${env:VCPKG_DOWNLOADS} New-Item -Type Directory -Force ${env:VCPKG_DEFAULT_BINARY_CACHE} - - uses: actions/cache@v3 + - uses: actions/cache@v3.3.2 with: - key: "v2339-${{ runner.os }}-${{ matrix.triplet }}" + key: "v2351-${{ runner.os }}-${{ matrix.triplet }}" path: | C:/vcpkg/downloads C:/vcpkg/archives - - uses: microsoft/setup-msbuild@v1.1 + - uses: microsoft/setup-msbuild@v1.3.1 with: msbuild-architecture: x64 - uses: lukka/run-vcpkg@v11.3 @@ -44,7 +44,7 @@ jobs: cuda_x64: runs-on: ["self-hosted", "Windows", "CUDA"] continue-on-error: true - timeout-minutes: 60 + timeout-minutes: 180 env: VCPKG_DOWNLOADS: "C:/vcpkg/downloads" VCPKG_DEFAULT_BINARY_CACHE: "C:/vcpkg/archives" @@ -57,7 +57,7 @@ jobs: New-Item -Type Directory -Force ${env:VCPKG_DOWNLOADS} New-Item -Type Directory -Force ${env:VCPKG_DEFAULT_BINARY_CACHE} Move-Item -Path "test/self-hosted-cuda.json" -Destination "test/vcpkg.json" -Force - - uses: microsoft/setup-msbuild@v1.1 + - uses: microsoft/setup-msbuild@v1.3.1 with: msbuild-architecture: x64 - uses: lukka/run-vcpkg@v11.3 @@ -70,6 +70,6 @@ jobs: env: VCPKG_DEFAULT_TRIPLET: "x64-windows" - - uses: yumis-coconudge/clean-workspace-action@v1 + - uses: yumis-coconudge/clean-workspace-action@v1.0.5 with: additional-path: "C:/vcpkg/installed" diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 981dafe2..f946a394 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -22,8 +22,8 @@ jobs: name: github-pages url: ${{ steps.deployment.outputs.page_url }} steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4.1.1 + - uses: actions/setup-python@v5.0.0 with: python-version: 3.11 architecture: x64 @@ -35,8 +35,8 @@ jobs: run: > python3 -m mkdocs build --site-dir _site - - uses: actions/configure-pages@v3 - - uses: actions/upload-pages-artifact@v2 + - uses: actions/configure-pages@v4.0.0 + - uses: actions/upload-pages-artifact@v3.0.0 with: path: '_site' - - uses: actions/deploy-pages@v2 + - uses: actions/deploy-pages@v4.0.0 diff --git a/README.md b/README.md index b44ac50c..5e0dd3ef 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Targets... -* [vcpkg](https://github.com/microsoft/vcpkg): recommend [2023.06.20](https://github.com/microsoft/vcpkg/releases/tag/2023.06.20) or later +* [vcpkg](https://github.com/microsoft/vcpkg): recommend [2023.10.19](https://github.com/microsoft/vcpkg/releases/tag/2023.10.19) or later * [vcpkg-tool](https://github.com/microsoft/vcpkg-tool) follows the vcpkg ### References diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d2b4eefd..631d0b86 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,7 +1,7 @@ # # https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema # https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables -# https://github.com/actions/virtual-environments/tree/main/images +# https://github.com/actions/runner-images/tree/main/images # https://github.com/lukka/CppBuildTasks # @@ -38,7 +38,7 @@ stages: - job: "triplet_ios_simulator" displayName: "iOS Simulator" pool: - vmImage: macOS-12 + vmImage: macOS-13 variables: - name: vcpkg.overlay.triplets # VCPKG_OVERLAY_TRIPLETS value: $(Build.SourcesDirectory)/triplets # --overlay-triplets $(Build.SourcesDirectory)/triplets @@ -47,16 +47,16 @@ stages: displayName: "Install HomeBrew packages" - powershell: New-Item -Type Directory -Force "$env:VCPKG_DEFAULT_BINARY_CACHE" - task: run-vcpkg@0 - displayName: "arm64-ios-simulator: zlib-ng" + displayName: "arm64-ios-simulator" inputs: - vcpkgArguments: "zlib-ng --debug" + vcpkgArguments: "zlib-ng" vcpkgGitCommitId: $(vcpkg.commit) env: VCPKG_DEFAULT_TRIPLET: arm64-ios-simulator - task: run-vcpkg@0 - displayName: "x64-ios-simulator: zlib-ng" + displayName: "x64-ios-simulator" inputs: - vcpkgArguments: "zlib-ng --debug" + vcpkgArguments: "zlib-ng" vcpkgGitCommitId: $(vcpkg.commit) env: VCPKG_DEFAULT_TRIPLET: x64-ios-simulator @@ -67,7 +67,7 @@ stages: - job: "osx_host_tools" displayName: "MacTools" pool: - vmImage: macOS-12 + vmImage: macOS-13 timeoutInMinutes: "120" steps: - powershell: brew install autoconf automake libtool pkg-config @@ -75,9 +75,9 @@ stages: - powershell: New-Item -Type Directory -Force "$env:VCPKG_DEFAULT_BINARY_CACHE" - task: Cache@2 inputs: - key: '"v2349-bin-mac-host"' + key: '"v2351-bin-mac-host"' restoreKeys: | - "v2349-bin-mac-host" + "v2351-bin-mac-host" path: $(vcpkg.default.binary.cache) - task: run-vcpkg@0 displayName: "coreml-tools" @@ -103,65 +103,6 @@ stages: - stage: "Ports" dependsOn: "Triplets" jobs: - - job: "port_windows" - displayName: "Windows" - pool: - vmImage: windows-2022 - strategy: - matrix: - x64_windows: - vcpkg.default.triplet: "x64-windows" - x86_windows: - vcpkg.default.triplet: "x86-windows" - x64_uwp: - vcpkg.default.triplet: "x64-uwp" - steps: - - powershell: New-Item -Type Directory -Force "$env:VCPKG_DEFAULT_BINARY_CACHE" - - task: Cache@2 - inputs: - key: '"v2349-down-windows"' - restoreKeys: | - "v2349-down-windows" - path: $(Build.BinariesDirectory)/vcpkg/downloads - - task: Cache@2 - inputs: - key: '"v2349-bin-windows" | "$(vcpkg.default.triplet)"' - restoreKeys: | - "v2349-bin-windows" | "$(vcpkg.default.triplet)" - path: $(vcpkg.default.binary.cache) - - task: run-vcpkg@0 - displayName: "libdispatch" - inputs: - vcpkgArguments: "libdispatch" - vcpkgGitCommitId: $(vcpkg.commit) - condition: ne(variables['vcpkg.default.triplet'], 'x64-uwp') - - task: run-vcpkg@0 - displayName: "tensorflow-lite" - inputs: - vcpkgArguments: "tensorflow-lite[google-angle,gpu]" - vcpkgGitCommitId: $(vcpkg.commit) - condition: ne(variables['vcpkg.default.triplet'], 'x64-uwp') - - task: run-vcpkg@0 - inputs: - vcpkgArguments: "cpuinfo" - vcpkgGitCommitId: $(vcpkg.commit) - - task: CopyFiles@2 - displayName: "Copy logs in buildtrees" - inputs: - SourceFolder: "$(Build.BinariesDirectory)/vcpkg/buildtrees/" - Contents: | - **/?(*.log|*.txt|*.cmake) - TargetFolder: "$(Build.ArtifactStagingDirectory)/log" - OverWrite: true - condition: always() - - task: PublishBuildArtifacts@1 - displayName: "Publish logs" - inputs: - PathtoPublish: "$(Build.ArtifactStagingDirectory)/log" - ArtifactName: "$(Agent.JobName)" - condition: always() - timeoutInMinutes: "500" - - job: "port_windows_arm" displayName: "Windows(ARM)" pool: @@ -176,24 +117,22 @@ stages: - powershell: New-Item -Type Directory -Force "$env:VCPKG_DEFAULT_BINARY_CACHE" - task: Cache@2 inputs: - key: '"v2349-down-uwp"' + key: '"v2351-down-uwp"' restoreKeys: | - "v2349-down-uwp" + "v2351-down-uwp" path: $(Build.BinariesDirectory)/vcpkg/downloads - task: Cache@2 inputs: - key: '"v2349-bin-uwp" | "$(vcpkg.default.triplet)"' + key: '"v2351-bin-uwp" | "$(vcpkg.default.triplet)"' restoreKeys: | - "v2349-bin-uwp" | "$(vcpkg.default.triplet)" + "v2351-bin-uwp" | "$(vcpkg.default.triplet)" path: $(vcpkg.default.binary.cache) - task: run-vcpkg@0 - displayName: "libdispatch" inputs: vcpkgArguments: "libdispatch" vcpkgGitCommitId: $(vcpkg.commit) condition: ne(variables['vcpkg.default.triplet'], 'arm64-uwp') - task: run-vcpkg@0 - displayName: "tensorflow-lite" inputs: vcpkgArguments: "tensorflow-lite" vcpkgGitCommitId: $(vcpkg.commit) @@ -220,39 +159,33 @@ stages: vmImage: ubuntu-22.04 strategy: matrix: - x64_clang14: + x64_clang15: vcpkg.default.triplet: "x64-linux" - cc: clang-14 - cxx: clang++-14 - x64_gcc11: + cc: clang-15 + cxx: clang++-15 + x64_gcc13: vcpkg.default.triplet: "x64-linux" - cc: gcc-11 - cxx: g++-11 + cc: gcc-13 + cxx: g++-13 steps: - powershell: sudo apt-get install -y nuget nasm libnuma-dev libopenmpi-dev libx11-dev libxi-dev libxext-dev libx11-xcb-dev displayName: "Install APT packages" - powershell: New-Item -Type Directory -Force "$env:VCPKG_DEFAULT_BINARY_CACHE" - task: Cache@2 inputs: - key: '"v2349-down-ubuntu"' + key: '"v2351-down-ubuntu"' restoreKeys: | - "v2349-down-ubuntu" + "v2351-down-ubuntu" path: $(Build.BinariesDirectory)/vcpkg/downloads - task: Cache@2 inputs: - key: '"v2349-bin-ubuntu" | "$(vcpkg.default.triplet)_$(cc)"' + key: '"v2351-bin-ubuntu" | "$(vcpkg.default.triplet)_$(cc)"' restoreKeys: | - "v2349-bin-ubuntu" | "$(vcpkg.default.triplet)_$(cc)" + "v2351-bin-ubuntu" | "$(vcpkg.default.triplet)_$(cc)" path: $(vcpkg.default.binary.cache) - task: run-vcpkg@0 - displayName: "tensorflow-lite" - inputs: - vcpkgArguments: "tensorflow-lite[gpu]" - vcpkgGitCommitId: $(vcpkg.commit) - - task: run-vcpkg@0 - displayName: "onnxruntime" inputs: - vcpkgArguments: "onnxruntime[xnnpack]" + vcpkgArguments: "tensorflow-lite[gpu] onnxruntime[xnnpack]" vcpkgGitCommitId: $(vcpkg.commit) - task: CopyFiles@2 displayName: "Copy logs in buildtrees" @@ -277,29 +210,29 @@ stages: vmImage: ubuntu-22.04 strategy: matrix: - arm64_ndk25: + arm64_ndk26: vcpkg.default.triplet: "arm64-android" - android.ndk.home: "/usr/local/lib/android/sdk/ndk/25.2.9519653" - x64_ndk25: + android.ndk.home: "/usr/local/lib/android/sdk/ndk/26.1.10909125" + x64_ndk26: vcpkg.default.triplet: "x64-android" - android.ndk.home: "/usr/local/lib/android/sdk/ndk/25.2.9519653" + android.ndk.home: "/usr/local/lib/android/sdk/ndk/26.1.10909125" steps: - powershell: New-Item -Type Directory -Force "$env:VCPKG_DEFAULT_BINARY_CACHE" - task: Cache@2 inputs: - key: '"v2349-down-android"' + key: '"v2351-down-android"' restoreKeys: | - "v2349-down-android" + "v2351-down-android" path: $(Build.BinariesDirectory)/vcpkg/downloads - task: Cache@2 inputs: - key: '"v2349-bin-android" | "$(vcpkg.default.triplet)"' + key: '"v2351-bin-android" | "$(vcpkg.default.triplet)"' restoreKeys: | - "v2349-bin-android" | "$(vcpkg.default.triplet)" + "v2351-bin-android" | "$(vcpkg.default.triplet)" path: $(vcpkg.default.binary.cache) - task: run-vcpkg@0 inputs: - vcpkgArguments: "openssl3 libdispatch fft2d farmhash ruy eigen3 google-jni-bind" + vcpkgArguments: "openssl3 libdispatch google-jni-bind tensorflow-lite[gpu]" vcpkgGitCommitId: $(vcpkg.commit) env: VCPKG_OVERLAY_TRIPLETS: $(Build.SourcesDirectory)/triplets @@ -323,7 +256,7 @@ stages: - job: "port_osx" displayName: "Mac" pool: - vmImage: macOS-12 + vmImage: macOS-13 strategy: matrix: arm64_osx: @@ -344,29 +277,19 @@ stages: - powershell: New-Item -Type Directory -Force "$env:VCPKG_DEFAULT_BINARY_CACHE" - task: Cache@2 inputs: - key: '"v2349-down-apple"' + key: '"v2351-down-apple"' restoreKeys: | - "v2349-down-apple" + "v2351-down-apple" path: $(Build.BinariesDirectory)/vcpkg/downloads - task: Cache@2 inputs: - key: '"v2349-bin-osx" | "$(vcpkg.default.triplet)"' + key: '"v2351-bin-osx" | "$(vcpkg.default.triplet)"' restoreKeys: | - "v2349-bin-osx" | "$(vcpkg.default.triplet)" + "v2351-bin-osx" | "$(vcpkg.default.triplet)" path: $(vcpkg.default.binary.cache) - task: run-vcpkg@0 inputs: - vcpkgArguments: "metal-cpp" - vcpkgGitCommitId: $(vcpkg.commit) - - task: run-vcpkg@0 - displayName: "tensorflow-lite" - inputs: - vcpkgArguments: "tensorflow-lite[gpu]" - vcpkgGitCommitId: $(vcpkg.commit) - - task: run-vcpkg@0 - displayName: "onnxruntime" - inputs: - vcpkgArguments: "onnxruntime[xnnpack,coreml]" + vcpkgArguments: "tensorflow-lite[gpu] onnxruntime[xnnpack,coreml]" vcpkgGitCommitId: $(vcpkg.commit) - task: CopyFiles@2 displayName: "Copy logs in buildtrees" @@ -388,7 +311,7 @@ stages: - job: "port_ios" displayName: "iOS" pool: - vmImage: macOS-12 + vmImage: macOS-13 strategy: matrix: arm64_ios: @@ -399,34 +322,34 @@ stages: - powershell: New-Item -Type Directory -Force "$env:VCPKG_DEFAULT_BINARY_CACHE" - task: Cache@2 inputs: - key: '"v2349-down-apple"' + key: '"v2351-down-apple"' restoreKeys: | - "v2349-down-apple" + "v2351-down-apple" path: $(Build.BinariesDirectory)/vcpkg/downloads - task: Cache@2 inputs: - key: '"v2349-bin-ios" | "$(vcpkg.default.triplet)"' + key: '"v2351-bin-ios" | "$(vcpkg.default.triplet)"' restoreKeys: | - "v2349-bin-ios" | "$(vcpkg.default.triplet)" + "v2351-bin-ios" | "$(vcpkg.default.triplet)" path: $(vcpkg.default.binary.cache) - task: run-vcpkg@0 displayName: "arm64-ios" inputs: - vcpkgArguments: "coreml-tools tensorflow-lite metal-cpp" + vcpkgArguments: "tensorflow-lite" vcpkgGitCommitId: $(vcpkg.commit) env: VCPKG_DEFAULT_TRIPLET: arm64-ios - task: run-vcpkg@0 displayName: "arm64-ios-simulator" inputs: - vcpkgArguments: "coreml-tools tensorflow-lite" + vcpkgArguments: "tensorflow-lite" vcpkgGitCommitId: $(vcpkg.commit) env: VCPKG_DEFAULT_TRIPLET: arm64-ios-simulator - task: run-vcpkg@0 displayName: "x64-ios-simulator" inputs: - vcpkgArguments: "xnnpack" + vcpkgArguments: "tensorflow-lite" vcpkgGitCommitId: $(vcpkg.commit) env: VCPKG_DEFAULT_TRIPLET: x64-ios-simulator diff --git a/test/self-hosted-cuda.json b/test/self-hosted-cuda.json index 29d175a7..33ba77d1 100644 --- a/test/self-hosted-cuda.json +++ b/test/self-hosted-cuda.json @@ -1,6 +1,6 @@ { "name": "test", - "version-date": "2023-12-09", + "version-date": "2023-12-20", "description": "vcpkg registry maintained by @luncliff", "homepage": "https://github.com/luncliff/vcpkg-registry", "supports": "windows", @@ -8,16 +8,12 @@ { "name": "onnxruntime", "features": [ - "cuda" - ], - "platform": "x64 & windows" - }, - { - "name": "opencv4", - "features": [ - "cudnn" + "cuda", + "directml", + "python", + "xnnpack" ], "platform": "x64 & windows" } ] -} \ No newline at end of file +} diff --git a/test/vcpkg.json b/test/vcpkg.json index 2e055e1c..d2a1d564 100644 --- a/test/vcpkg.json +++ b/test/vcpkg.json @@ -1,6 +1,6 @@ { "name": "test", - "version-date": "2023-09-27", + "version-date": "2023-12-20", "description": "vcpkg registry maintained by @luncliff", "homepage": "https://github.com/luncliff/vcpkg-registry", "supports": "windows | osx | ios", @@ -13,7 +13,16 @@ "name": "apple-nio-ssl", "platform": "osx | ios" }, + { + "name": "metal-cpp", + "platform": "osx | ios" + }, + { + "name": "coreml-tools", + "platform": "osx | ios" + }, "etcpak", + "libdispatch", { "name": "llama-cpp", "features": [ @@ -28,6 +37,18 @@ "openssl3", "rvo2", "rvo2-3d", + { + "name": "tensorflow-lite", + "platform": "windows" + }, + { + "name": "tensorflow-lite", + "features": [ + "google-angle", + "gpu" + ], + "platform": "windows & !arm" + }, { "name": "vcpkg-cmake", "host": true