diff --git a/.github/workflows/cache-clean.yml b/.github/workflows/cache-clean.yml index fcb54cf00b..3ccbea4891 100644 --- a/.github/workflows/cache-clean.yml +++ b/.github/workflows/cache-clean.yml @@ -2,6 +2,16 @@ name: Cache clean on: workflow_dispatch: + inputs: + clean_opt: + description: 'Level of cleaning required' + type: choice + default: push-requests + options: + - pull-requests + - ccache + - idf-tools + - ccache+idf jobs: cleanup: @@ -12,7 +22,26 @@ jobs: gh extension install actions/gh-actions-cache echo "Fetching list of cache keys" - cacheKeys=$(gh actions-cache list -R $REPO -L 100 | grep -v develop | cut -f 1 ) + case $CLEAN_OPT in + pull-requests) + filter="-v develop" + ;; + ccache) + filter="ccache" + ;; + idf-tools) + filter="idf" + ;; + ccache+idf) + filter="ccache\|idf" + ;; + *) + echo "Unknown option '$CLEAN_OPT'" + exit 1 + ;; + esac + + cacheKeys=$(gh actions-cache list -R $REPO -L 100 | grep $filter | cut -f 1 ) echo "Deleting caches..." set +e @@ -23,3 +52,4 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} + CLEAN_OPT: ${{ inputs.clean_opt }} diff --git a/.github/workflows/cache-rebuild.yml b/.github/workflows/cache-rebuild.yml deleted file mode 100644 index 8bac555e3a..0000000000 --- a/.github/workflows/cache-rebuild.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: Cache rebuild - -on: - workflow_dispatch: - inputs: - clean_all: - description: 'Clean all caches, not just esp32 ones' - default: false - -jobs: - cleanup: - runs-on: ubuntu-latest - steps: - - name: Cleanup - id: cleanup - run: | - gh extension install actions/gh-actions-cache - - echo "Fetching list of cache keys" - if [ -z "$CLEAN_ALL" ]; then - cacheKeys=$(gh actions-cache list -R "$REPO" -L 100 | cut -f 1 ) - else - cacheKeys=$(gh actions-cache list -R "$REPO" -L 100 | grep -w "idf\|esp32" | cut -f 1 ) - fi - - echo "Deleting caches..." - set +e - for cacheKey in $cacheKeys; do - gh actions-cache delete "$cacheKey" -R "$REPO" --confirm - done - echo "Done" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} - CLEAN_ALL: ${{ inputs.clean_all }} - - build: - needs: cleanup - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - idf_version: ["4.4", "5.2"] - include: - - os: ubuntu-latest - idf_version: "4.3" - - os: ubuntu-latest - idf_version: "5.0" - exclude: - - os: macos-latest - idf_version: "4.4" - - runs-on: ${{ matrix.os }} - - env: - SMING_ARCH: Esp32 - SMING_SOC: esp32 - INSTALL_IDF_VER: ${{ matrix.idf_version }} - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.idf_version == '4.3' && '3.8' || '3.12' }} - - - name: Configure environment - shell: pwsh - run: | - "CI_BUILD_DIR=" + (Resolve-Path ".").path >> $env:GITHUB_ENV - "SMING_HOME=" + (Resolve-Path "Sming").path >> $env:GITHUB_ENV - - - name: Fix permissions - if: matrix.os != 'windows-latest' - run: | - sudo chown $USER /opt - - - name: Install build tools for Ubuntu / MacOS - if: matrix.os != 'windows-latest' - run: | - Tools/ci/install.sh - - - name: Install build tools for Windows - if: matrix.os == 'windows-latest' - run: | - . Tools/ci/setenv.ps1 - Tools/ci/install.cmd - - - name: Cache ESP-IDF and build tools - uses: actions/cache/save@v4 - with: - path: | - /opt/esp-idf-${{ matrix.idf_version }} - /opt/esp32 - key: ${{ matrix.os }}-idf-${{ matrix.idf_version }} diff --git a/Sming/Arch/Esp32/Tools/idf_tools-4.3.lst b/Sming/Arch/Esp32/Tools/idf_tools-4.3.lst new file mode 100644 index 0000000000..8b49fd1b38 --- /dev/null +++ b/Sming/Arch/Esp32/Tools/idf_tools-4.3.lst @@ -0,0 +1,7 @@ +xtensa-esp32-elf +xtensa-esp32s2-elf +xtensa-esp32s3-elf +riscv32-esp-elf +esp32ulp-elf +esp32s2ulp-elf +openocd-esp32 diff --git a/Sming/Arch/Esp32/Tools/idf_tools-4.4.lst b/Sming/Arch/Esp32/Tools/idf_tools-4.4.lst new file mode 100644 index 0000000000..d0ea4bf7b2 --- /dev/null +++ b/Sming/Arch/Esp32/Tools/idf_tools-4.4.lst @@ -0,0 +1,8 @@ +xtensa-esp32-elf +xtensa-esp32s2-elf +xtensa-esp32s3-elf +riscv32-esp-elf +esp32ulp-elf +xtensa-esp-elf-gdb +riscv32-esp-elf-gdb +openocd-esp32 diff --git a/Sming/Arch/Esp32/Tools/idf_tools-5.0.lst b/Sming/Arch/Esp32/Tools/idf_tools-5.0.lst new file mode 100644 index 0000000000..33c5dcb00b --- /dev/null +++ b/Sming/Arch/Esp32/Tools/idf_tools-5.0.lst @@ -0,0 +1,9 @@ +xtensa-esp-elf-gdb +riscv32-esp-elf-gdb +xtensa-esp32-elf +xtensa-esp32s2-elf +xtensa-esp32s3-elf +riscv32-esp-elf +esp32ulp-elf +esp-rom-elfs +openocd-esp32 diff --git a/Sming/Arch/Esp32/Tools/idf_tools-5.2.lst b/Sming/Arch/Esp32/Tools/idf_tools-5.2.lst new file mode 100644 index 0000000000..e121b28480 --- /dev/null +++ b/Sming/Arch/Esp32/Tools/idf_tools-5.2.lst @@ -0,0 +1,7 @@ +xtensa-esp-elf-gdb +riscv32-esp-elf-gdb +xtensa-esp-elf +riscv32-esp-elf +esp32ulp-elf +esp-rom-elfs +openocd-esp32 diff --git a/Sming/Arch/Esp32/Tools/install.cmd b/Sming/Arch/Esp32/Tools/install.cmd index 552d43b9e6..8e8b5b42c6 100644 --- a/Sming/Arch/Esp32/Tools/install.cmd +++ b/Sming/Arch/Esp32/Tools/install.cmd @@ -45,18 +45,21 @@ goto :setup if exist "%IDF_PATH%" rmdir /q %IDF_PATH% mklink /j %IDF_PATH% %IDF_CLONE_PATH% -if "%CI_BUILD_DIR%"=="" ( -set IDF_TOOL_PACKAGES=*openocd* -) else ( - REM Skip installation for CI if already present +REM Install IDF tools and packages (unless CI has restored from cache) +if "%CI_BUILD_DIR%" NEQ "" ( if exist "%IDF_TOOLS_PATH%\tools" ( + echo Skipping IDF tools installation, "%IDF_TOOLS_PATH%\tools" exists goto :EOF ) ) -REM Install IDF tools and packages -python "%IDF_PATH%\tools\idf_tools.py" --non-interactive install install "*elf*" %IDF_TOOL_PACKAGES% -python "%IDF_PATH%\tools\idf_tools.py" --non-interactive install-python-env +REM Be specific about which tools we want to install for each IDF version +tr '\n' ' ' < "%~dp0idf_tools-%INSTALL_IDF_VER%.lst" > toolver.txt || goto :EOF +for /f "tokens=*" %%x in (toolver.txt) do set IDF_TOOL_PACKAGES=%%x +del toolver.txt +echo Install: %IDF_TOOL_PACKAGES% +python "%IDF_PATH%\tools\idf_tools.py" --non-interactive install %IDF_TOOL_PACKAGES% || goto :EOF +python "%IDF_PATH%\tools\idf_tools.py" --non-interactive install-python-env || goto :EOF if "%CI_BUILD_DIR%" NEQ "" ( del /q "%IDF_TOOLS_PATH%\dist\*" diff --git a/Sming/Arch/Esp32/Tools/install.sh b/Sming/Arch/Esp32/Tools/install.sh index 11a216ef36..0a872db39d 100755 --- a/Sming/Arch/Esp32/Tools/install.sh +++ b/Sming/Arch/Esp32/Tools/install.sh @@ -61,22 +61,24 @@ rm -f "$IDF_PATH" ln -s "$IDF_CLONE_PATH" "$IDF_PATH" -# Skip installation for CI if already present -if [ -z "$CI_BUILD_DIR" ] || [ ! -d "$IDF_TOOLS_PATH/tools" ]; then - -# Install IDF tools and packages -python3 "$IDF_PATH/tools/idf_tools.py" --non-interactive install "*elf*" -if [ -n "$VIRTUAL_ENV" ]; then - unset VIRTUAL_ENV - unset VIRTUAL_ENV_PROMPT - export PATH="${PATH/$VIRTUAL_ENV\/bin:/}" -fi -python3 "$IDF_PATH/tools/idf_tools.py" --non-interactive install-python-env +# Install IDF tools and packages (unless CI has restored from cache) +if [ -n "$CI_BUILD_DIR" ] && [ -d "$IDF_TOOLS_PATH/tools" ]; then + printf "\n\n** Skipping IDF tools installation: '%s/tools' exists\n\n" "$IDF_TOOLS_PATH" +else + # Be specific about which tools we want to install for each IDF version + IDF_TOOL_PACKAGES=$(tr '\n' ' ' < "$SMING_HOME/Arch/Esp32/Tools/idf_tools-${INSTALL_IDF_VER}.lst") + echo "Install: $IDF_TOOL_PACKAGES" + python3 "$IDF_PATH/tools/idf_tools.py" --non-interactive install $IDF_TOOL_PACKAGES + if [ -n "$VIRTUAL_ENV" ]; then + unset VIRTUAL_ENV + unset VIRTUAL_ENV_PROMPT + export PATH="${PATH/$VIRTUAL_ENV\/bin:/}" + fi + python3 "$IDF_PATH/tools/idf_tools.py" --non-interactive install-python-env -if [ -z "$KEEP_DOWNLOADS" ]; then - rm -rf "$IDF_TOOLS_PATH/dist" + if [ -z "$KEEP_DOWNLOADS" ]; then + rm -rf "$IDF_TOOLS_PATH/dist" + fi fi -fi # CI install - fi diff --git a/Tools/ci/README.rst b/Tools/ci/README.rst index f8e39c3281..4d330b3bb0 100644 --- a/Tools/ci/README.rst +++ b/Tools/ci/README.rst @@ -7,11 +7,31 @@ Github actions See ``.github/workflows``. Cache clean - Dispatch workflow to remove caches for pull requests, but leave those for the default (develop) branch intact. + Dispatch workflow as convenient way to clean action cache. Cleaning levels are: + + - pull-requests + Any items created by pull requests, excludes anything in develop branch. + Normally pull requests will make use of caches present in the develop branch, but if there isn't one it will create its own. This becomes redundant when merged to develop. + + Such caches should be expired automatically, but this option can be used to remove them. + + - ccache + All ccache items. Use if pull requests are taking too long to build. + + - idf-tools + All IDF tool. Use before merging to develop if IDF toolchains have been updated. + + - ccache+idf + All ccache and IDF tool caches. + + Note that cleaning can always be done locally using ``gh``:: + + gh cache list # For fork + gh cache list -R SmingHub/Sming + gh cache delete --all -R SmingHub/Sming + + etc. -Cache rebuild - Dispatch workflow to rebuild the esp32 idf/tool caches. - By default, cleans only idf/esp32 caches but has option to perform full clean. CodeQL Performs code quality analysis when develop branch is updated. diff --git a/Tools/ci/clean-tools.py b/Tools/ci/clean-tools.py index 75762ca914..71466003bd 100644 --- a/Tools/ci/clean-tools.py +++ b/Tools/ci/clean-tools.py @@ -20,10 +20,9 @@ # These filters are matched from the **start** of the path so there's an implicit .* at the end FILTERS = { IDF_TOOLS_PATH: [ - # Leave versioned directory to avoid re-installation - r'.*esp-elf-gdb/.*/.*esp-elf-gdb/', - r'esp32ulp-elf/.*/esp32ulp-elf/', - r'openocd-esp32/.*/openocd-esp32/', + r'tools/.*esp-elf-gdb/', + r'tools/esp32ulp-elf/', + r'tools/openocd-esp32/', # Libraries not required by Sming # r'.*/riscv32-esp-elf/lib/{ARC}', r'.*/riscv32-esp-elf/lib/rv32i_.*',