From 18857df4e66a889bb5e45ccb437c9cc1c9c4ae3e Mon Sep 17 00:00:00 2001 From: mikee47 Date: Tue, 9 Jul 2024 09:49:16 +0100 Subject: [PATCH] Add cache clean & rebuild workflows --- .github/workflows/cache-clean.yml | 25 ++++++++ .github/workflows/cache-rebuild.yml | 97 +++++++++++++++++++++++++++++ Tools/ci/README.rst | 75 +++++++++++++++++++--- 3 files changed, 190 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/cache-clean.yml create mode 100644 .github/workflows/cache-rebuild.yml diff --git a/.github/workflows/cache-clean.yml b/.github/workflows/cache-clean.yml new file mode 100644 index 0000000000..fcb54cf00b --- /dev/null +++ b/.github/workflows/cache-clean.yml @@ -0,0 +1,25 @@ +name: Cache clean + +on: + workflow_dispatch: + +jobs: + cleanup: + runs-on: ubuntu-latest + steps: + - name: Cleanup + run: | + gh extension install actions/gh-actions-cache + + echo "Fetching list of cache keys" + cacheKeys=$(gh actions-cache list -R $REPO -L 100 | grep -v develop | cut -f 1 ) + + echo "Deleting caches..." + set +e + for cacheKey in $cacheKeys; do + gh actions-cache delete "$cacheKey" -R "$REPO" --confirm + done + echo "Done" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} diff --git a/.github/workflows/cache-rebuild.yml b/.github/workflows/cache-rebuild.yml new file mode 100644 index 0000000000..8bac555e3a --- /dev/null +++ b/.github/workflows/cache-rebuild.yml @@ -0,0 +1,97 @@ +name: Cache rebuild + +on: + workflow_dispatch: + inputs: + clean_all: + description: 'Clean all caches, not just esp32 ones' + default: false + +jobs: + cleanup: + runs-on: ubuntu-latest + steps: + - name: Cleanup + id: cleanup + run: | + gh extension install actions/gh-actions-cache + + echo "Fetching list of cache keys" + if [ -z "$CLEAN_ALL" ]; then + cacheKeys=$(gh actions-cache list -R "$REPO" -L 100 | cut -f 1 ) + else + cacheKeys=$(gh actions-cache list -R "$REPO" -L 100 | grep -w "idf\|esp32" | cut -f 1 ) + fi + + echo "Deleting caches..." + set +e + for cacheKey in $cacheKeys; do + gh actions-cache delete "$cacheKey" -R "$REPO" --confirm + done + echo "Done" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + CLEAN_ALL: ${{ inputs.clean_all }} + + build: + needs: cleanup + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + idf_version: ["4.4", "5.2"] + include: + - os: ubuntu-latest + idf_version: "4.3" + - os: ubuntu-latest + idf_version: "5.0" + exclude: + - os: macos-latest + idf_version: "4.4" + + runs-on: ${{ matrix.os }} + + env: + SMING_ARCH: Esp32 + SMING_SOC: esp32 + INSTALL_IDF_VER: ${{ matrix.idf_version }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.idf_version == '4.3' && '3.8' || '3.12' }} + + - name: Configure environment + shell: pwsh + run: | + "CI_BUILD_DIR=" + (Resolve-Path ".").path >> $env:GITHUB_ENV + "SMING_HOME=" + (Resolve-Path "Sming").path >> $env:GITHUB_ENV + + - name: Fix permissions + if: matrix.os != 'windows-latest' + run: | + sudo chown $USER /opt + + - name: Install build tools for Ubuntu / MacOS + if: matrix.os != 'windows-latest' + run: | + Tools/ci/install.sh + + - name: Install build tools for Windows + if: matrix.os == 'windows-latest' + run: | + . Tools/ci/setenv.ps1 + Tools/ci/install.cmd + + - name: Cache ESP-IDF and build tools + uses: actions/cache/save@v4 + with: + path: | + /opt/esp-idf-${{ matrix.idf_version }} + /opt/esp32 + key: ${{ matrix.os }}-idf-${{ matrix.idf_version }} diff --git a/Tools/ci/README.rst b/Tools/ci/README.rst index 00e9e67ace..f8e39c3281 100644 --- a/Tools/ci/README.rst +++ b/Tools/ci/README.rst @@ -1,8 +1,69 @@ -CI logs -======= +Continuous Integration testing +============================== + +Github actions +-------------- + +See ``.github/workflows``. + +Cache clean + Dispatch workflow to remove caches for pull requests, but leave those for the default (develop) branch intact. + +Cache rebuild + Dispatch workflow to rebuild the esp32 idf/tool caches. + By default, cleans only idf/esp32 caches but has option to perform full clean. + +CodeQL + Performs code quality analysis when develop branch is updated. + +Continuous Integration (CI) + Tests for all architectures except esp32. Run for every pull request and merge to develop. + +Continuous Integration (CI) for Esp32 + Tests for esp32 architecture. Requires a separate workflow as matrix becomes too complex otherwise. + +Continuous Integration (CI) for Library + Used indirectly by library workflows for testing. See workflows/library. + +Coverity Scan + Code quality analyser. Does instrumented build then uploads database to coverity servers for analysis. + +Release + Run manually during release phase only. + +Spelling Check + Run for all pull requests and merge to develop. + -Overview --------- +Esp32 IDF and tools cleaning +---------------------------- + +Because the IDF and associated tools are large and relatively time-consuming to install, these are cached. +There's so much bloat that it doesn't take much to fill the 10GB github cache allocation. + +So after installing the tools - before it gets cached - the `clean-tools.py` script gets run. +This tool contains a list of filters (regular expressions) which match various paths. +Candidates for removal were identified by inspection using the Gnome disk usage analyzer. +Some other stuff (like test code and examples) are fairly safe candidates to remove as well. + +To evaluate how much would be removed run this command (it's safe):: + + python clean-tools.py scan + +To perform 'dry-run' of a clean operation, without actually deleteing anything:: + + python clean-tools.py clean + +To actually delete stuff requires a confirmation flag:: + + python clean-tools.py clean --delete + +Note that some unused submodules are cleaned, but by default the IDF pulls them back in again! +To prevent this behaviour, set `IDF_SKIP_CHECK_SUBMODULES=1`. + + +CI logs +------- Analysing CI logs is important for several reasons: @@ -23,7 +84,7 @@ The log files related to a run and therefore only two are required (the main bui Setup ------ +~~~~~ The github CLI client must be installed and authenticated with the Sming repo (or fork). @@ -31,7 +92,7 @@ See https://docs.github.com/en/github-cli/github-cli/quickstart. Usage ------ +~~~~~ Fetch and scan the most recent build:: @@ -67,7 +128,7 @@ The named exclusion file contains a list of regular expressions to match against vscode ------- +~~~~~ The warnings output using the scanlog tool can be used as hyperlinks in vscode: