Add cache clean & rebuild workflows

.github/workflows/cache-clean.yml

@@ -0,0 +1,25 @@
+name: Cache clean
+
+on:
+  workflow_dispatch:
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cleanup
+        run: |
+          gh extension install actions/gh-actions-cache
+
+          echo "Fetching list of cache keys"
+          cacheKeys=$(gh actions-cache list -R $REPO -L 100 | grep -v develop | cut -f 1 )
+
+          echo "Deleting caches..."
+          set +e
+          for cacheKey in $cacheKeys; do
+            gh actions-cache delete "$cacheKey" -R "$REPO" --confirm
+          done
+          echo "Done"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}

.github/workflows/cache-rebuild.yml

@@ -0,0 +1,97 @@
+name: Cache rebuild
+
+on:
+  workflow_dispatch:
+    inputs:
+      clean_all:
+        description: 'Clean all caches, not just esp32 ones'
+        default: false
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cleanup
+        id: cleanup
+        run: |
+          gh extension install actions/gh-actions-cache
+
+          echo "Fetching list of cache keys"
+          if [ -z "$CLEAN_ALL" ]; then
+            cacheKeys=$(gh actions-cache list -R "$REPO" -L 100 | cut -f 1 )
+          else
+            cacheKeys=$(gh actions-cache list -R "$REPO" -L 100 | grep -w "idf\|esp32" | cut -f 1 )
+          fi
+
+          echo "Deleting caches..."
+          set +e
+          for cacheKey in $cacheKeys; do
+            gh actions-cache delete "$cacheKey" -R "$REPO" --confirm
+          done
+          echo "Done"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          CLEAN_ALL: ${{ inputs.clean_all }}
+
+  build:
+    needs: cleanup
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        idf_version: ["4.4", "5.2"]
+        include:
+          - os: ubuntu-latest
+            idf_version: "4.3"
+          - os: ubuntu-latest
+            idf_version: "5.0"
+        exclude:
+          - os: macos-latest
+            idf_version: "4.4"
+
+    runs-on: ${{ matrix.os }}
+
+    env:
+      SMING_ARCH: Esp32
+      SMING_SOC: esp32
+      INSTALL_IDF_VER: ${{ matrix.idf_version }}
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Setup python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.idf_version == '4.3' && '3.8' || '3.12' }}
+
+    - name: Configure environment
+      shell: pwsh
+      run: |
+        "CI_BUILD_DIR=" + (Resolve-Path ".").path >> $env:GITHUB_ENV
+        "SMING_HOME=" + (Resolve-Path "Sming").path >> $env:GITHUB_ENV
+
+    - name: Fix permissions
+      if: matrix.os != 'windows-latest'
+      run: |
+        sudo chown $USER /opt
+
+    - name: Install build tools for Ubuntu / MacOS
+      if: matrix.os != 'windows-latest'
+      run: |
+        Tools/ci/install.sh
+
+    - name: Install build tools for Windows
+      if: matrix.os == 'windows-latest'
+      run: |
+         . Tools/ci/setenv.ps1
+         Tools/ci/install.cmd
+
+    - name: Cache ESP-IDF and build tools
+      uses: actions/cache/save@v4
+      with:
+        path: |
+          /opt/esp-idf-${{ matrix.idf_version }}
+          /opt/esp32
+        key: ${{ matrix.os }}-idf-${{ matrix.idf_version }}

Tools/ci/README.rst

@@ -1,8 +1,69 @@
-CI logs
-=======
+Continuous Integration testing
+==============================
+
+Github actions
+--------------
+
+See ``.github/workflows``.
+
+Cache clean
+      Dispatch workflow to remove caches for pull requests, but leave those for the default (develop) branch intact.
+
+Cache rebuild
+      Dispatch workflow to rebuild the esp32 idf/tool caches.
+      By default, cleans only idf/esp32 caches but has option to perform full clean.
+
+CodeQL
+      Performs code quality analysis when develop branch is updated.
+
+Continuous Integration (CI)
+      Tests for all architectures except esp32. Run for every pull request and merge to develop.
+
+Continuous Integration (CI) for Esp32
+      Tests for esp32 architecture. Requires a separate workflow as matrix becomes too complex otherwise.
+
+Continuous Integration (CI) for Library
+      Used indirectly by library workflows for testing. See workflows/library.
+
+Coverity Scan
+      Code quality analyser. Does instrumented build then uploads database to coverity servers for analysis.
+
+Release
+      Run manually during release phase only.
+
+Spelling Check
+      Run for all pull requests and merge to develop.
+
 
-Overview
---------
+Esp32 IDF and tools cleaning
+----------------------------
+
+Because the IDF and associated tools are large and relatively time-consuming to install, these are cached.
+There's so much bloat that it doesn't take much to fill the 10GB github cache allocation.
+
+So after installing the tools - before it gets cached - the `clean-tools.py` script gets run.
+This tool contains a list of filters (regular expressions) which match various paths.
+Candidates for removal were identified by inspection using the Gnome disk usage analyzer.
+Some other stuff (like test code and examples) are fairly safe candidates to remove as well.
+
+To evaluate how much would be removed run this command (it's safe)::
+
+      python clean-tools.py scan
+
+To perform 'dry-run' of a clean operation, without actually deleteing anything::
+
+      python clean-tools.py clean
+
+To actually delete stuff requires a confirmation flag::
+
+      python clean-tools.py clean --delete
+
+Note that some unused submodules are cleaned, but by default the IDF pulls them back in again!
+To prevent this behaviour, set `IDF_SKIP_CHECK_SUBMODULES=1`.
+
+
+CI logs
+-------
 
 Analysing CI logs is important for several reasons:
 
@@ -23,15 +84,15 @@ The log files related to a run and therefore only two are required (the main bui
 
 
 Setup
------
+~~~~~
 
 The github CLI client must be installed and authenticated with the Sming repo (or fork).
 
 See https://docs.github.com/en/github-cli/github-cli/quickstart.
 
 
 Usage
------
+~~~~~
 
 Fetch and scan the most recent build::
 
@@ -67,7 +128,7 @@ The named exclusion file contains a list of regular expressions to match against
 
 
 vscode
-------
+~~~~~
 
 The warnings output using the scanlog tool can be used as hyperlinks in vscode: