CI ccache rework

.github/workflows/cache-clean.yml

@@ -0,0 +1,25 @@
+name: Cache clean
+
+on:
+  workflow_dispatch:
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cleanup
+        run: |
+          gh extension install actions/gh-actions-cache
+
+          echo "Fetching list of cache keys"
+          cacheKeys=$(gh actions-cache list -R $REPO -L 100 | grep -v develop | cut -f 1 )
+
+          echo "Deleting caches..."
+          set +e
+          for cacheKey in $cacheKeys; do
+            gh actions-cache delete "$cacheKey" -R "$REPO" --confirm
+          done
+          echo "Done"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}

.github/workflows/cache-rebuild.yml

@@ -0,0 +1,97 @@
+name: Cache rebuild
+
+on:
+  workflow_dispatch:
+    inputs:
+      clean_all:
+        description: 'Clean all caches, not just esp32 ones'
+        default: false
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cleanup
+        id: cleanup
+        run: |
+          gh extension install actions/gh-actions-cache
+
+          echo "Fetching list of cache keys"
+          if [ -z "$CLEAN_ALL" ]; then
+            cacheKeys=$(gh actions-cache list -R "$REPO" -L 100 | cut -f 1 )
+          else
+            cacheKeys=$(gh actions-cache list -R "$REPO" -L 100 | grep -w "idf\|esp32" | cut -f 1 )
+          fi
+
+          echo "Deleting caches..."
+          set +e
+          for cacheKey in $cacheKeys; do
+            gh actions-cache delete "$cacheKey" -R "$REPO" --confirm
+          done
+          echo "Done"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          CLEAN_ALL: ${{ inputs.clean_all }}
+
+  build:
+    needs: cleanup
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        idf_version: ["4.4", "5.2"]
+        include:
+          - os: ubuntu-latest
+            idf_version: "4.3"
+          - os: ubuntu-latest
+            idf_version: "5.0"
+        exclude:
+          - os: macos-latest
+            idf_version: "4.4"
+
+    runs-on: ${{ matrix.os }}
+
+    env:
+      SMING_ARCH: Esp32
+      SMING_SOC: esp32
+      INSTALL_IDF_VER: ${{ matrix.idf_version }}
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Setup python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.idf_version == '4.3' && '3.8' || '3.12' }}
+
+    - name: Configure environment
+      shell: pwsh
+      run: |
+        "CI_BUILD_DIR=" + (Resolve-Path ".").path >> $env:GITHUB_ENV
+        "SMING_HOME=" + (Resolve-Path "Sming").path >> $env:GITHUB_ENV
+
+    - name: Fix permissions
+      if: matrix.os != 'windows-latest'
+      run: |
+        sudo chown $USER /opt
+
+    - name: Install build tools for Ubuntu / MacOS
+      if: matrix.os != 'windows-latest'
+      run: |
+        Tools/ci/install.sh
+
+    - name: Install build tools for Windows
+      if: matrix.os == 'windows-latest'
+      run: |
+         . Tools/ci/setenv.ps1
+         Tools/ci/install.cmd
+
+    - name: Cache ESP-IDF and build tools
+      uses: actions/cache/save@v4
+      with:
+        path: |
+          /opt/esp-idf-${{ matrix.idf_version }}
+          /opt/esp32
+        key: ${{ matrix.os }}-idf-${{ matrix.idf_version }}

.github/workflows/ci-esp32.yml

@@ -2,7 +2,9 @@ name: Continuous Integration (CI) for Esp32
 
 on:
   push:
-
+
+  workflow_dispatch:
+
   pull_request:
     branches: [ develop ]
 
@@ -38,6 +40,7 @@ jobs:
       SMING_ARCH: Esp32
       SMING_SOC: ${{ matrix.variant }}
       INSTALL_IDF_VER: ${{ matrix.idf_version }}
+      IDF_SKIP_CHECK_SUBMODULES: 1
       ENABLE_CCACHE: 1
 
     steps:
@@ -60,7 +63,7 @@ jobs:
         "SMING_HOME=" + (Resolve-Path "Sming").path >> $env:GITHUB_ENV
 
     - name: Fix permissions
-      if: ${{ matrix.os != 'windows-latest' }}
+      if: matrix.os != 'windows-latest'
       run: |
         sudo chown $USER /opt
 
@@ -72,30 +75,34 @@ jobs:
           /opt/esp32
         key: ${{ matrix.os }}-idf-${{ matrix.idf_version }}
 
+    - name: Compiler Cache
+      uses: actions/cache@v4
+      with:
+        path: .ccache
+        key: ${{ matrix.os }}-ccache-${{ matrix.variant }}-${{ matrix.idf_version }}
+
     - name: Install build tools for Ubuntu / MacOS
-      if: ${{ matrix.os != 'windows-latest' }}
+      if: matrix.os != 'windows-latest'
       run: |
         Tools/ci/install.sh
 
-    - name: Install build tools for Windows   
-      if: ${{ matrix.os == 'windows-latest' }}
+    - name: Install build tools for Windows
+      if: matrix.os == 'windows-latest'
       run: |
          . Tools/ci/setenv.ps1
          Tools/ci/install.cmd
 
-    - name: ccache
-      uses: hendrikmuhs/[email protected]
-      with:
-        key: ${{ matrix.os }}-${{ matrix.variant }}-${{ matrix.idf_version }}
-
     - name: Build and test for ${{matrix.variant}} with IDF v${{matrix.idf_version}} on Ubuntu / MacOS
-      if: ${{ matrix.os != 'windows-latest' }}
+      if: matrix.os != 'windows-latest'
       run: |
         source $SMING_HOME/../Tools/export.sh
         Tools/ci/build.sh
 
     - name: Build and test for ${{matrix.variant}} with IDF v${{matrix.idf_version}} on Windows
-      if: ${{ matrix.os == 'windows-latest' }}
+      if: matrix.os == 'windows-latest'
       run: |
         . Tools/ci/setenv.ps1
         Tools/ci/build.cmd
+
+    - name: Compiler Cache stats
+      run: ccache -sv

.github/workflows/ci.yml

@@ -2,7 +2,9 @@ name: Continuous Integration (CI)
 
 on:
   push:
-
+
+  workflow_dispatch:
+
   pull_request:
     branches: [ develop ]
 
@@ -30,7 +32,7 @@ jobs:
           - variant: rp2040
             arch: Rp2040
 
-    concurrency: 
+    concurrency:
       group: ${{ github.workflow }}-${{ github.ref }}-${{ toJson(matrix) }}
       cancel-in-progress: true
 
@@ -62,32 +64,36 @@ jobs:
         "CI_BUILD_DIR=" + (Resolve-Path ".").path >> $env:GITHUB_ENV
         "SMING_HOME=" + (Resolve-Path "Sming").path >> $env:GITHUB_ENV
 
+    - name: Compiler Cache
+      uses: actions/cache@v4
+      with:
+        path: .ccache
+        key: ${{ matrix.os }}-ccache-${{ matrix.toolchain }}-${{ matrix.variant }}
+
     - name: Install build tools for Ubuntu / MacOS
-      if: ${{ matrix.os != 'windows-latest' }}
+      if: matrix.os != 'windows-latest'
       run: |
         Tools/ci/install.sh
 
-    - name: Install build tools for Windows   
-      if: ${{ matrix.os == 'windows-latest' }}
+    - name: Install build tools for Windows
+      if: matrix.os == 'windows-latest'
       run: |
          . Tools/ci/setenv.ps1
          Tools/ci/install.cmd
 
-    - name: ccache
-      uses: hendrikmuhs/[email protected]
-      with:
-        key: ${{ matrix.os }}-${{ matrix.toolchain }}-${{ matrix.variant }}
-
     - name: Build and test for ${{matrix.variant}} on Ubuntu / MacOS
       env:
         CLANG_FORMAT: clang-format-8
-      if: ${{ matrix.os != 'windows-latest' }}
+      if: matrix.os != 'windows-latest'
       run: |
         source $SMING_HOME/../Tools/export.sh
         Tools/ci/build.sh
 
     - name: Build and test for ${{matrix.variant}} on Windows
-      if: ${{ matrix.os == 'windows-latest' }}
+      if: matrix.os == 'windows-latest'
       run: |
         . Tools/ci/setenv.ps1
         Tools/ci/build.cmd
+
+    - name: Compiler Cache stats
+      run: ccache -sv

.github/workflows/coverity-scan.yml

@@ -44,19 +44,19 @@ jobs:
         echo "CHECK_SCA=$CHECK_SCA" >> $GITHUB_ENV
 
     - name: Setup SMING_HOME for Ubuntu
-      if: ${{ env.CHECK_SCA == 1 }}
+      if: env.CHECK_SCA == 1
       run: |
         echo "CI_BUILD_DIR=$GITHUB_WORKSPACE" >> $GITHUB_ENV
         echo "SMING_HOME=$GITHUB_WORKSPACE/Sming" >> $GITHUB_ENV
         echo "SMING_ARCH=Host" >> $GITHUB_ENV
 
     - name: Install Sming Framework on Ubuntu
-      if: ${{ env.CHECK_SCA == 1 }}
+      if: env.CHECK_SCA == 1
       run:  |
         Tools/ci/install.sh
 
     - name: Run Coverity Scan
-      if: ${{ env.CHECK_SCA == 1 }}
+      if: env.CHECK_SCA == 1
       env:
           COVERITY_SCAN_TOKEN: ${{secrets.COVERITY_SCAN_TOKEN}}
       run: |
@@ -67,7 +67,7 @@ jobs:
         $SMING_HOME/Arch/Host/Tools/ci/coverity-scan.sh
 
     - name: Archive scan log
-      if: ${{ env.CHECK_SCA == 1 }}
+      if: env.CHECK_SCA == 1
       uses: actions/upload-artifact@v3
       with:
         name: coverity-scan-report

.github/workflows/library.yml

@@ -62,7 +62,7 @@ jobs:
           - os: macos-latest
             idf_version: "4.4"
 
-    concurrency: 
+    concurrency:
       group: ${{ github.workflow }}-${{ github.ref }}-${{ toJson(matrix) }}
       cancel-in-progress: true
 
@@ -89,7 +89,7 @@ jobs:
         python-version: "3.12"
 
     - name: Create library alias
-      if: ${{ inputs.alias }}
+      if: inputs.alias
       shell: pwsh
       run: |
         New-Item -ItemType SymbolicLink -Path "../${{ inputs.alias }}" -Target (Resolve-Path ".").path
@@ -107,12 +107,12 @@ jobs:
         "CI_MAKEFILE=" + (Resolve-Path "../../sming/Tools/ci/library/Makefile") >> $env:GITHUB_ENV
 
     - name: Fix permissions
-      if: ${{ matrix.os != 'windows-latest' }}
+      if: matrix.os != 'windows-latest'
       run: |
         sudo chown $USER /opt
 
     - name: Cache ESP-IDF and build tools
-      if: ${{ matrix.arch == 'Esp32' }}
+      if: matrix.arch == 'Esp32'
       uses: actions/cache@v4
       with:
         path: |
@@ -121,12 +121,12 @@ jobs:
         key: ${{ matrix.os }}-idf-${{ env.INSTALL_IDF_VER }}
 
     - name: Install build tools for Ubuntu / MacOS
-      if: ${{ matrix.os != 'windows-latest' }}
+      if: matrix.os != 'windows-latest'
       run: |
         $SMING_HOME/../Tools/ci/install.sh
 
-    - name: Install build tools for Windows   
-      if: ${{ matrix.os == 'windows-latest' }}
+    - name: Install build tools for Windows
+      if: matrix.os == 'windows-latest'
       run: |
         cd $env:SMING_HOME/..
         . Tools/ci/setenv.ps1
@@ -135,13 +135,13 @@ jobs:
     - name: Build and Test for ${{matrix.arch}} on Ubuntu / MacOS
       env:
         CLANG_FORMAT: clang-format-8
-      if: ${{ matrix.os != 'windows-latest' }}
+      if: matrix.os != 'windows-latest'
       run: |
         source $SMING_HOME/../Tools/export.sh
         make -j$(nproc) -f $CI_MAKEFILE
 
     - name: Build and Test for ${{matrix.arch}} on Windows
-      if: ${{ matrix.os == 'windows-latest' }}
+      if: matrix.os == 'windows-latest'
       run: |
         . "$env:SMING_HOME/../Tools/ci/setenv.ps1"
         make -j $env:NUMBER_OF_PROCESSORS -f $env:CI_MAKEFILE

.github/workflows/release.yml

@@ -5,7 +5,7 @@ on:
   push:
     tags:
       - '[0-9]+.[0-9]+.[0-9]+'
-      
+
 # TODO: check if the tag is pointing to the tip of the master branch
 
 jobs:
@@ -14,22 +14,22 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - uses: trstringer/manual-approval@v1
-      if: ${{ github.ref_type == 'tag' }}
+      if: github.ref_type == 'tag'
       with:
          secret: ${{ github.TOKEN }}
          approvers: slaff
     - name: Install xmlstarlet
-      if: ${{ github.ref_type == 'tag' }}
+      if: github.ref_type == 'tag'
       run: sudo apt-get install -y jq xmlstarlet
     - name: Build docs
-      if: ${{ github.ref_type == 'tag' }}
+      if: github.ref_type == 'tag'
       run: |
         Tools/install.sh doc
         make -C docs html
         zip -r sming-docs.zip docs/build/html
     - name: Release New Version
-      if: ${{ github.ref_type == 'tag' }}
-      env: 
+      if: github.ref_type == 'tag'
+      env:
         SMING_ARCH: Host
         RELEASE_TOKEN: ${{secrets.RELEASE_TOKEN}}
         CI_REPO_NAME: ${{github.repository}}