Cache IDF and tools for esp32 CI builds (#2848)

Saves time and resources downloading the IDF itself, all its submodules and build tools.
Just need to clear the cache if we update IDF versions.

- The IDF itself is shared amongst all platforms
- The build tools are specific to each OS

Total cache used is 5.58GB.

Note: Changed CI builds **only** to use a shallow clone for IDF (with submodules),
otherwise cache consumed hits 10.25GB which exceeds allowance.
Individually, the IDF size goes from from 1.5GB+ to 250MB.
Most of this is attributable to `components/esp_wifi/libs`.

Note: Windows esp32 tools come in around 1.1-1.3GB compared with 350-520MB for other OSes.

.github/workflows/ci-esp32.yml

@@ -32,7 +32,7 @@ jobs:
           - os: windows-latest
             idf_version: "5.0"
 
-    concurrency: 
+    concurrency:
       group: ${{ github.workflow }}-${{ github.ref }}-${{ toJson(matrix) }}
       cancel-in-progress: true
 
@@ -62,8 +62,26 @@ jobs:
         "CI_BUILD_DIR=" + (Resolve-Path ".").path >> $env:GITHUB_ENV
         "SMING_HOME=" + (Resolve-Path "Sming").path >> $env:GITHUB_ENV
 
+    - name: Fix permissions
+      if: ${{ matrix.os != 'windows-latest' }}
+      run: |
+        sudo chown $USER /opt
+
+    - name: Cache ESP-IDF
+      uses: actions/cache@v4
+      with:
+        path: /opt/esp-idf-${{ matrix.idf_version }}
+        key: idf-${{ matrix.idf_version }}
+        enableCrossOsArchive: true
+
+    - name: Cache build tools
+      uses: actions/cache@v4
+      with:
+        path: /opt/esp32
+        key: ${{ matrix.os }}-esp32-${{ matrix.idf_version }}
+
     - name: Install build tools for Ubuntu
-      if: ${{ matrix.os ==  'ubuntu-latest' }}
+      if: ${{ matrix.os == 'ubuntu-latest' }}
       run: |
         Tools/ci/install.sh
 

Sming/Arch/Esp32/Tools/install.cmd

@@ -3,18 +3,23 @@ REM Esp32 install.cmd
 if "%IDF_PATH%"=="" goto :EOF
 if "%IDF_TOOLS_PATH%"=="" goto :EOF
 
+if exist "%IDF_PATH%" goto :setup
+
 if "%IDF_REPO%"=="" set IDF_REPO="https://github.com/mikee47/esp-idf.git"
 if "%INSTALL_IDF_VER%"=="" set INSTALL_IDF_VER=5.2
 set IDF_BRANCH="sming/release/v%INSTALL_IDF_VER%"
 
-git clone -b %IDF_BRANCH% %IDF_REPO% %IDF_PATH%
+if "%CI_BUILD_DIR%" NEQ "" (
+    set IDF_INSTALL_OPTIONS=--depth 1 --recurse-submodules --shallow-submodules
+)
+git clone -b %IDF_BRANCH% %IDF_REPO% %IDF_PATH% %IDF_INSTALL_OPTIONS%
+
+:setup
 
 REM Install IDF tools and packages
 python "%IDF_PATH%\tools\idf_tools.py" --non-interactive install
 python -m pip install %SMINGTOOLS%/gevent-1.5.0-cp39-cp39-win_amd64.whl
-set IDF_REQUIREMENTS="%IDF_PATH%\requirements.txt"
-if not exist "%IDF_REQUIREMENTS%" set IDF_REQUIREMENTS="%IDF_PATH%\tools\requirements\requirements.core.txt"
-python -m pip install -r "%IDF_REQUIREMENTS%"
+python3 "%IDF_PATH%\tools\idf_tools.py" --non-interactive install-python-env
 
 if "%INSTALL_IDF_VER%" == "5.0" goto :install_python
 if "%INSTALL_IDF_VER%" == "5.2" goto :install_python

Sming/Arch/Esp32/Tools/install.sh

@@ -44,20 +44,22 @@ if [ ! -L "$IDF_PATH" ] && [ -d "$IDF_PATH" ]; then
 fi
 
 INSTALL_IDF_VER="${INSTALL_IDF_VER:=5.2}"
-IDF_CLONE_PATH="$IDF_PATH/../esp-idf-${INSTALL_IDF_VER}"
+IDF_CLONE_PATH="$(dirname "$IDF_PATH")/esp-idf-${INSTALL_IDF_VER}"
 IDF_REPO="${IDF_REPO:=https://github.com/mikee47/esp-idf.git}"
 IDF_BRANCH="sming/release/v${INSTALL_IDF_VER}"
 
 if [ -d "$IDF_CLONE_PATH" ]; then
     printf "\n\n** Skipping ESP-IDF clone: '%s' exists\n\n" "$IDF_CLONE_PATH"
 else
+    if [ -n "$CI_BUILD_DIR" ]; then
+        IDF_INSTALL_OPTIONS="--depth 1 --recurse-submodules --shallow-submodules"
+    fi
     echo "git clone -b $IDF_BRANCH $IDF_REPO $IDF_CLONE_PATH"
-    git clone -b "$IDF_BRANCH" "$IDF_REPO" "$IDF_CLONE_PATH"
+    git clone -b "$IDF_BRANCH" "$IDF_REPO" "$IDF_CLONE_PATH" $IDF_INSTALL_OPTIONS
 fi
-IDF_CLONE_PATH=$(realpath "$IDF_CLONE_PATH")
 
 # Create link to clone
-rm -rf "$IDF_PATH"
+rm -f "$IDF_PATH"
 ln -s "$IDF_CLONE_PATH" "$IDF_PATH"
 
 # Install IDF tools and packages

Tools/ci/setenv.ps1

@@ -1,9 +1,4 @@
-if($IsWindows) {
-    $TOOLS_DIR = "C:\tools"
-} else {
-    $TOOLS_DIR = "/opt"
-    sudo chown appveyor:appveyor /opt
-}
+$TOOLS_DIR = "D:/opt"
 
 $env:CI_BUILD_DIR = (Resolve-Path "$PSScriptRoot/../..").Path
 $env:SMING_HOME = Join-Path $env:CI_BUILD_DIR "Sming"
@@ -12,7 +7,7 @@ $env:SMING_HOME = Join-Path $env:CI_BUILD_DIR "Sming"
 $env:ESP_HOME = Join-Path $TOOLS_DIR "esp-quick-toolchain"
 
 # Esp32
-$env:IDF_PATH = Join-Path $TOOLS_DIR "esp-idf"
+$env:IDF_PATH = Join-Path $TOOLS_DIR "esp-idf-$env:INSTALL_IDF_VER"
 $env:IDF_TOOLS_PATH = Join-Path $TOOLS_DIR "esp32"
 
 # Rp2040