Add ccache support (#2856)

This PR adds [ccache](https://ccache.dev/) support to the framework and improves general build performance. Of particular importance is keeping the CI runs efficient to conserve resources without sacrificing effectiveness. If they complete a bit faster too, that's a bonus :-) **Reduce CI installation overhead** Since we're using more CI cache space to cache builds, it's at a premium. Fortunately there's quite a lot of stuff we can remove so running a script (`clean-tools.py`) after tool installation (CI only) clears this out so it doesn't end up in the cache. In practice this saves about 1.4GB of (compressed) cache space. Also, for IDF4.3 we can reduce that to a single run for the standard `esp32` SOC. **Installer improvements** Optional packages (e.g. clang-format) only required for Host builds, so don't install them otherwise. Review Windows install scripts: - Install ninja, ccache - Check for undefined environment variables - Check if tools already installed, skip if so and print message - Don't overwrite MinGW if it already exists **Add ccache support** I've never used ccache so wondered what it was doing in the install scripts. Certainly didn't seem to be used anywhere so thinking we could remove it... Nope. Let's use it. Add `ENABLE_CCACHE` setting, and enable in CI builds. Stats (`ccache -sv`) shows half of the calls aren't cacheable. Turns out ccache doesn't support `-MM` which we use for generating header build dependency files. After some investigations turns out we can just use `-MMD` which has several benefits: 1. Allows 100% hit rate via ccache 2. Reduces number of compiler invocations required since `.d` files are generated at the same time as `.o` files 3. Previously, if a source file is moved we got an error such as `No rule to make target '...file.cpp', needed by '...file.cpp.d'. Stop.`. This no longer happens. 4. Works on Windows without `sed` workaround for paths so makefile is simpler/faster. Note that a typical ccache installation may add soft links which override system compilers. For example, `gcc` would actually run `/usr/lib64/ccache/gcc` instead of `/usr/bin/gcc`. If `ENABLE_CCACHE` isn't set then Sming overrides this behaviour (using `CCACHE_DISABLE`) to ensure behaviour consistent with other (cross-compiled) architectures. **Fixes** Update ws_parser submodule. During testing I was getting some complaints about the particular commit not being available. It looks like this repo. got rebased at some point and the current master has changed. No change to content. Python script fixes: `make ide-vscode` fails in Windows, and there's a regex fix to `boardtool.py`. Fix fallthrough warning in LiveDebug sample.
SmingHub · Jul 7, 2024 · 045a85a · 045a85a
1 parent b7eec40
commit 045a85a
Show file tree

Hide file tree

Showing 22 changed files with 329 additions and 53 deletions.
diff --git a/.github/workflows/ci-esp32.yml b/.github/workflows/ci-esp32.yml
@@ -13,22 +13,18 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
         variant: [esp32, esp32s2, esp32c3, esp32s3, esp32c2]
-        idf_version: ["4.3", "4.4", "5.0", "5.2"]
-        exclude:
-          - variant: esp32s3
-            idf_version: "4.3"
-          - variant: esp32c2
+        idf_version: ["4.4", "5.0", "5.2"]
+        include:
+          - os: ubuntu-latest
+            variant: esp32
             idf_version: "4.3"
+        exclude:
           - variant: esp32c2
             idf_version: "4.4"
-          - os: macos-latest
-            idf_version: "4.3"
           - os: macos-latest
             idf_version: "4.4"
           - os: macos-latest
             idf_version: "5.0"
-          - os: windows-latest
-            idf_version: "4.3"
           - os: windows-latest
             idf_version: "5.0"
 
@@ -42,6 +38,7 @@ jobs:
       SMING_ARCH: Esp32
       SMING_SOC: ${{ matrix.variant }}
       INSTALL_IDF_VER: ${{ matrix.idf_version }}
+      ENABLE_CCACHE: 1
 
     steps:
     - name: Fix autocrlf setting
@@ -86,6 +83,11 @@ jobs:
          . Tools/ci/setenv.ps1
          Tools/ci/install.cmd
 
+    - name: ccache
+      uses: hendrikmuhs/[email protected]
+      with:
+        key: ${{ matrix.os }}-${{ matrix.variant }}-${{ matrix.idf_version }}
+
     - name: Build and test for ${{matrix.variant}} with IDF v${{matrix.idf_version}} on Ubuntu / MacOS
       if: ${{ matrix.os != 'windows-latest' }}
       run: |

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -41,6 +41,7 @@ jobs:
       SMING_SOC: ${{ matrix.variant }}
       CLANG_BUILD: ${{ matrix.toolchain == 'clang' && '15' || '0' }}
       BUILD64: ${{ matrix.toolchain == 'gcc64' && 1 || 0 }}
+      ENABLE_CCACHE: 1
 
     steps:
     - name: Fix autocrlf setting
@@ -72,6 +73,11 @@ jobs:
          . Tools/ci/setenv.ps1
          Tools/ci/install.cmd
 
+    - name: ccache
+      uses: hendrikmuhs/[email protected]
+      with:
+        key: ${{ matrix.os }}-${{ matrix.toolchain }}-${{ matrix.variant }}
+
     - name: Build and test for ${{matrix.variant}} on Ubuntu / MacOS
       env:
         CLANG_FORMAT: clang-format-8

diff --git a/Sming/Arch/Esp32/Tools/install.cmd b/Sming/Arch/Esp32/Tools/install.cmd
@@ -1,9 +1,9 @@
 REM Esp32 install.cmd
 
-if "%IDF_PATH%"=="" goto :EOF
-if "%IDF_TOOLS_PATH%"=="" goto :EOF
+if "%IDF_PATH%"=="" goto :undefined
+if "%IDF_TOOLS_PATH%"=="" goto :undefined
 
-if exist "%IDF_PATH%" goto :setup
+if exist "%IDF_PATH%" goto :cloned
 
 if "%IDF_REPO%"=="" set IDF_REPO="https://github.com/mikee47/esp-idf.git"
 if "%INSTALL_IDF_VER%"=="" set INSTALL_IDF_VER=5.2
@@ -14,6 +14,23 @@ if "%CI_BUILD_DIR%" NEQ "" (
 )
 git clone -b %IDF_BRANCH% %IDF_REPO% %IDF_PATH% %IDF_INSTALL_OPTIONS%
 
+goto :setup
+
+
+:undefined
+echo.
+echo ** Cannot install Esp32 tools: IDF_PATH or IDF_TOOLS_PATH not defined
+echo.
+goto :EOF
+
+
+:cloned
+echo.
+echo ** Skipping ESP-IDF clone: '%IDF_PATH%' exists
+echo.
+goto :setup
+
+
 :setup
 
 REM Install IDF tools and packages
@@ -24,10 +41,3 @@ python3 "%IDF_PATH%\tools\idf_tools.py" --non-interactive install-python-env
 if "%CI_BUILD_DIR%" NEQ "" (
     del /q "%IDF_TOOLS_PATH%\dist\*"
 )
-
-if "%INSTALL_IDF_VER%" == "5.0" goto :install_python
-if "%INSTALL_IDF_VER%" == "5.2" goto :install_python
-goto :EOF
-
-:install_python
-python "%IDF_PATH%\tools\idf_tools.py" --non-interactive install-python-env
diff --git a/Sming/Arch/Esp32/Tools/install.sh b/Sming/Arch/Esp32/Tools/install.sh
@@ -12,7 +12,6 @@ case $DIST in
     debian)
         PACKAGES+=(\
             bison \
-            ccache \
             flex \
             gperf \
             libffi-dev \
@@ -23,7 +22,6 @@ case $DIST in
     fedora)
         PACKAGES+=(\
             bison \
-            ccache \
             flex \
             gperf \
             libffi-devel \

diff --git a/Sming/Arch/Esp32/build.mk b/Sming/Arch/Esp32/build.mk
@@ -103,6 +103,10 @@ endif
 IDF_PATH_LIST += $(ESP32_IDFEXE_PATH)
 endif
 
+ifeq ($(ENABLE_CCACHE),1)
+export IDF_CCACHE_ENABLE := 1
+endif
+
 DEBUG_VARS += NINJA
 NINJA := $(if $(ESP32_NINJA_PATH),$(ESP32_NINJA_PATH)/,)ninja
 

diff --git a/Sming/Arch/Esp8266/Tools/install.cmd b/Sming/Arch/Esp8266/Tools/install.cmd
@@ -1,8 +1,27 @@
 REM Esp8266 install.cmd
 
+if "%ESP_HOME%" == "" goto :undefined
+if exist "%ESP_HOME%" goto :installed
+
 set EQT_REPO=https://github.com/earlephilhower/esp-quick-toolchain/releases/download/3.2.0-gcc10.3
 set EQT_TOOLCHAIN=x86_64-w64-mingw32.xtensa-lx106-elf-c791b74.230224.zip
 
 mkdir %ESP_HOME%
 curl -Lo %DOWNLOADS%/%EQT_TOOLCHAIN% %EQT_REPO%/%EQT_TOOLCHAIN%
 7z -o%ESP_HOME% x %DOWNLOADS%/%EQT_TOOLCHAIN%
+
+goto :EOF
+
+
+:undefined
+echo.
+echo ** Cannot install Esp8266 tools: ESP_HOME not defined
+echo.
+goto :EOF
+
+
+:installed
+echo.
+echo ** Skipping Esp8266 tools installation: '%ESP_HOME%' exists
+echo.
+goto :EOF
diff --git a/Sming/Arch/Rp2040/Components/picotool/component.mk b/Sming/Arch/Rp2040/Components/picotool/component.mk
@@ -7,7 +7,13 @@ COMPONENT_LIBNAME :=
 
 DEBUG_VARS		+= PICOTOOL
 
-CMAKE_OPTIONS :=
+PICOTOOL_CMAKE_OPTIONS :=
+
+ifeq ($(ENABLE_CCACHE),1)
+PICOTOOL_CMAKE_OPTIONS += \
+	-DCMAKE_C_COMPILER_LAUNCHER=$(CCACHE) \
+	-DCMAKE_CPP_COMPILER_LAUNCHER=$(CCACHE)
+endif
 
 ifeq ($(UNAME),Windows)
 PICOTOOL_CMAKE_OPTIONS += \

diff --git a/Sming/Arch/Rp2040/Components/rp2040/component.mk b/Sming/Arch/Rp2040/Components/rp2040/component.mk
@@ -118,6 +118,12 @@ RP2040_CMAKE_OPTIONS := \
 	-DCMAKE_MAKE_PROGRAM=$(NINJA) \
 	-DCMAKE_BUILD_TYPE=$(if $(subst 1,,$(PICO_DEBUG)),RelWithDebInfo,Debug)
 
+ifeq ($(ENABLE_CCACHE),1)
+RP2040_CMAKE_OPTIONS += \
+	-DCMAKE_C_COMPILER_LAUNCHER=$(CCACHE) \
+	-DCMAKE_CPP_COMPILER_LAUNCHER=$(CCACHE)
+endif
+
 COMPONENT_PREREQUISITES := $(PICO_CONFIG)
 
 BOOTLOADER := $(PICO_BUILD_DIR)/pico-sdk/src/rp2_common/boot_stage2/bs2_default_padded_checksummed.S

diff --git a/Sming/Arch/Rp2040/Tools/install.cmd b/Sming/Arch/Rp2040/Tools/install.cmd
@@ -1,6 +1,8 @@
 REM Rp2040 install.cmd
 
-if exist "%PICO_TOOLCHAIN_PATH%/arm-none-eabi" goto :already_got
+if "%PICO_TOOLCHAIN_PATH%"=="" goto :undefined
+if exist "%PICO_TOOLCHAIN_PATH%/arm-none-eabi" goto :installed
+
 set TOOLCHAIN_VERSION=13.2.rel1
 set TOOLCHAIN_BASE_URL=https://developer.arm.com/-/media/Files/downloads/gnu
 set TOOLCHAIN_NAME=arm-gnu-toolchain-%TOOLCHAIN_VERSION%-mingw-w64-i686-arm-none-eabi
@@ -10,10 +12,19 @@ curl -Lo tmp.zip %TOOLCHAIN_BASE_URL%/%TOOLCHAIN_VERSION%/binrel/%TOOLCHAIN_FILE
 del tmp.zip
 move "%PICO_TOOLCHAIN_PATH%-tmp/%TOOLCHAIN_NAME%" "%PICO_TOOLCHAIN_PATH%"
 rmdir "%PICO_TOOLCHAIN_PATH%-tmp"
+
+goto :EOF
+
+
+:undefined
+echo.
+echo ** Cannot install Rp2040 tools: PICO_TOOLCHAIN_PATH not defined
+echo.
 goto :EOF
 
 
-:already_got
+:installed
 echo.
 echo ** Skipping Rp2040 tools installation: '%PICO_TOOLCHAIN_PATH%' exists
 echo.
+goto :EOF
diff --git a/Sming/Components/lwip/component.mk b/Sming/Components/lwip/component.mk
@@ -30,6 +30,11 @@ LWIP_CMAKE_OPTIONS		:= \
 	-DLWIP_DIR=$(COMPONENT_PATH)/lwip \
 	-DCMAKE_MAKE_PROGRAM="$(NINJA)"
 
+ifeq ($(ENABLE_CCACHE),1)
+LWIP_CMAKE_OPTIONS += \
+	-DCMAKE_C_COMPILER_LAUNCHER=$(CCACHE)
+endif
+
 ifeq ($(ENABLE_LWIPDEBUG), 1)
 LWIP_CMAKE_OPTIONS		+= -DCMAKE_BUILD_TYPE=Debug
 else

diff --git a/Sming/Components/ws_parser b/Sming/Components/ws_parser
diff --git a/Sming/README.rst b/Sming/README.rst
@@ -81,6 +81,20 @@ Change it like this::
    However, it does not treat them as errors because of the above which could be false-positives.
 
 
+Build caching
+~~~~~~~~~~~~~
+
+.. envvar:: ENABLE_CCACHE
+
+   Default: 0 (disabled)
+
+   Set to 1 to run (most) compilation through `ccache <https://ccache.dev/>`__.
+   This speeds up re-compilation of code considerably at the expense of disk space
+   and slightly extended initial compilation.
+
+   This setting was introduced mainly for CI builds as relatively little changes between runs.
+
+
 Release builds
 ~~~~~~~~~~~~~~
 

diff --git a/Sming/build.mk b/Sming/build.mk
@@ -130,6 +130,10 @@ ARCH_COMPONENTS	= $(ARCH_BASE)/Components
 DEBUG_VARS	+= GIT
 GIT ?= git
 
+# ccache
+DEBUG_VARS += CCACHE
+CCACHE ?= ccache
+
 # CMake command
 DEBUG_VARS	+= CMAKE
 CMAKE ?= cmake
@@ -221,6 +225,10 @@ ifneq ($(STRICT),1)
 	CXXFLAGS += -Wno-reorder
 endif
 
+# ccache can speed up re-builds considerably
+BUILD_VARS += ENABLE_CCACHE
+ENABLE_CCACHE ?= 0
+
 include $(ARCH_BASE)/build.mk
 
 ifndef MAKE_CLEAN

diff --git a/Sming/component-wrapper.mk b/Sming/component-wrapper.mk
@@ -110,42 +110,38 @@ CFLAGS		+= $(COMPONENT_CFLAGS)
 CPPFLAGS	+= $(COMPONENT_CPPFLAGS)
 CXXFLAGS	+= $(COMPONENT_CXXFLAGS)
 
-# GCC 10 escapes ':' in path names which breaks GNU make for Windows so filter them
-ifeq ($(UNAME),Windows)
-OUTPUT_DEPS := | $(SED) "s/\\\\:/:/g" > $$@
-else
-OUTPUT_DEPS := -MF $$@
-endif
-
 # Additional flags to pass to clang
 CLANG_FLAG_EXTRA ?=
 
+ifneq ($(ENABLE_CCACHE),1)
+CCACHE :=
+export CCACHE_DISABLE=1
+endif
+
 # $1 -> absolute source directory, no trailing path separator
 # $2 -> relative output build directory, with trailing path separator
 define GenerateCompileTargets
 BUILD_DIRS += $2
 ifneq (,$(filter $1/%.s,$(SOURCE_FILES)))
 $2%.o: $1/%.s
 	$(vecho) "AS $$<"
-	$(Q) $(AS) $(addprefix -I,$(INCDIR)) $(CPPFLAGS) $(CFLAGS) -c $$< -o $$@
+	$(Q) $(CCACHE) $(AS) $(addprefix -I,$(INCDIR)) $(CPPFLAGS) $(CFLAGS) -c $$< -o $$@
 endif
 ifneq (,$(filter $1/%.S,$(SOURCE_FILES)))
 $2%.o: $1/%.S
 	$(vecho) "AS $$<"
-	$(Q) $(AS) $(addprefix -I,$(INCDIR)) $(CPPFLAGS) $(CFLAGS) -c $$< -o $$@
+	$(Q) $(CCACHE) $(AS) $(addprefix -I,$(INCDIR)) $(CPPFLAGS) $(CFLAGS) -c $$< -o $$@
 endif
 ifneq (,$(filter $1/%.c,$(SOURCE_FILES)))
 ifdef CLANG_TIDY
 $2%.o: $1/%.c
 	$(vecho) "TIDY $$<"
 	$(Q) $(CLANG_TIDY) $$< -- $(addprefix -I,$(INCDIR)) $(CPPFLAGS) $(CFLAGS) $(CLANG_FLAG_EXTRA)
 else
-$2%.o: $1/%.c $2%.c.d
+$2%.o: $1/%.c
 	$(vecho) "CC $$<"
-	$(Q) $(CC) $(addprefix -I,$(INCDIR)) $(CPPFLAGS) $(CFLAGS) -c $$< -o $$@
-$2%.c.d: $1/%.c
-	$(Q) $(CC) $(addprefix -I,$(INCDIR)) $(CPPFLAGS) $(CFLAGS) -MM -MT $2$$*.o $$< $(OUTPUT_DEPS)
-.PRECIOUS: $2%.c.d
+	$(Q) $(CCACHE) $(CC) $(addprefix -I,$(INCDIR)) $(CPPFLAGS) $(CFLAGS) -c $$< -o $$@ -MMD -MF $${@:.o=.c.d}
+-include $2%.c.d
 endif
 endif
 ifneq (,$(filter $1/%.cpp,$(SOURCE_FILES)))
@@ -154,12 +150,10 @@ $2%.o: $1/%.cpp
 	$(vecho) "TIDY $$<"
 	$(Q) $(CLANG_TIDY) $$< -- $(addprefix -I,$(INCDIR)) $(CPPFLAGS) $(CXXFLAGS) $(CLANG_FLAG_EXTRA)
 else
-$2%.o: $1/%.cpp $2%.cpp.d
+$2%.o: $1/%.cpp
 	$(vecho) "C+ $$<"
-	$(Q) $(CXX) $(addprefix -I,$(INCDIR)) $(CPPFLAGS) $(CXXFLAGS) -c $$< -o $$@
-$2%.cpp.d: $1/%.cpp
-	$(Q) $(CXX) $(addprefix -I,$(INCDIR)) $(CPPFLAGS) $(CXXFLAGS) -MM -MT $2$$*.o $$< $(OUTPUT_DEPS)
-.PRECIOUS: $2%.cpp.d
+	$(Q) $(CCACHE) $(CXX) $(addprefix -I,$(INCDIR)) $(CPPFLAGS) $(CXXFLAGS) -c $$< -o $$@ -MMD -MF $${@:.o=.cpp.d}
+-include $2%.cpp.d
 endif
 endif
 endef

diff --git a/Tools/boardtool.py b/Tools/boardtool.py
@@ -221,7 +221,7 @@ def addSignal(e, f):
 
     def parse_peripherals(self, spec):
         for name_spec, periphdef in spec.items():
-            p = re.compile('\[([0-9]+)-([0-9]+)\]')
+            p = re.compile(r'\[([0-9]+)-([0-9]+)\]')
             matches = p.findall(name_spec)
             if len(matches) == 0:
                 indexRange = range(-1, 0)