Add IDF caching for libraries (#2850)

Cache esp-idf and tools for library builds. Be nice if we could use the cache from `SmingHub/Sming` directly but that sort of cross-repo sharing isn't supported. Also discovered that cross-OS caching breaks because the cache action uses paths relative to the build directory, even though it's been given absolute paths. So, just simplify the thing and use a separate, combined cache for each OS/IDF combination. Total's about 5GB. In all the .yml files we don't need to repeat steps for Ubuntu/MacOS as they're the same, so we can use a combined step for those, ie. `not windows`. Scanlog tool: - Fix scanlog tool windows path normalisation - didn't account for change from c:/tools/ to d:/opt/, and that Windows idf path contains version. - Print warnings for each job by default, add 'm' flag to merge them Also fix a few more warnings.
SmingHub · Jul 3, 2024 · f1d1a65 · f1d1a65
1 parent 5b70cc1
commit f1d1a65
Show file tree

Hide file tree

Showing 13 changed files with 196 additions and 209 deletions.
diff --git a/.github/workflows/ci-esp32.yml b/.github/workflows/ci-esp32.yml
@@ -67,26 +67,16 @@ jobs:
       run: |
         sudo chown $USER /opt
 
-    - name: Cache ESP-IDF
+    - name: Cache ESP-IDF and build tools
       uses: actions/cache@v4
       with:
-        path: /opt/esp-idf-${{ matrix.idf_version }}
-        key: idf-${{ matrix.idf_version }}
-        enableCrossOsArchive: true
+        path: |
+          /opt/esp-idf-${{ matrix.idf_version }}
+          /opt/esp32
+        key: ${{ matrix.os }}-idf-${{ matrix.idf_version }}
 
-    - name: Cache build tools
-      uses: actions/cache@v4
-      with:
-        path: /opt/esp32
-        key: ${{ matrix.os }}-esp32-${{ matrix.idf_version }}
-
-    - name: Install build tools for Ubuntu
-      if: ${{ matrix.os == 'ubuntu-latest' }}
-      run: |
-        Tools/ci/install.sh
-
-    - name: Install build tools for MacOS
-      if: ${{ matrix.os == 'macos-latest' }}
+    - name: Install build tools for Ubuntu / MacOS
+      if: ${{ matrix.os != 'windows-latest' }}
       run: |
         Tools/ci/install.sh
 
@@ -96,14 +86,8 @@ jobs:
          . Tools/ci/setenv.ps1
          Tools/ci/install.cmd
 
-    - name: Build and test for ${{matrix.variant}} with IDF v${{matrix.idf_version}} on Ubuntu
-      if: ${{ matrix.os == 'ubuntu-latest' }}
-      run: |
-        source $SMING_HOME/../Tools/export.sh
-        Tools/ci/build.sh
-
-    - name: Build and test for ${{matrix.variant}} on MacOS
-      if: ${{ matrix.os == 'macos-latest' }}
+    - name: Build and test for ${{matrix.variant}} with IDF v${{matrix.idf_version}} on Ubuntu / MacOS
+      if: ${{ matrix.os != 'windows-latest' }}
       run: |
         source $SMING_HOME/../Tools/export.sh
         Tools/ci/build.sh

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -61,13 +61,8 @@ jobs:
         "CI_BUILD_DIR=" + (Resolve-Path ".").path >> $env:GITHUB_ENV
         "SMING_HOME=" + (Resolve-Path "Sming").path >> $env:GITHUB_ENV
 
-    - name: Install build tools for Ubuntu
-      if: ${{ matrix.os == 'ubuntu-latest' }}
-      run: |
-        Tools/ci/install.sh
-
-    - name: Install build tools for MacOS
-      if: ${{ matrix.os == 'macos-latest' }}
+    - name: Install build tools for Ubuntu / MacOS
+      if: ${{ matrix.os != 'windows-latest' }}
       run: |
         Tools/ci/install.sh
 
@@ -77,17 +72,10 @@ jobs:
          . Tools/ci/setenv.ps1
          Tools/ci/install.cmd
 
-    - name: Build and test for ${{matrix.variant}} on Ubuntu
+    - name: Build and test for ${{matrix.variant}} on Ubuntu / MacOS
       env:
         CLANG_FORMAT: clang-format-8
-      if: ${{ matrix.os == 'ubuntu-latest' }}
-      run: |
-        source $SMING_HOME/../Tools/export.sh
-        $CLANG_FORMAT --version
-        Tools/ci/build.sh
-
-    - name: Build and test for ${{matrix.variant}} on MacOS
-      if: ${{ matrix.os == 'macos-latest' }}
+      if: ${{ matrix.os != 'windows-latest' }}
       run: |
         source $SMING_HOME/../Tools/export.sh
         Tools/ci/build.sh

diff --git a/.github/workflows/library.yml b/.github/workflows/library.yml
@@ -71,7 +71,7 @@ jobs:
     env:
       SMING_ARCH: ${{ matrix.arch }}
       SMING_SOC: ${{ matrix.variant }}
-      INSTALL_IDF_VER: ${{ matrix.idf_version }}
+      INSTALL_IDF_VER: ${{ matrix.idf_version || '5.2' }}
       CLANG_BUILD: ${{ matrix.toolchain == 'clang' && '15' || '0' }}
       BUILD64: ${{ matrix.toolchain == 'gcc64' && 1 || 0 }}
 
@@ -101,37 +101,41 @@ jobs:
     - name: Configure environment
       shell: pwsh
       run: |
+        "CI_BUILD_DIR=" + (Resolve-Path ".").path >> $env:GITHUB_ENV
         "SMING_HOME=" + (Resolve-Path "../../sming/Sming").path >> $env:GITHUB_ENV
         "COMPONENT_SEARCH_DIRS=" + (Resolve-Path "..").path >> $env:GITHUB_ENV
         "CI_MAKEFILE=" + (Resolve-Path "../../sming/Tools/ci/library/Makefile") >> $env:GITHUB_ENV
 
-    - name: Install build tools for Ubuntu
-      if: ${{ matrix.os ==  'ubuntu-latest' }}
+    - name: Fix permissions
+      if: ${{ matrix.os != 'windows-latest' }}
       run: |
-        $SMING_HOME/../Tools/ci/install.sh
+        sudo chown $USER /opt
+
+    - name: Cache ESP-IDF and build tools
+      if: ${{ matrix.arch == 'Esp32' }}
+      uses: actions/cache@v4
+      with:
+        path: |
+          /opt/esp-idf-${{ env.INSTALL_IDF_VER }}
+          /opt/esp32
+        key: ${{ matrix.os }}-idf-${{ env.INSTALL_IDF_VER }}
 
-    - name: Install build tools for MacOS
-      if: ${{ matrix.os == 'macos-latest' }}
+    - name: Install build tools for Ubuntu / MacOS
+      if: ${{ matrix.os != 'windows-latest' }}
       run: |
         $SMING_HOME/../Tools/ci/install.sh
 
     - name: Install build tools for Windows   
-      if: ${{ matrix.os ==  'windows-latest' }}
+      if: ${{ matrix.os == 'windows-latest' }}
       run: |
         cd $env:SMING_HOME/..
         . Tools/ci/setenv.ps1
         Tools/ci/install.cmd
 
-    - name: Build and Test for ${{matrix.arch}} on Ubuntu
+    - name: Build and Test for ${{matrix.arch}} on Ubuntu / MacOS
       env:
         CLANG_FORMAT: clang-format-8
-      if: ${{ matrix.os == 'ubuntu-latest' }}
-      run: |
-        source $SMING_HOME/../Tools/export.sh
-        make -j$(nproc) -f $CI_MAKEFILE
-
-    - name: Build and test for ${{matrix.variant}} on MacOS
-      if: ${{ matrix.os == 'macos-latest' }}
+      if: ${{ matrix.os != 'windows-latest' }}
       run: |
         source $SMING_HOME/../Tools/export.sh
         make -j$(nproc) -f $CI_MAKEFILE

diff --git a/Sming/Libraries/MultipartParser/src/HttpMultipartResource.cpp b/Sming/Libraries/MultipartParser/src/HttpMultipartResource.cpp
@@ -12,7 +12,7 @@
 
 #include "HttpMultipartResource.h"
 
-int HttpMultipartResource::setFileMap(HttpServerConnection& connection, HttpRequest& request, HttpResponse& response)
+int HttpMultipartResource::setFileMap(HttpServerConnection&, HttpRequest& request, HttpResponse& response)
 {
 	String contentType = request.headers[HTTP_HEADER_CONTENT_TYPE];
 	String mimeType = toString(MIME_FORM_MULTIPART);
@@ -24,7 +24,3 @@ int HttpMultipartResource::setFileMap(HttpServerConnection& connection, HttpRequ
 
 	return 0;
 }
-
-void HttpMultipartResource::shutdown(HttpServerConnection& connection)
-{
-}
diff --git a/Sming/Libraries/MultipartParser/src/HttpMultipartResource.h b/Sming/Libraries/MultipartParser/src/HttpMultipartResource.h
@@ -50,7 +50,9 @@ class HttpMultipartResource : public HttpResource
 	 */
 	virtual int setFileMap(HttpServerConnection& connection, HttpRequest& request, HttpResponse& response);
 
-	void shutdown(HttpServerConnection& connection) override;
+	void shutdown(HttpServerConnection&) override
+	{
+	}
 
 private:
 	HttpFilesMapper mapper;

diff --git a/Sming/Libraries/SwitchJoycon/src/SwitchJoycon.h b/Sming/Libraries/SwitchJoycon/src/SwitchJoycon.h
@@ -121,7 +121,7 @@ class SwitchJoycon
 	void resetButtons();
 
 protected:
-	virtual void onStarted(NimBLEServer* pServer){};
+	virtual void onStarted(NimBLEServer*){};
 
 private:
 	bool started{false};

diff --git a/Sming/Libraries/nanopb/src/Callback.cpp b/Sming/Libraries/nanopb/src/Callback.cpp
@@ -25,7 +25,7 @@ bool OutputCallback::encode(pb_ostream_t* stream, const pb_field_t* field)
 	return pb_encode_string(stream, data, length);
 }
 
-bool InputCallback::decode(pb_istream_t* stream, const pb_field_t* field)
+bool InputCallback::decode(pb_istream_t* stream, const pb_field_t*)
 {
 	size_t available = stream->bytes_left;
 	auto new_buf = realloc(data, length + available);

diff --git a/Sming/Libraries/nanopb/src/Stream.cpp b/Sming/Libraries/nanopb/src/Stream.cpp
@@ -24,7 +24,7 @@ bool InputStream::decode(const pb_msgdesc_t* fields, void* dest_struct)
 	is.callback = [](pb_istream_t* stream, pb_byte_t* buf, size_t count) -> bool {
 		auto self = static_cast<InputStream*>(stream->state);
 		assert(self != nullptr);
-		size_t read = self->stream.readBytes(reinterpret_cast<char*>(buf), count);
+		self->stream.readBytes(reinterpret_cast<char*>(buf), count);
 		return true;
 	};
 	is.state = this;

diff --git a/Sming/Libraries/nanopb/src/include/Protobuf/Stream.h b/Sming/Libraries/nanopb/src/include/Protobuf/Stream.h
@@ -63,7 +63,7 @@ class OutputStream : public Stream
 class DummyOutputStream : public OutputStream
 {
 protected:
-	bool write(const pb_byte_t* buf, size_t count) override
+	bool write(const pb_byte_t*, size_t) override
 	{
 		return true;
 	}

diff --git a/Tools/ci/README.rst b/Tools/ci/README.rst
@@ -55,10 +55,28 @@ To explicitly specify the repository to fetch from::
 
 To list all source locations with warnings::
 
-      python3 scanlog.py last-build.txt -w
+      python3 scanlog.py last-build.txt -w -m
+
+Note: The 'm' flag merges warnings from all jobs. Omitting this shows warnings for each job separately.
 
 To filter out warnings::
 
-      python3 scanlog.py last-build.txt -w --exclude warn-exclude.lst
+      python3 scanlog.py last-build.txt -w -m --exclude warn-exclude.lst
 
 The named exclusion file contains a list of regular expressions to match against.
+
+
+vscode
+------
+
+The warnings output using the scanlog tool can be used as hyperlinks in vscode:
+
+- Select a project, e.g. ``tests/HostTests`` and run ``make ide-vscode``
+- Open the resulting workspace in vscode
+- Add the ``sming`` folder to the project
+- Open an integrated terminal and dump the warnings as shown above.
+  Or, redirect them into a file and ``cat`` it.
+
+The file locations act as links to the source.
+Note that this isn't perfect. For example, esp-idf paths are not resolved to the specific version in use.
+Listing warnings for each job can be helpful as it shows which IDF version was used.