Add Host MacOS support (#2804)

This PR adds support for the Host Emulator to MacOS. See #2773. **64-bit only** Standard compiler is 'Apple Clang', supports only 64-bit builds: no 32-bit compatibility. GCC can be installed via `brew install gcc` and compiles as usual with the `-m32` flag. The GNU linker is absent from brew's `coreutils` package (see Homebrew/homebrew-core#17794) so the standard system linker must be used. This only supports 64-bit targets. Apple have removed 32-bit application support from their ecosystem. Advantages No need for 32-bit compatibibility libraries in GNU/Linux. For MacOS build and run host emulator using standard Apple tools, no GCC required. General improvements in portability, less tolerant of incorrect type usage. Disadvantages Types such as long, size_t now 64-bit, don't match target hardware Pointers are 64-bit so cannot be cast to 32-bit intrinsics. Code must use correct `intptr_t` types. Must use `ENABLE_STORAGE_SIZE64=1` as emulated flash addresses must be 64-bit. **Symbol wrapping** MacOS linker doesn't support symbol wrapping which is used by `malloc_count` Component to hook `malloc`, etc. Don't currently have a solution for this so malloc_count is disabled for MacOS. **CI builds** Added MacOS builds plus running tests. For esp32 only IDF 5.2 is tested. **Notes** MacOS identified in makefiles with `UNAME=Darwin`. In code, use `#ifdef __APPLE__` when running on apple toolchains. Also requires `#ifdef __aarch64__` for Apple Silicon. The `rbpf` library requires `bpf` target with clang. Not supported by apple clang - use brew clang. I managed to get MacOS Ventura running in Qemu via https://github.com/kholia/OSX-KVM. This runs in x86_64 mode, adequate for dev/testing. The github actions runner `macos-latest` uses arm64, so introduces a few extra quirks. **Tools** - Compiler. Standard compiler is 'Apple Clang', supports only 64-bit builds: no 32-bit compatibility. Doesn't support `alias` attribute so use linker to do that instead. - Archive. Standard `ar` does not support - Linker. GNU Linker not available. MacOS linker does not support symbol wrapping. - GDB. Can be installed with `brew` but requires some additional steps for security reasons. Separate `lldb` target added. - sed. GNU 'sed' tool has extensions which MacOS version does not support. Added `SED` build variable which should always point to GNU version `gsed`, but only for Darwin.
SmingHub · Jun 24, 2024 · 9a097d8 · 9a097d8
1 parent edf87fd
commit 9a097d8
Show file tree

Hide file tree

Showing 50 changed files with 725 additions and 239 deletions.
diff --git a/.github/workflows/ci-esp32.yml b/.github/workflows/ci-esp32.yml
@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
         variant: [esp32, esp32s2, esp32c3, esp32s3, esp32c2]
         idf_version: ["4.3", "4.4", "5.0", "5.2"]
         exclude:
@@ -21,10 +21,16 @@ jobs:
             idf_version: "4.3"
           - variant: esp32c2
             idf_version: "4.4"
-          - idf_version: "4.3"
-            os: windows-latest
-          - idf_version: "5.0"
-            os: windows-latest
+          - os: macos-latest
+            idf_version: "4.3"
+          - os: macos-latest
+            idf_version: "4.4"
+          - os: macos-latest
+            idf_version: "5.0"
+          - os: windows-latest
+            idf_version: "4.3"
+          - os: windows-latest
+            idf_version: "5.0"
 
     concurrency: 
       group: ${{ github.workflow }}-${{ github.ref }}-${{ toJson(matrix) }}
@@ -61,6 +67,11 @@ jobs:
       run: |
         Tools/ci/install.sh
 
+    - name: Install build tools for MacOS
+      if: ${{ matrix.os == 'macos-latest' }}
+      run: |
+        Tools/ci/install.sh
+
     - name: Install build tools for Windows   
       if: ${{ matrix.os == 'windows-latest' }}
       run: |
@@ -73,6 +84,12 @@ jobs:
         source $SMING_HOME/../Tools/export.sh
         Tools/ci/build.sh
 
+    - name: Build and test for ${{matrix.variant}} on MacOS
+      if: ${{ matrix.os == 'macos-latest' }}
+      run: |
+        source $SMING_HOME/../Tools/export.sh
+        Tools/ci/build.sh
+
     - name: Build and test for ${{matrix.variant}} with IDF v${{matrix.idf_version}} on Windows
       if: ${{ matrix.os == 'windows-latest' }}
       run: |

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
         variant: [esp8266, host, rp2040]
         toolchain: [gcc]
         include:
@@ -66,6 +66,11 @@ jobs:
       run: |
         Tools/ci/install.sh
 
+    - name: Install build tools for MacOS
+      if: ${{ matrix.os == 'macos-latest' }}
+      run: |
+        Tools/ci/install.sh
+
     - name: Install build tools for Windows   
       if: ${{ matrix.os == 'windows-latest' }}
       run: |
@@ -81,6 +86,12 @@ jobs:
         $CLANG_FORMAT --version
         Tools/ci/build.sh
 
+    - name: Build and test for ${{matrix.variant}} on MacOS
+      if: ${{ matrix.os == 'macos-latest' }}
+      run: |
+        source $SMING_HOME/../Tools/export.sh
+        Tools/ci/build.sh
+
     - name: Build and test for ${{matrix.variant}} on Windows
       if: ${{ matrix.os == 'windows-latest' }}
       run: |

diff --git a/.github/workflows/library.yml b/.github/workflows/library.yml
@@ -1,4 +1,4 @@
-name: Continuous Integration (CI)
+name: Continuous Integration (CI) for Library
 
 on:
   workflow_call:
@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
         variant: [esp8266, host, esp32, esp32s2, esp32c3, esp32s3, esp32c2, rp2040]
         idf_version: ["4.4", ""] # "" denotes default, currently 5.2
         toolchain: [gcc]
@@ -34,6 +34,10 @@ jobs:
             variant: host
             arch: Host
             toolchain: clang
+          - os: ubuntu-latest
+            variant: host
+            arch: Host
+            toolchain: gcc64
           - variant: esp32
             arch: Esp32
           - variant: esp32s2
@@ -55,6 +59,8 @@ jobs:
             idf_version: "4.4"
           - variant: rp2040
             idf_version: "4.4"
+          - os: macos-latest
+            idf_version: "4.4"
 
     concurrency: 
       group: ${{ github.workflow }}-${{ github.ref }}-${{ toJson(matrix) }}
@@ -67,6 +73,7 @@ jobs:
       SMING_SOC: ${{ matrix.variant }}
       INSTALL_IDF_VER: ${{ matrix.idf_version }}
       CLANG_BUILD: ${{ matrix.toolchain == 'clang' && '15' || '0' }}
+      BUILD64: ${{ matrix.toolchain == 'gcc64' && 1 || 0 }}
 
     steps:
     - name: Fix autocrlf setting
@@ -103,6 +110,11 @@ jobs:
       run: |
         $SMING_HOME/../Tools/ci/install.sh
 
+    - name: Install build tools for MacOS
+      if: ${{ matrix.os == 'macos-latest' }}
+      run: |
+        $SMING_HOME/../Tools/ci/install.sh
+
     - name: Install build tools for Windows   
       if: ${{ matrix.os ==  'windows-latest' }}
       run: |
@@ -118,6 +130,12 @@ jobs:
         source $SMING_HOME/../Tools/export.sh
         make -j$(nproc) -f $CI_MAKEFILE
 
+    - name: Build and test for ${{matrix.variant}} on MacOS
+      if: ${{ matrix.os == 'macos-latest' }}
+      run: |
+        source $SMING_HOME/../Tools/export.sh
+        make -j$(nproc) -f $CI_MAKEFILE
+
     - name: Build and Test for ${{matrix.arch}} on Windows
       if: ${{ matrix.os == 'windows-latest' }}
       run: |

diff --git a/Sming/Arch/Esp32/Tools/install.sh b/Sming/Arch/Esp32/Tools/install.sh
@@ -5,30 +5,37 @@
 if [ -n "$IDF_PATH" ] && [ -n "$IDF_TOOLS_PATH" ]; then
 
 PACKAGES=(\
-    bison \
-    ccache \
-    dfu-util \
-    flex \
-    gperf \
-    ninja-build \
+    dfu-util
     )
 
 case $DIST in
     debian)
         PACKAGES+=(\
+            bison \
+            ccache \
+            flex \
+            gperf \
             libffi-dev \
             libssl-dev \
             )
         ;;
 
     fedora)
         PACKAGES+=(\
+            bison \
+            ccache \
+            flex \
+            gperf \
             libffi-devel \
             )
         ;;
+
+    darwin)
+        ;;
+
 esac
 
-$PKG_INSTALL ${PACKAGES[*]}
+$PKG_INSTALL "${PACKAGES[@]}"
 
 # If directory exists and isn't a symlink then rename it
 if [ ! -L "$IDF_PATH" ] && [ -d "$IDF_PATH" ]; then
@@ -37,23 +44,29 @@ if [ ! -L "$IDF_PATH" ] && [ -d "$IDF_PATH" ]; then
 fi
 
 INSTALL_IDF_VER="${INSTALL_IDF_VER:=5.2}"
-IDF_CLONE_PATH="$(readlink -m "$IDF_PATH/..")/esp-idf-${INSTALL_IDF_VER}"
+IDF_CLONE_PATH="$IDF_PATH/../esp-idf-${INSTALL_IDF_VER}"
 IDF_REPO="${IDF_REPO:=https://github.com/mikee47/esp-idf.git}"
 IDF_BRANCH="sming/release/v${INSTALL_IDF_VER}"
 
 if [ -d "$IDF_CLONE_PATH" ]; then
-    printf "\n\n** Skipping ESP-IDF clone: '$IDF_CLONE_PATH' exists\n\n"
+    printf "\n\n** Skipping ESP-IDF clone: '%s' exists\n\n" "$IDF_CLONE_PATH"
 else
     echo "git clone -b $IDF_BRANCH $IDF_REPO $IDF_CLONE_PATH"
     git clone -b "$IDF_BRANCH" "$IDF_REPO" "$IDF_CLONE_PATH"
 fi
+IDF_CLONE_PATH=$(realpath "$IDF_CLONE_PATH")
 
 # Create link to clone
-rm -f "$IDF_PATH"
+rm -rf "$IDF_PATH"
 ln -s "$IDF_CLONE_PATH" "$IDF_PATH"
 
 # Install IDF tools and packages
 python3 "$IDF_PATH/tools/idf_tools.py" --non-interactive install
+if [ -n "$VIRTUAL_ENV" ]; then
+    unset VIRTUAL_ENV
+    unset VIRTUAL_ENV_PROMPT
+    export PATH="${PATH/$VIRTUAL_ENV\/bin:/}"
+fi
 python3 "$IDF_PATH/tools/idf_tools.py" --non-interactive install-python-env
 
 if [ -z "$KEEP_DOWNLOADS" ]; then

diff --git a/Sming/Arch/Esp8266/Tools/install.sh b/Sming/Arch/Esp8266/Tools/install.sh
@@ -3,10 +3,15 @@
 # Esp8266 install.sh
 
 EQT_REPO="https://github.com/earlephilhower/esp-quick-toolchain/releases/download/3.2.0-gcc10.3"
-EQT_TOOLCHAIN="$(uname -m)-linux-gnu.xtensa-lx106-elf-c791b74.230224.tar.gz"
+if [ "$DIST" = "darwin" ]; then
+EQT_PLATFORM="x86_64-apple-darwin14"
+else
+EQT_PLATFORM="$(uname -m)-linux-gnu"
+fi
+EQT_TOOLCHAIN="$EQT_PLATFORM.xtensa-lx106-elf-c791b74.230224.tar.gz"
 
 if [ -d "$ESP_HOME" ]; then
-    printf "\n\n** Skipping Esp8266 tools installation: '$ESP_HOME' exists\n\n"
+    printf "\n\n** Skipping Esp8266 tools installation: '%s' exists\n\n" "$ESP_HOME"
 else
     $WGET "$EQT_REPO/$EQT_TOOLCHAIN" -O "$DOWNLOADS/$EQT_TOOLCHAIN"
     mkdir -p "$ESP_HOME"

diff --git a/Sming/Arch/Host/Components/gdbstub/component.mk b/Sming/Arch/Host/Components/gdbstub/component.mk
@@ -4,3 +4,13 @@ GDBSTUB_DIR := $(COMPONENT_PATH)
 
 CACHE_VARS	+= GDB_CMDLINE
 GDB_CMDLINE = trap '' INT; $(GDB) -x $(GDBSTUB_DIR)/gdbcmds --args $(TARGET_OUT_0) $(CLI_TARGET_OPTIONS) --pause -- $(HOST_PARAMETERS)
+
+# LLVM debugger
+CACHE_VARS	+= LLDB_CMDLINE
+LLDB_CMDLINE = lldb --source $(GDBSTUB_DIR)/lldbcmds $(TARGET_OUT_0) -- $(CLI_TARGET_OPTIONS) --pause $(HOST_PARAMETERS)
+
+##@Tools
+
+.PHONY: lldb
+lldb: ##Run the LLVM debugger console
+	$(LLDB_CMDLINE)
diff --git a/Sming/Arch/Host/Components/gdbstub/lldbcmds b/Sming/Arch/Host/Components/gdbstub/lldbcmds
@@ -0,0 +1,5 @@
+# Used for interrupt emulation
+process handle -p true -s false -n false SIGUSR1 SIGUSR2
+
+# Display a welcome prompt
+script print("\nWelcome to SMING!\nType 'r' to run application\n\n")
diff --git a/Sming/Arch/Host/Components/hostlib/component.mk b/Sming/Arch/Host/Components/hostlib/component.mk
@@ -2,7 +2,7 @@ EXTRA_LIBS		:= pthread
 
 ifeq ($(UNAME),Windows)
 	EXTRA_LIBS	+= wsock32
-else
+else ifneq ($(UNAME),Darwin)
 	EXTRA_LIBS	+= rt
 endif
 

diff --git a/Sming/Arch/Host/Components/hostlib/hostlib.c b/Sming/Arch/Host/Components/hostlib/hostlib.c
@@ -21,6 +21,10 @@
 #include <time.h>
 #include <string.h>
 
+#ifdef __APPLE__
+#include <mach-o/dyld.h>
+#endif
+
 int msleep(unsigned ms)
 {
 	struct timespec req, rem;
@@ -38,6 +42,10 @@ size_t getHostAppDir(char* path, size_t bufSize)
 #ifdef __WIN32
 	size_t len = GetModuleFileName(NULL, path, bufSize);
 	char sep = '\\';
+#elif defined(__APPLE__)
+	uint32_t size = bufSize;
+	size_t len = _NSGetExecutablePath(path, &size) ? 0 : size;
+	char sep = '/';
 #else
 	size_t len = readlink("/proc/self/exe", path, bufSize - 1);
 	char sep = '/';

diff --git a/Sming/Arch/Host/Components/hostlib/startup.cpp b/Sming/Arch/Host/Components/hostlib/startup.cpp
@@ -254,6 +254,7 @@ int main(int argc, char* argv[])
 	commandLine.parse(argc - i, &argv[i]);
 
 	if(!host_flashmem_init(config.flash)) {
+		host_debug_e("Flash init failed\n");
 		return 1;
 	}