[SymForce-External] Workflow and build support for pip wheels

See `documentation.rst` for usage description.  This primarily consists
of a workflow (manually triggered) to build wheels on GitHub Actions,
and supporting changes to our build to make this work, i.e. for the
right files to end up in the wheels in the right places (including
getting the README to look right on PyPI).

You can now:
- Build a wheel in a fresh environment, with basically whatever
  method you'd like (`python -m build`, `pip wheel`,
  `python setup.py bdist_wheel`).  Running this repeatedly to build
  multiple wheels also works, but may require cleaning the build
  directory depending on the method used (see the TODO on symenginepy
  in CMakeLists.txt)
- `pip install .` repeatedly to re-install symforce in the same
  environment
- `pip install -e .` repeatedly, manually cleaning the build
  (technically just the symenginepy build) between runs

Closes #157

GitOrigin-RevId: 0241cd20589e070c8e3d857f61fc2132746113c9

.clang-format

@@ -0,0 +1 @@
+symforce/.clang-format

.github/scripts/mac_install_gmp.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# ----------------------------------------------------------------------------
+# SymForce - Copyright 2022, Skydio, Inc.
+# This source code is under the Apache 2.0 license found in the LICENSE file.
+# ----------------------------------------------------------------------------
+
+set -ex
+
+if [ "$CIBW_ARCHS_MACOS" = "arm64" ]; then
+  mkdir arm-homebrew
+  curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C arm-homebrew
+  response=$(./arm-homebrew/bin/brew fetch --force --bottle-tag=arm64_big_sur gmp | grep "Downloaded to: " | cut -c 16-)
+  echo $response
+  # parsed=($response)
+  ./arm-homebrew/bin/brew install $response
+fi

.github/workflows/build_wheels.yml

@@ -0,0 +1,84 @@
+name: Build wheels
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build-extra:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Build Wheels
+      run: |
+        pip3 install build
+        python3 -m build --sdist --wheel third_party/skymarshal --outdir ./dist
+        python3 -m build --sdist --wheel gen/python --outdir ./dist
+
+    - name: Upload Wheels
+      uses: actions/upload-artifact@v3
+      with:
+        name: symforce-wheels
+        path: dist/*
+
+  build-macos:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest]
+        python-version: [cp38, cp39, cp310]
+        arch: [x86_64, arm64]
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Install cibuildwheel
+      run: python3 -m pip install cibuildwheel==2.5.0
+
+    - name: Build wheels
+      run: python3 -m cibuildwheel --output-dir wheelhouse
+      env:
+        CIBW_BUILD: ${{ matrix.python-version }}-*
+        CIBW_ARCHS_MACOS: ${{ matrix.arch }}
+        CIBW_BUILD_FRONTEND: build
+        GMP_ROOT: /Users/runner/work/symforce/symforce/arm-homebrew/opt/gmp
+        CIBW_BEFORE_BUILD: ./.github/scripts/mac_install_gmp.sh
+        CIBW_ENVIRONMENT: SYMFORCE_REWRITE_LOCAL_DEPENDENCIES=True
+
+    - name: Upload wheels
+      uses: actions/upload-artifact@v3
+      with:
+        name: symforce-wheels
+        path: wheelhouse/*.whl
+
+  build-linux:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [cp38, cp39, cp310]
+        arch: [x86_64]
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Install cibuildwheel
+      run: python3 -m pip install cibuildwheel==2.5.0
+
+    - name: Build wheels
+      run: python3 -m cibuildwheel --output-dir wheelhouse
+      env:
+        CIBW_BUILD: ${{ matrix.python-version }}-manylinux_${{ matrix.arch }}
+        CIBW_BUILD_FRONTEND: build
+        CIBW_BEFORE_BUILD: yum install -y gmp-devel git
+        CIBW_ENVIRONMENT: SYMFORCE_REWRITE_LOCAL_DEPENDENCIES=True
+
+    - name: Upload Wheels
+      uses: actions/upload-artifact@v3
+      with:
+        name: symforce-wheels
+        path: wheelhouse/*.whl

CMakeLists.txt

@@ -64,9 +64,15 @@ option(SYMFORCE_BUILD_STATIC_LIBRARIES
   OFF
 )
 
-find_program(SYMFORCE_DEFAULT_PYTHON python3 NO_CACHE)
-set(SYMFORCE_PYTHON "${SYMFORCE_DEFAULT_PYTHON}" CACHE STRING "Python executable to use")
-
+set(SYMFORCE_PYTHON_OVERRIDE "" CACHE STRING
+  "Python executable to use - if empty (the default), find python in the environment"
+)
+if(SYMFORCE_PYTHON_OVERRIDE STREQUAL "")
+  find_program(SYMFORCE_DEFAULT_PYTHON python3 NO_CACHE)
+  set(SYMFORCE_PYTHON ${SYMFORCE_DEFAULT_PYTHON})
+else()
+  set(SYMFORCE_PYTHON ${SYMFORCE_PYTHON_OVERRIDE})
+endif()
 
 set(SYMFORCE_SYMENGINE_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/symengine_install CACHE STRING
   "Directory to install symengine"
@@ -86,6 +92,11 @@ set(SYMFORCE_EIGEN_TARGET "Eigen3::Eigen" CACHE STRING
 # Setup
 # ==============================================================================
 
+# If we're cross-compiling macOS arm64 wheels, set the architecture appropriately
+if("$ENV{CIBW_ARCHS_MACOS}" STREQUAL "arm64")
+  set(CMAKE_OSX_ARCHITECTURES arm64)
+endif()
+
 if(SYMFORCE_BUILD_STATIC_LIBRARIES)
   set(SYMFORCE_LIBRARY_TYPE STATIC)
 else()
@@ -318,6 +329,8 @@ if(SYMFORCE_BUILD_SYMENGINE)
     CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>
                -DBUILD_TESTS=OFF
                -DBUILD_BENCHMARKS=OFF
+               -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}
+               -DCMAKE_POLICY_DEFAULT_CMP0074=NEW
   )
 
   # ------------------------------------------------------------------------------
@@ -327,16 +340,21 @@ if(SYMFORCE_BUILD_SYMENGINE)
     DEPENDS symengine
     SOURCE_DIR ${PROJECT_SOURCE_DIR}/third_party/symenginepy
     CONFIGURE_COMMAND ""
+    # TODO(aaron): This depends on SYMFORCE_PYTHON, which can change between builds (e.g. when
+    # running `python -m build` multiple times on the same build directory), currently if you're
+    # doing that you need to clean this manually
     BUILD_COMMAND ${SYMFORCE_PYTHON} ${PROJECT_SOURCE_DIR}/third_party/symenginepy/setup.py
         --verbose
         build_ext
         --build-type=Release
         --symengine-dir ${SYMFORCE_SYMENGINE_INSTALL_PREFIX}/lib/
+        --define CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}
     INSTALL_COMMAND env ${SYMENGINEPY_INSTALL_ENV} ${SYMFORCE_PYTHON}
         ${PROJECT_SOURCE_DIR}/third_party/symenginepy/setup.py
         --verbose
         install
         --prefix ${SYMFORCE_SYMENGINE_INSTALL_PREFIX}
+        --define CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}
   )
 
   # Rebuild symengine[py] on changed files

README.md

@@ -2,7 +2,9 @@
     somewhere.  The closest thing is to add a #gh-light-mode-only to the target, which does this,
     but is kinda confusing -->
 ![SymForce](docs/static/images/symforce_banner.png#gh-light-mode-only)
+<!-- DARK_MODE_ONLY -->
 ![SymForce](docs/static/images/symforce_banner_dark.png#gh-dark-mode-only)
+<!-- /DARK_MODE_ONLY -->
 
 <p align="center">
 <a href="https://symforce.org"><img alt="Documentation" src="https://img.shields.io/badge/api-docs-blue" /></a>
@@ -113,8 +115,10 @@ landmark_body = pose.inverse() * landmark
 ```
 <img src="https://render.githubusercontent.com/render/math?math={\begin{bmatrix}  R_{re} L_0 %2B R_{im} L_1 - R_{im} t_1 - R_{re} t_0 \\  -R_{im} L_0 %2B R_{re} L_1 %2B R_{im} t_0 %2B R_{re} t_1\end{bmatrix}}#gh-light-mode-only"
     width="250px" />
+<!-- DARK_MODE_ONLY -->
 <img src="https://render.githubusercontent.com/render/math?math={\color{white} \begin{bmatrix}  R_{re} L_0 %2B R_{im} L_1 - R_{im} t_1 - R_{re} t_0 \\  -R_{im} L_0 %2B R_{re} L_1 %2B R_{im} t_0 %2B R_{re} t_1\end{bmatrix}}#gh-dark-mode-only"
     width="250px" />
+<!-- /DARK_MODE_ONLY -->
 
 <!-- $
 \begin{bmatrix}
@@ -132,8 +136,10 @@ landmark_body.jacobian(pose)
 ```
 <img src="https://render.githubusercontent.com/render/math?math={\begin{bmatrix}-L_0 R_{im} %2B L_1 R_{re} %2B t_0 R_{im} - t_1 R_{re}, %26 -R_{re}, %26 -R_{im} \\ -L_0 R_{re} - L_1 R_{im} %2B t_0 R_{re} %2B t_1 R_{im}, %26  R_{im}, %26 -R_{re}\end{bmatrix}}#gh-light-mode-only"
     width="350px" />
+<!-- DARK_MODE_ONLY -->
 <img src="https://render.githubusercontent.com/render/math?math={\color{white} \begin{bmatrix}-L_0 R_{im} %2B L_1 R_{re} %2B t_0 R_{im} - t_1 R_{re}, %26 -R_{re}, %26 -R_{im} \\ -L_0 R_{re} - L_1 R_{im} %2B t_0 R_{re} %2B t_1 R_{im}, %26  R_{im}, %26 -R_{re}\end{bmatrix}}#gh-dark-mode-only"
     width="350px" />
+<!-- /DARK_MODE_ONLY -->
 
 <!-- $
 \begin{bmatrix}
@@ -151,8 +157,10 @@ sm.atan2(landmark_body[1], landmark_body[0])
 ```
 <img src="https://render.githubusercontent.com/render/math?math={atan_2(-R_{im} L_0 %2B R_{re} L_1 %2B R_{im} t_0 %2B R_{re} t_1, R_{re} L_0 %2B R_{im} L_1 - R_{im} t_1 - R_{re} t_0)}#gh-light-mode-only"
     width="500px" />
+<!-- DARK_MODE_ONLY -->
 <img src="https://render.githubusercontent.com/render/math?math={\color{white} atan_2(-R_{im} L_0 %2B R_{re} L_1 %2B R_{im} t_0 %2B R_{re} t_1, R_{re} L_0 %2B R_{im} L_1 - R_{im} t_1 - R_{re} t_0)}#gh-dark-mode-only"
     width="500px" />
+<!-- /DARK_MODE_ONLY -->
 
 <!-- $
 atan_2(-R_{im} L_0 + R_{re} L_1 + R_{im} t_0 + R_{re} t_1, R_{re} L_0  + R_{im} L_1 - R_{im} t_1 - R_{re} t_0)
@@ -165,8 +173,10 @@ geo.V3.symbolic("x").norm(epsilon=sm.epsilon)
 ```
 <img src="https://render.githubusercontent.com/render/math?math={\sqrt{x_0^2 %2B x_1^2 %2B x_2^2 %2B \epsilon}}#gh-light-mode-only"
     width="135px" />
+<!-- DARK_MODE_ONLY -->
 <img src="https://render.githubusercontent.com/render/math?math={\color{white} \sqrt{x_0^2 %2B x_1^2 %2B x_2^2 %2B \epsilon}}#gh-dark-mode-only"
     width="135px" />
+<!-- /DARK_MODE_ONLY -->
 
 <!-- $\sqrt{x_0^2 + x_1^2 + x_2^2 + \epsilon}$ -->
 
@@ -593,9 +603,11 @@ SymForce is developed and maintained by [Skydio](https://skydio.com/). It is rel
 <a href="http://skydio.com#gh-light-mode-only">
     <img alt="Skydio Logo" src="docs/static/images/skydio-logo-2.png" width="300px" />
 </a>
+<!-- DARK_MODE_ONLY -->
 <a href="http://skydio.com#gh-dark-mode-only">
     <img alt="Skydio Logo" src="docs/static/images/skydio-logo-2-white.png" width="300px" />
 </a>
+<!-- /DARK_MODE_ONLY -->
 
 # Contributing
 

docs/development.rst

@@ -100,3 +100,21 @@ Symbolic Backends
 SymForce uses the `SymPy <https://www.sympy.org/en/index.html>`_ API, but supports two backend implementations of it. The SymPy backend is pure Python, whereas the `SymEngine <https://github.com/symengine/symengine>`_ backend is wrapped C++. It can be 100-200 times faster for many operations, but is less fully featured and requires a C++ build.
 
 To set the backend, you can either use :func:`symforce.set_backend()` before any other imports, or use the ``SYMFORCE_BACKEND`` environment variable with the options ``sympy`` or ``symengine``. By default SymEngine will be used if found, otherwise SymPy.
+
+*************************************************
+Building wheels
+*************************************************
+
+You should be able to build Python wheels of symforce the standard ways.  We recommend using
+``build``, i.e. running ``python3 -m build --wheel`` from the ``symforce`` directory.  By default,
+this will build a wheel that includes local dependencies on the ``skymarshal`` and ``symforce-sym``
+packages (which are separate Python packages from ``symforce`` itself).  For distribution, you'll
+typically want to set the environment variable ``SYMFORCE_REWRITE_LOCAL_DEPENDENCIES=True`` when
+building, and also run ``python3 -m build --wheel third_party/skymarshal`` and
+``python3 -m build --wheel gen/python`` to build wheels for those packages separately.
+
+For SymForce releases, all of this is handled by the ``build_wheels`` GitHub Actions workflow.  This
+workflow is currently run manually on a commit, and produces a ``symforce-wheels.zip`` artifact with
+wheels (and sdists) for distribution (e.g. on PyPI).  It doesn't upload them to PyPI - to do that
+(after verifying that the built wheels work as expected) you should download and unzip the archive,
+and upload to PyPI with ``python -m twine upload [--repository testpypi] --verbose *``.