Fix streaming (#65)

* add openpilot

* remove opendbc and cereal

* symlinks

* use uv!

* source venv

* fix FindCereal.cmake

* bump openpilot

* add common too

* not needed

* fix build

* fix macos

* test

* test

* need op too

* portaudio

.github/workflows/openpilot.yaml

@@ -16,12 +16,12 @@ jobs:
         arch: ${{ fromJson(
            ((github.repository == 'commaai/plotjuggler') &&
               ((github.event_name != 'pull_request') || 
-               (github.event.pull_request.head.repo.full_name == 'commaai/plotjuggler'))) && '["x86_64", "aarch64"]' || '["x86_64"]' ) }}    
+               (github.event.pull_request.head.repo.full_name == 'commaai/plotjuggler'))) && '["x86_64", "aarch64"]' || '["x86_64"]' ) }}
     runs-on: ${{ (matrix.arch == 'aarch64') && 'buildjet-4vcpu-ubuntu-2204-arm' || 'ubuntu-20.04' }}
     steps:
     - uses: actions/checkout@v2
       with:
-        submodules: true
+        submodules: recursive
     - name: build
       run: ./build.sh
     - name: zip plugins
@@ -42,18 +42,21 @@ jobs:
     steps:
     - uses: actions/checkout@v2
       with:
-        submodules: true
+        submodules: recursive
     - uses: actions/setup-python@v2
       with:
         python-version: '3.11.x'
     - name: setup dependencies
       run: |
-        brew install cmake bzip2 capnp qt@5 zeromq protobuf@3 gnu-tar
-        sudo pip3 install --no-cache-dir -r 3rdparty/opendbc_repo/requirements.txt
+        brew install cmake bzip2 capnp qt@5 zeromq protobuf@3 gnu-tar portaudio
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        uv venv
+        uv pip install 3rdparty/openpilot 3rdparty/openpilot/opendbc_repo
     - name: build
       run: |
         export PYTHONPATH=$PWD/3rdparty
         # force usage of protobuf@3 instead of protobuf
+        source .venv/bin/activate
         mkdir -p build && cd build && cmake -DCMAKE_PREFIX_PATH="$(brew --prefix)/opt/protobuf@3" .. && make -j8
     - name: zip plugins
       run: eval "$ZIP"

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "3rdparty/cereal"]
-	path = 3rdparty/cereal
-	url = ../../commaai/cereal.git
-[submodule "3rdparty/opendbc"]
-	path = 3rdparty/opendbc_repo
-	url = ../../commaai/opendbc.git
+[submodule "3rdparty/openpilot"]
+	path = 3rdparty/openpilot
+	url = https://github.com/commaai/openpilot

3rdparty/cereal

@@ -0,0 +1 @@
+openpilot/cereal

3rdparty/common

@@ -0,0 +1 @@
+openpilot/common

3rdparty/msgq

@@ -0,0 +1 @@
+openpilot/msgq_repo/msgq

3rdparty/opendbc

@@ -1 +1 @@
-opendbc_repo/opendbc
+openpilot/opendbc_repo/opendbc

Dockerfile

@@ -35,17 +35,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   libsqlite3-dev \
   clang \
   ocl-icd-opencl-dev \
-  opencl-headers
+  opencl-headers \
+  portaudio19-dev
 
-RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
-ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}"
-RUN pyenv install 3.11.4 && \
-    pyenv global 3.11.4 && \
-    pyenv rehash
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+ENV PATH="/root/.cargo/bin:${PATH}"
+RUN uv venv --python 3.11.4
+RUN uv pip install pkgconfig jinja2 Cython
 
-RUN pip3 install pkgconfig jinja2
-
-# installs scons, pycapnp, cython, etc.
 ENV PYTHONPATH /tmp/plotjuggler/3rdparty
-COPY 3rdparty/opendbc_repo/requirements.txt /tmp/
-RUN pip3 install Cython && pip3 install --no-cache-dir -r /tmp/requirements.txt
+COPY 3rdparty/openpilot/opendbc_repo/pyproject.toml /tmp/opendbc_repo/pyproject.toml
+COPY 3rdparty/openpilot/pyproject.toml /tmp/openpilot/pyproject.toml
+RUN uv pip install --no-cache-dir /tmp/opendbc_repo
+RUN uv pip install --no-cache-dir /tmp/openpilot

build.sh

@@ -6,4 +6,4 @@ docker run \
   --rm \
   --volume $PWD:/tmp/plotjuggler \
   --workdir /tmp/plotjuggler plotjuggler:latest \
-  /bin/bash -c "mkdir -p build && cd build && cmake .. && make -j$(nproc)"
+  /bin/bash -c "source /.venv/bin/activate && mkdir -p build && cd build && cmake .. && make -j$(nproc)"

cmake/FindCereal.cmake

@@ -5,8 +5,13 @@ else (Cereal_LIBRARIES AND Cereal_INCLUDE_DIRS)
 
   # build cereal before we search for required Cereal plugin libs below
   execute_process(
-    COMMAND scons -j8 --minimal
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/3rdparty/cereal
+    COMMAND scons -j8 --minimal -u
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/3rdparty/openpilot/cereal
+  )
+
+  execute_process(
+    COMMAND scons -j8 --minimal -u
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/3rdparty/openpilot/msgq
   )
 
   find_path(Cereal_INCLUDE_DIRS
@@ -29,14 +34,14 @@ else (Cereal_LIBRARIES AND Cereal_INCLUDE_DIRS)
     NAMES
       cereal
     PATHS
-      ${CMAKE_SOURCE_DIR}/3rdparty/cereal
+      ${CMAKE_SOURCE_DIR}/3rdparty/openpilot/cereal
   )
 
-  find_library(Cereal_LIBRARY_MESSAGING
+  find_library(Cereal_LIBRARY_MSGQ
     NAMES
-      messaging
+      msgq
     PATHS
-      ${CMAKE_SOURCE_DIR}/3rdparty/cereal
+      ${CMAKE_SOURCE_DIR}/3rdparty/openpilot/msgq_repo
   )
 
   find_library(Cereal_LIBRARY_ZMQ
@@ -47,7 +52,7 @@ else (Cereal_LIBRARIES AND Cereal_INCLUDE_DIRS)
   set(Cereal_LIBRARIES
       ${Cereal_LIBRARIES}
       ${Cereal_LIBRARY_CEREAL}
-      ${Cereal_LIBRARY_MESSAGING}
+      ${Cereal_LIBRARY_MSGQ}
       ${Cereal_LIBRARY_ZMQ}
   )