Merge branch 'lp' into lpdp

.github/workflows/tests.yml

@@ -42,7 +42,7 @@ jobs:
     - name: C++ tests
       run: |
         $RUN "export ${{ matrix.backend }}=1 && \
-              scons --test ${{ matrix.flags }} -j$(nproc) && \
+              scons ${{ matrix.flags }} -j$(nproc) && \
               messaging/test_runner && \
               visionipc/test_runner"
     - name: python tests

.pre-commit-config.yaml

@@ -1,15 +1,17 @@
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
     -   id: check-ast
     -   id: check-yaml
+    -   id: check-executables-have-shebangs
+    -   id: check-shebang-scripts-are-executable
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.5.1
+    rev: v1.8.0
     hooks:
     -   id: mypy
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.0.285
+    rev: v0.1.9
     hooks:
     -   id: ruff
 -   repo: local
@@ -35,9 +37,9 @@ repos:
         - --quiet
         - --counting=detailed
         - --linelength=240
-        - --filter=-legal,-build/include_order,-readability/casting,-whitespace/braces,-whitespace/indent,-whitespace/operators,-build/c++11,-readability/todo,-runtime/explicit,-runtime/int,-whitespace/comments,-readability/braces,-runtime/arrays,-runtime/references
+        - --filter=-build,-legal,-readability,-runtime,-whitespace,+build/include_subdir,+build/forward_decl,+build/include_what_you_use,+build/deprecated,+whitespace/comma,+whitespace/line_length,+whitespace/empty_if_body,+whitespace/empty_loop_body,+whitespace/empty_conditional_body,+whitespace/forcolon,+whitespace/parens,+whitespace/semicolon,+whitespace/tab,+readability/braces
 -   repo: https://github.com/codespell-project/codespell
-    rev: v2.2.5
+    rev: v2.2.6
     hooks:
     -   id: codespell
         args:

README.md

@@ -22,7 +22,7 @@ All `Events` have a `logMonoTime` and a `valid`. Then a big union defines the pa
 
 ### Maintaining backwards-compatibility
 
-When making changes to the messaging spec you want to maintain backwards-compatability, such that old logs can
+When making changes to the messaging spec you want to maintain backwards-compatibility, such that old logs can
 be parsed with a new version of cereal. Adding structs and adding members to structs is generally safe, most other
 things are not. Read more details [here](https://capnproto.org/language.html).
 

SConscript

@@ -71,7 +71,7 @@ else:
 envCython.Program('visionipc/visionipc_pyx.so', 'visionipc/visionipc_pyx.pyx',
                   LIBS=vipc_libs, FRAMEWORKS=vipc_frameworks)
 
-if GetOption('test'):
+if GetOption('extras'):
   env.Program('messaging/test_runner', ['messaging/test_runner.cc', 'messaging/msgq_tests.cc'], LIBS=[messaging_lib, common])
 
   env.Program('visionipc/test_runner', ['visionipc/test_runner.cc', 'visionipc/visionipc_tests.cc'],

SConstruct

@@ -23,9 +23,11 @@ libpath = [
   '/opt/homebrew/lib',
 ]
 
-AddOption('--test',
-          action='store_true',
-          help='build test files')
+AddOption('--minimal',
+          action='store_false',
+          dest='extras',
+          default=True,
+          help='the minimum build. no tests, tools, etc.')
 
 AddOption('--asan',
           action='store_true',

car.capnp

@@ -51,7 +51,6 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     parkBrake @29;
     manualRestart @30;
     lowSpeedLockout @31;
-    plannerError @32;
     joystickDebug @34;
     steerTempUnavailableSilent @35;
     resumeRequired @36;
@@ -63,7 +62,6 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     driverUnresponsive @45;
     belowSteerSpeed @46;
     lowBattery @48;
-    vehicleModelInvalid @50;
     accFaulted @51;
     sensorDataInvalid @52;
     commIssue @53;
@@ -106,7 +104,6 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     roadCameraError @100;
     driverCameraError @101;
     wideRoadCameraError @102;
-    localizerMalfunction @103;
     highCpuUsage @105;
     cruiseMismatch @106;
     lkasDisabled @107;
@@ -115,6 +112,10 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     resumeBlocked @113;
     steerTimeLimit @115;
     vehicleSensorsInvalid @116;
+    locationdTemporaryError @103;
+    locationdPermanentError @118;
+    paramsdTemporaryError @50;
+    paramsdPermanentError @119;
 
     radarCanErrorDEPRECATED @15;
     communityFeatureDisallowedDEPRECATED @62;
@@ -140,6 +141,7 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     startupFuzzyFingerprintDEPRECATED @97;
     noTargetDEPRECATED @25;
     brakeUnavailableDEPRECATED @2;
+    plannerErrorDEPRECATED @32;
   }
 }
 
@@ -458,8 +460,8 @@ struct CarParams {
   lateralParams @48 :LateralParams;
   lateralTuning :union {
     pid @26 :LateralPIDTuning;
-    indi @27 :LateralINDITuning;
-    lqr @40 :LateralLQRTuning;
+    indiDEPRECATED @27 :LateralINDITuning;
+    lqrDEPRECATED @40 :LateralLQRTuning;
     torque @67 :LateralTorqueTuning;
   }
 
@@ -482,6 +484,7 @@ struct CarParams {
   openpilotLongitudinalControl @37 :Bool; # is openpilot doing the longitudinal control?
   carVin @38 :Text; # VIN number queried during fingerprinting
   dashcamOnly @41: Bool;
+  passive @73: Bool;   # is openpilot in control?
   transmissionType @43 :TransmissionType;
   carFw @44 :List(CarFw);
 
@@ -598,7 +601,8 @@ struct CarParams {
   enum SteerControlType {
     torque @0;
     angle @1;
-    curvature @2;
+
+    curvatureDEPRECATED @2;
   }
 
   enum TransmissionType {

messaging/__init__.py

@@ -11,7 +11,7 @@
 from collections import deque
 
 from cereal import log
-from cereal.services import service_list
+from cereal.services import SERVICE_LIST
 
 assert MultiplePublishersError
 assert MessagingError
@@ -40,10 +40,13 @@ def log_from_bytes(dat: bytes) -> capnp.lib.capnp._DynamicStructReader:
   return log.Event.from_bytes(dat, traversal_limit_in_words=NO_TRAVERSAL_LIMIT)
 
 
-def new_message(service: Optional[str] = None, size: Optional[int] = None) -> capnp.lib.capnp._DynamicStructBuilder:
-  dat = log.Event.new_message()
-  dat.logMonoTime = int(time.monotonic() * 1e9)
-  dat.valid = True
+def new_message(service: Optional[str], size: Optional[int] = None, **kwargs) -> capnp.lib.capnp._DynamicStructBuilder:
+  args = {
+    'valid': False,
+    'logMonoTime': int(time.monotonic() * 1e9),
+    **kwargs
+  }
+  dat = log.Event.new_message(**args)
   if service is not None:
     if size is None:
       dat.init(service)
@@ -153,7 +156,7 @@ def recv_one_retry(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader:
 class SubMaster:
   def __init__(self, services: List[str], poll: Optional[List[str]] = None,
                ignore_alive: Optional[List[str]] = None, ignore_avg_freq: Optional[List[str]] = None,
-               addr: str = "127.0.0.1"):
+               ignore_valid: Optional[List[str]] = None, addr: str = "127.0.0.1"):
     self.frame = -1
     self.updated = {s: False for s in services}
     self.rcv_time = {s: 0. for s in services}
@@ -173,22 +176,24 @@ def __init__(self, services: List[str], poll: Optional[List[str]] = None,
 
     self.ignore_average_freq = [] if ignore_avg_freq is None else ignore_avg_freq
     self.ignore_alive = [] if ignore_alive is None else ignore_alive
+    self.ignore_valid = [] if ignore_valid is None else ignore_valid
     self.simulation = bool(int(os.getenv("SIMULATION", "0")))
 
     for s in services:
       if addr is not None:
         p = self.poller if s not in self.non_polled_services else None
         self.sock[s] = sub_sock(s, poller=p, addr=addr, conflate=True)
-      self.freq[s] = service_list[s].frequency
+      self.freq[s] = SERVICE_LIST[s].frequency
 
       try:
         data = new_message(s)
       except capnp.lib.capnp.KjException:  # pylint: disable=c-extension-no-member
         data = new_message(s, 0) # lists
 
-      self.data[s] = getattr(data, s)
+      self.data[s] = getattr(data.as_reader(), s)
       self.logMonoTime[s] = 0
-      self.valid[s] = data.valid
+      # TODO: this should default to False
+      self.valid[s] = True
 
   def __getitem__(self, s: str) -> capnp.lib.capnp._DynamicStructReader:
     return self.data[s]
@@ -265,7 +270,7 @@ def all_freq_ok(self, service_list=None) -> bool:
   def all_valid(self, service_list=None) -> bool:
     if service_list is None:  # check all
       service_list = self.valid.keys()
-    return all(self.valid[s] for s in service_list)
+    return all(self.valid[s] for s in service_list if s not in self.ignore_valid)
 
   def all_checks(self, service_list=None) -> bool:
     if service_list is None:  # check all
@@ -286,8 +291,7 @@ def send(self, s: str, dat: Union[bytes, capnp.lib.capnp._DynamicStructBuilder])
       dat = dat.to_bytes()
     self.sock[s].send(dat)
 
-  def wait_for_readers_to_update(self, s: str, timeout: int) -> bool:
-    dt = 0.05
+  def wait_for_readers_to_update(self, s: str, timeout: int, dt: float = 0.05) -> bool:
     for _ in range(int(timeout*(1./dt))):
       if self.sock[s].all_readers_updated():
         return True

messaging/impl_fake.h

@@ -63,5 +63,5 @@ class FakePoller: public Poller {
 public:
   void registerSocket(SubSocket *socket) override;
   std::vector<SubSocket*> poll(int timeout) override;
-  ~FakePoller() {};
+  ~FakePoller() {}
 };

messaging/impl_msgq.h

@@ -63,5 +63,5 @@ class MSGQPoller : public Poller {
 public:
   void registerSocket(SubSocket *socket);
   std::vector<SubSocket*> poll(int timeout);
-  ~MSGQPoller(){};
+  ~MSGQPoller(){}
 };

messaging/impl_zmq.cc

@@ -3,6 +3,7 @@
 #include <iostream>
 #include <cstdlib>
 #include <cerrno>
+#include <unistd.h>
 
 #include "cereal/services.h"
 #include "cereal/messaging/impl_zmq.h"
@@ -108,14 +109,19 @@ int ZMQPubSocket::connect(Context *context, std::string endpoint, bool check_end
     full_endpoint += endpoint;
   }
 
+  // ZMQ pub sockets cannot be shared between processes, so we need to ensure pid stays the same
+  pid = getpid();
+
   return zmq_bind(sock, full_endpoint.c_str());
 }
 
-int ZMQPubSocket::sendMessage(Message *message){
+int ZMQPubSocket::sendMessage(Message *message) {
+  assert(pid == getpid());
   return zmq_send(sock, message->getData(), message->getSize(), ZMQ_DONTWAIT);
 }
 
-int ZMQPubSocket::send(char *data, size_t size){
+int ZMQPubSocket::send(char *data, size_t size) {
+  assert(pid == getpid());
   return zmq_send(sock, data, size, ZMQ_DONTWAIT);
 }
 

messaging/impl_zmq.h

@@ -46,6 +46,7 @@ class ZMQPubSocket : public PubSocket {
 private:
   void * sock;
   std::string full_endpoint;
+  int pid = -1;
 public:
   int connect(Context *context, std::string endpoint, bool check_endpoint=true);
   int sendMessage(Message *message);
@@ -63,5 +64,5 @@ class ZMQPoller : public Poller {
 public:
   void registerSocket(SubSocket *socket);
   std::vector<SubSocket*> poll(int timeout);
-  ~ZMQPoller(){};
+  ~ZMQPoller(){}
 };

messaging/messaging.h

@@ -33,7 +33,7 @@ class Message {
   virtual void close() = 0;
   virtual size_t getSize() = 0;
   virtual char * getData() = 0;
-  virtual ~Message(){};
+  virtual ~Message(){}
 };
 
 
@@ -45,7 +45,7 @@ class SubSocket {
   virtual void * getRawSocket() = 0;
   static SubSocket * create();
   static SubSocket * create(Context * context, std::string endpoint, std::string address="127.0.0.1", bool conflate=false, bool check_endpoint=true);
-  virtual ~SubSocket(){};
+  virtual ~SubSocket(){}
 };
 
 class PubSocket {
@@ -57,7 +57,7 @@ class PubSocket {
   static PubSocket * create();
   static PubSocket * create(Context * context, std::string endpoint, bool check_endpoint=true);
   static PubSocket * create(Context * context, std::string endpoint, int port, bool check_endpoint=true);
-  virtual ~PubSocket(){};
+  virtual ~PubSocket(){}
 };
 
 class Poller {
@@ -66,7 +66,7 @@ class Poller {
   virtual std::vector<SubSocket*> poll(int timeout) = 0;
   static Poller * create();
   static Poller * create(std::vector<SubSocket*> sockets);
-  virtual ~Poller(){};
+  virtual ~Poller(){}
 };
 
 class SubMaster {

messaging/socketmaster.cc

@@ -24,7 +24,7 @@ static inline bool inList(const std::vector<const char *> &list, const char *val
 
 class MessageContext {
 public:
-  MessageContext() : ctx_(nullptr) {};
+  MessageContext() : ctx_(nullptr) {}
   ~MessageContext() { delete ctx_; }
   inline Context *context() {
     std::call_once(init_flag, [=]() { ctx_ = Context::create(); });
@@ -178,7 +178,7 @@ uint64_t SubMaster::rcv_time(const char *name) const {
 
 cereal::Event::Reader &SubMaster::operator[](const char *name) const {
   return services_.at(name)->event;
-};
+}
 
 SubMaster::~SubMaster() {
   delete poller_;

messaging/tests/test_messaging.py

@@ -11,9 +11,9 @@
 
 from cereal import log, car
 import cereal.messaging as messaging
-from cereal.services import service_list
+from cereal.services import SERVICE_LIST
 
-events = [evt for evt in log.Event.schema.union_fields if evt in service_list.keys()]
+events = [evt for evt in log.Event.schema.union_fields if evt in SERVICE_LIST.keys()]
 
 def random_sock():
   return random.choice(events)
@@ -123,7 +123,7 @@ def test_new_message(self, evt):
     except capnp.lib.capnp.KjException:
       msg = messaging.new_message(evt, random.randrange(200))
     self.assertLess(time.monotonic() - msg.logMonoTime, 0.1)
-    self.assertTrue(msg.valid)
+    self.assertFalse(msg.valid)
     self.assertEqual(evt, msg.which())
 
   @parameterized.expand(events)

messaging/tests/test_services.py

@@ -6,21 +6,21 @@
 from parameterized import parameterized
 
 import cereal.services as services
-from cereal.services import service_list, RESERVED_PORT, STARTING_PORT
+from cereal.services import SERVICE_LIST, RESERVED_PORT, STARTING_PORT
 
 
 class TestServices(unittest.TestCase):
 
-  @parameterized.expand(service_list.keys())
+  @parameterized.expand(SERVICE_LIST.keys())
   def test_services(self, s):
-    service = service_list[s]
+    service = SERVICE_LIST[s]
     self.assertTrue(service.port != RESERVED_PORT)
     self.assertTrue(service.port >= STARTING_PORT)
     self.assertTrue(service.frequency <= 104)
 
   def test_no_duplicate_port(self):
     ports: Dict[int, str] = {}
-    for name, service in service_list.items():
+    for name, service in SERVICE_LIST.items():
       self.assertFalse(service.port in ports.keys(), f"duplicate port {service.port}")
       ports[service.port] = name
 

visionipc/tests/test_visionipc.py

@@ -33,7 +33,7 @@ def test_connect(self):
     self.assertTrue(self.client.is_connected)
 
   def test_available_streams(self):
-    for k in range(0, 4):
+    for k in range(4):
       stream_types = set(random.choices([x.value for x in VisionStreamType], k=k))
       self.setup_vipc("camerad", *stream_types)
       available_streams = VisionIpcClient.available_streams("camerad", True)