Merge remote-tracking branch 'comma/master' into lp

.pre-commit-config.yaml

@@ -5,27 +5,13 @@ repos:
     -   id: check-ast
     -   id: check-yaml
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.4.1
+    rev: v1.5.1
     hooks:
     -   id: mypy
--   repo: https://github.com/PyCQA/flake8
-    rev: 6.1.0
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.0.285
     hooks:
-    -   id: flake8
-        args:
-        - --select=F,E112,E113,E304,E501,E502,E701,E702,E703,E71,E72,E731,W191,W6
-        - --max-line-length=240
-        - --statistics
--   repo: local
-    hooks:
-    -   id: pylint
-        name: pylint
-        entry: pylint
-        language: system
-        types: [python]
-        exclude: 'site_scons/'
-        args:
-        - --disable=C,R,W0613,W0511,W0212,W0201,W0311,W0106,W0603,W0621,W0703,E1136
+    -   id: ruff
 -   repo: local
     hooks:
     -   id: cppcheck

Dockerfile

@@ -38,7 +38,7 @@ ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}"
 RUN pyenv install 3.11.4 && \
     pyenv global 3.11.4 && \
     pyenv rehash && \
-    pip3 install --no-cache-dir pyyaml Cython scons pycapnp==1.1.0 pre-commit pylint parameterized coverage numpy
+    pip3 install --no-cache-dir pyyaml Cython scons pycapnp pre-commit ruff parameterized coverage numpy
 
 WORKDIR /project/
 RUN cd /tmp/ && \

car.capnp

@@ -450,6 +450,7 @@ struct CarParams {
 
   # things we can derive
   rotationalInertia @22 :Float32;    # [kg*m2] body rotational inertia
+  tireStiffnessFactor @72 :Float32;  # scaling factor used in calculating tireStiffness[Front,Rear]
   tireStiffnessFront @23 :Float32;   # [N/rad] front tire coeff of stiff
   tireStiffnessRear @24 :Float32;    # [N/rad] rear tire coeff of stiff
 
@@ -591,6 +592,7 @@ struct CarParams {
     hongqi @26;
     body @27;
     hyundaiCanfd @28;
+    volkswagenMqbEvo @29;
   }
 
   enum SteerControlType {

log.capnp

@@ -2098,6 +2098,8 @@ struct NavInstruction {
     left @1;
     right @2;
     straight @3;
+    slightLeft @4;
+    slightRight @5;
   }
 
   enum SpeedLimitSign {
@@ -2188,6 +2190,7 @@ struct Event {
     magnetometer @95 :SensorEventData;
     lightSensor @96 :SensorEventData;
     temperatureSensor @97 :SensorEventData;
+    temperatureSensor2 @123 :SensorEventData;
     pandaStates @81 :List(PandaState);
     peripheralState @80 :PeripheralState;
     radarState @13 :RadarState;

messaging/__init__.py

@@ -1,6 +1,7 @@
 # must be built with scons
-from .messaging_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event  # pylint: disable=no-name-in-module, import-error
-from .messaging_pyx import MultiplePublishersError, MessagingError  # pylint: disable=no-name-in-module, import-error
+from .messaging_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
+                                set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
+from .messaging_pyx import MultiplePublishersError, MessagingError
 
 import os
 import capnp
@@ -23,13 +24,6 @@
 NO_TRAVERSAL_LIMIT = 2**64-1
 AVG_FREQ_HISTORY = 100
 
-# sec_since_boot is faster, but allow to run standalone too
-try:
-  from common.realtime import sec_since_boot
-except ImportError:
-  sec_since_boot = time.time
-  print("Warning, using python time.time() instead of faster sec_since_boot")
-
 context = Context()
 
 
@@ -48,7 +42,7 @@ def log_from_bytes(dat: bytes) -> capnp.lib.capnp._DynamicStructReader:
 
 def new_message(service: Optional[str] = None, size: Optional[int] = None) -> capnp.lib.capnp._DynamicStructBuilder:
   dat = log.Event.new_message()
-  dat.logMonoTime = int(sec_since_boot() * 1e9)
+  dat.logMonoTime = int(time.monotonic() * 1e9)
   dat.valid = True
   if service is not None:
     if size is None:
@@ -211,7 +205,7 @@ def update(self, timeout: int = 1000) -> None:
     # non-blocking receive for non-polled sockets
     for s in self.non_polled_services:
       msgs.append(recv_one_or_none(self.sock[s]))
-    self.update_msgs(sec_since_boot(), msgs)
+    self.update_msgs(time.monotonic(), msgs)
 
   def update_msgs(self, cur_time: float, msgs: List[capnp.lib.capnp._DynamicStructReader]) -> None:
     self.frame += 1

messaging/bridge.cc

@@ -24,7 +24,7 @@ void sigpipe_handler(int sig) {
 static std::vector<std::string> get_services(std::string whitelist_str, bool zmq_to_msgq) {
   std::vector<std::string> service_list;
   for (const auto& it : services) {
-    std::string name = it.name;
+    std::string name = it.second.name;
     bool in_whitelist = whitelist_str.find(name) != std::string::npos;
     if (name == "plusFrame" || name == "uiLayoutState" || (zmq_to_msgq && !in_whitelist)) {
       continue;

messaging/impl_msgq.cc

@@ -17,12 +17,7 @@ void sig_handler(int signal) {
 }
 
 static bool service_exists(std::string path){
-  for (const auto& it : services) {
-    if (it.name == path) {
-      return true;
-    }
-  }
-  return false;
+  return services.count(path) > 0;
 }
 
 

messaging/impl_zmq.cc

@@ -8,17 +8,7 @@
 #include "cereal/messaging/impl_zmq.h"
 
 static int get_port(std::string endpoint) {
-  int port = -1;
-  for (const auto& it : services) {
-    std::string name = it.name;
-    if (name == endpoint) {
-      port = it.port;
-      break;
-    }
-  }
-
-  assert(port >= 0);
-  return port;
+  return services.at(endpoint).port;
 }
 
 ZMQContext::ZMQContext() {

messaging/messaging.h

@@ -116,6 +116,18 @@ class MessageBuilder : public capnp::MallocMessageBuilder {
     return heapArray_.asBytes();
   }
 
+  size_t getSerializedSize() {
+    return capnp::computeSerializedSizeInWords(*this) * sizeof(capnp::word);
+  }
+
+  int serializeToBuffer(unsigned char *buffer, size_t buffer_size) {
+    size_t serialized_size = getSerializedSize();
+    if (serialized_size > buffer_size) { return -1; }
+    kj::ArrayOutputStream out(kj::ArrayPtr<capnp::byte>(buffer, buffer_size));
+    capnp::writeMessage(out, *this);
+    return serialized_size;
+  }
+
 private:
   kj::Array<capnp::word> heapArray_;
 };

messaging/socketmaster.cc

@@ -15,13 +15,6 @@ static inline uint64_t nanos_since_boot() {
   return t.tv_sec * 1000000000ULL + t.tv_nsec;
 }
 
-static const service *get_service(const char *name) {
-  for (const auto &it : services) {
-    if (strcmp(it.name, name) == 0) return ⁢
-  }
-  return nullptr;
-}
-
 static inline bool inList(const std::vector<const char *> &list, const char *value) {
   for (auto &v : list) {
     if (strcmp(value, v) == 0) return true;
@@ -61,16 +54,17 @@ SubMaster::SubMaster(const std::vector<const char *> &service_list, const std::v
                      const char *address, const std::vector<const char *> &ignore_alive) {
   poller_ = Poller::create();
   for (auto name : service_list) {
-    const service *serv = get_service(name);
-    assert(serv != nullptr);
+    assert(services.count(std::string(name)) > 0);
+
+    service serv = services.at(std::string(name));
     SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true);
     assert(socket != 0);
     bool is_polled = inList(poll, name) || poll.empty();
     if (is_polled) poller_->registerSocket(socket);
     SubMessage *m = new SubMessage{
       .name = name,
       .socket = socket,
-      .freq = serv->frequency,
+      .freq = serv.frequency,
       .ignore_alive = inList(ignore_alive, name),
       .allocated_msg_reader = malloc(sizeof(capnp::FlatArrayMessageReader)),
       .is_polled = is_polled};
@@ -199,7 +193,7 @@ SubMaster::~SubMaster() {
 
 PubMaster::PubMaster(const std::vector<const char *> &service_list) {
   for (auto name : service_list) {
-    assert(get_service(name) != nullptr);
+    assert(services.count(name) > 0);
     PubSocket *socket = PubSocket::create(message_context.context(), name);
     assert(socket);
     sockets_[name] = socket;

messaging/tests/test_fake.py

@@ -3,6 +3,7 @@
 import multiprocessing
 import platform
 from parameterized import parameterized_class
+from typing import Optional
 
 import cereal.messaging as messaging
 
@@ -67,7 +68,7 @@ def test_wait_zero_timeout(self):
 @unittest.skipIf("ZMQ" in os.environ, "FakeSockets not supported on ZMQ")
 @parameterized_class([{"prefix": None}, {"prefix": "test"}])
 class TestFakeSockets(unittest.TestCase):
-  prefix = None
+  prefix: Optional[str] = None
 
   def setUp(self):
     messaging.toggle_fake_events(True)
@@ -151,7 +152,7 @@ def daemon_repub_process_run():
 
         bts = frame.to_bytes(8, 'little')
         pub_sock.send(bts)
-    
+
     carState_handle = messaging.fake_event_handle("carState", enable=True)
     recv_called = carState_handle.recv_called_event
     recv_ready = carState_handle.recv_ready_event

messaging/tests/test_messaging.py

@@ -19,7 +19,7 @@ def random_sock():
   return random.choice(events)
 
 def random_socks(num_socks=10):
-  return list(set([random_sock() for _ in range(num_socks)]))
+  return list({random_sock() for _ in range(num_socks)})
 
 def random_bytes(length=1000):
   return bytes([random.randrange(0xFF) for _ in range(length)])
@@ -169,13 +169,14 @@ def test_recv_sock(self):
     recvd = messaging.recv_sock(sub_sock)
     self.assertTrue(recvd is None)
 
-    # no wait and one msg in queue 
+    # no wait and one msg in queue
     msg = random_carstate()
     pub_sock.send(msg.to_bytes())
     time.sleep(0.01)
     recvd = messaging.recv_sock(sub_sock)
     self.assertIsInstance(recvd, capnp._DynamicStructReader)
-    assert_carstate(msg.carState, recvd.carState)
+    # https://github.com/python/mypy/issues/13038
+    assert_carstate(msg.carState, recvd.carState)  # type: ignore[union-attr]
 
   def test_recv_one(self):
     sock = "carState"
@@ -187,12 +188,12 @@ def test_recv_one(self):
     recvd = messaging.recv_one(sub_sock)
     self.assertTrue(recvd is None)
 
-    # one msg in queue 
+    # one msg in queue
     msg = random_carstate()
     pub_sock.send(msg.to_bytes())
     recvd = messaging.recv_one(sub_sock)
     self.assertIsInstance(recvd, capnp._DynamicStructReader)
-    assert_carstate(msg.carState, recvd.carState)
+    assert_carstate(msg.carState, recvd.carState)  # type: ignore[union-attr]
 
   @zmq_expected_failure
   def test_recv_one_or_none(self):
@@ -205,18 +206,18 @@ def test_recv_one_or_none(self):
     recvd = messaging.recv_one_or_none(sub_sock)
     self.assertTrue(recvd is None)
 
-    # one msg in queue 
+    # one msg in queue
     msg = random_carstate()
     pub_sock.send(msg.to_bytes())
     recvd = messaging.recv_one_or_none(sub_sock)
     self.assertIsInstance(recvd, capnp._DynamicStructReader)
-    assert_carstate(msg.carState, recvd.carState)
+    assert_carstate(msg.carState, recvd.carState)  # type: ignore[union-attr]
 
   def test_recv_one_retry(self):
     sock = "carState"
     sock_timeout = 0.1
     pub_sock = messaging.pub_sock(sock)
-    sub_sock = messaging.sub_sock(sock, timeout=sock_timeout*1000)
+    sub_sock = messaging.sub_sock(sock, timeout=round(sock_timeout*1000))
     zmq_sleep()
 
     # this test doesn't work with ZMQ since multiprocessing interrupts it

messaging/tests/test_pub_sub_master.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import random
 import time
+from typing import Sized, cast
 import unittest
 
 import cereal.messaging as messaging
@@ -20,7 +21,7 @@ def test_init(self):
     sm = messaging.SubMaster(events)
     for p in [sm.updated, sm.rcv_time, sm.rcv_frame, sm.alive,
               sm.sock, sm.freq, sm.data, sm.logMonoTime, sm.valid]:
-      self.assertEqual(len(p), len(events))
+      self.assertEqual(len(cast(Sized, p)), len(events))
 
   def test_init_state(self):
     socks = random_socks()
@@ -34,7 +35,7 @@ def test_init_state(self):
 
     for p in [sm.updated, sm.rcv_time, sm.rcv_frame, sm.alive,
               sm.sock, sm.freq, sm.data, sm.logMonoTime, sm.valid]:
-      self.assertEqual(len(p), len(socks))
+      self.assertEqual(len(cast(Sized, p)), len(socks))
 
   def test_getitem(self):
     sock = "carState"

messaging/tests/test_services.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import os
 import tempfile
+from typing import Dict
 import unittest
 from parameterized import parameterized
 
@@ -18,7 +19,7 @@ def test_services(self, s):
     self.assertTrue(service.frequency <= 104)
 
   def test_no_duplicate_port(self):
-    ports = {}
+    ports: Dict[int, str] = {}
     for name, service in service_list.items():
       self.assertFalse(service.port in ports.keys(), f"duplicate port {service.port}")
       ports[service.port] = name

pyproject.toml

@@ -0,0 +1,20 @@
+# https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml
+[tool.ruff]
+select = ["E", "F", "W", "PIE", "C4", "ISC", "RUF100", "A"]
+ignore = ["W292", "E741", "E402", "C408", "ISC003"]
+line-length = 160
+target-version="py311"
+flake8-implicit-str-concat.allow-multiline=false
+
+[mypy.tool]
+# third-party packages
+ignore_missing_imports=true
+
+# helpful warnings
+warn_redundant_casts=true
+warn_unreachable=true
+warn_unused_ignores=true
+
+# restrict dynamic typing
+warn_return_any=true
+check_untyped_defs=true