From eb1b2ce255371d7529e1b4341600ecca5052e1f4 Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Thu, 24 Aug 2023 12:29:59 +0100 Subject: [PATCH 01/54] services: add missing import (#521) --- services.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services.py b/services.py index da99a4094..4e18d5971 100755 --- a/services.py +++ b/services.py @@ -107,6 +107,7 @@ def build_header(): h += "#define __SERVICES_H\n" h += "#include \n" + h += "#include \n" h += "struct service { std::string name; int port; bool should_log; int frequency; int decimation; };\n" h += "static std::map services = {\n" From bf8db88407d8e1daf662adaa9901da639a979dc4 Mon Sep 17 00:00:00 2001 From: Dean Lee Date: Fri, 25 Aug 2023 02:31:53 +0800 Subject: [PATCH 02/54] cpplint: more filters (#522) more filters --- .pre-commit-config.yaml | 2 +- messaging/impl_fake.h | 2 +- messaging/impl_msgq.h | 2 +- messaging/impl_zmq.h | 2 +- messaging/messaging.h | 8 ++++---- messaging/socketmaster.cc | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 986426503..b49cd63d0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,7 @@ repos: - --quiet - --counting=detailed - --linelength=240 - - --filter=-legal,-build/include_order,-readability/casting,-whitespace/braces,-whitespace/indent,-whitespace/operators,-build/c++11,-readability/todo,-runtime/explicit,-runtime/int,-whitespace/comments,-readability/braces,-runtime/arrays,-runtime/references + - --filter=-build,-legal,-readability,-runtime,-whitespace,+build/include_subdir,+build/forward_decl,+build/include_what_you_use,+build/deprecated,+whitespace/comma,+whitespace/line_length,+whitespace/empty_if_body,+whitespace/empty_loop_body,+whitespace/empty_conditional_body,+whitespace/forcolon,+whitespace/parens,+whitespace/semicolon,+whitespace/tab,+readability/braces - repo: https://github.com/codespell-project/codespell rev: v2.2.5 hooks: diff --git a/messaging/impl_fake.h b/messaging/impl_fake.h index 0b3f6eb41..0ec8486a0 100644 --- a/messaging/impl_fake.h +++ b/messaging/impl_fake.h @@ -63,5 +63,5 @@ class FakePoller: public Poller { public: void registerSocket(SubSocket *socket) override; std::vector poll(int timeout) override; - ~FakePoller() {}; + ~FakePoller() {} }; diff --git a/messaging/impl_msgq.h b/messaging/impl_msgq.h index fdad52f9e..68235f0c6 100644 --- a/messaging/impl_msgq.h +++ b/messaging/impl_msgq.h @@ -63,5 +63,5 @@ class MSGQPoller : public Poller { public: void registerSocket(SubSocket *socket); std::vector poll(int timeout); - ~MSGQPoller(){}; + ~MSGQPoller(){} }; diff --git a/messaging/impl_zmq.h b/messaging/impl_zmq.h index deeac859c..34864f633 100644 --- a/messaging/impl_zmq.h +++ b/messaging/impl_zmq.h @@ -63,5 +63,5 @@ class ZMQPoller : public Poller { public: void registerSocket(SubSocket *socket); std::vector poll(int timeout); - ~ZMQPoller(){}; + ~ZMQPoller(){} }; diff --git a/messaging/messaging.h b/messaging/messaging.h index d768ccb9b..91106514a 100644 --- a/messaging/messaging.h +++ b/messaging/messaging.h @@ -33,7 +33,7 @@ class Message { virtual void close() = 0; virtual size_t getSize() = 0; virtual char * getData() = 0; - virtual ~Message(){}; + virtual ~Message(){} }; @@ -45,7 +45,7 @@ class SubSocket { virtual void * getRawSocket() = 0; static SubSocket * create(); static SubSocket * create(Context * context, std::string endpoint, std::string address="127.0.0.1", bool conflate=false, bool check_endpoint=true); - virtual ~SubSocket(){}; + virtual ~SubSocket(){} }; class PubSocket { @@ -57,7 +57,7 @@ class PubSocket { static PubSocket * create(); static PubSocket * create(Context * context, std::string endpoint, bool check_endpoint=true); static PubSocket * create(Context * context, std::string endpoint, int port, bool check_endpoint=true); - virtual ~PubSocket(){}; + virtual ~PubSocket(){} }; class Poller { @@ -66,7 +66,7 @@ class Poller { virtual std::vector poll(int timeout) = 0; static Poller * create(); static Poller * create(std::vector sockets); - virtual ~Poller(){}; + virtual ~Poller(){} }; class SubMaster { diff --git a/messaging/socketmaster.cc b/messaging/socketmaster.cc index f0f88dac8..3054b4f7d 100644 --- a/messaging/socketmaster.cc +++ b/messaging/socketmaster.cc @@ -24,7 +24,7 @@ static inline bool inList(const std::vector &list, const char *val class MessageContext { public: - MessageContext() : ctx_(nullptr) {}; + MessageContext() : ctx_(nullptr) {} ~MessageContext() { delete ctx_; } inline Context *context() { std::call_once(init_flag, [=]() { ctx_ = Context::create(); }); @@ -178,7 +178,7 @@ uint64_t SubMaster::rcv_time(const char *name) const { cereal::Event::Reader &SubMaster::operator[](const char *name) const { return services_.at(name)->event; -}; +} SubMaster::~SubMaster() { delete poller_; From 82bca3a9714b73c05414fdf848b6016a0ffac17d Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Thu, 24 Aug 2023 14:00:32 -0700 Subject: [PATCH 03/54] add --minimal build flag (#523) * add --minimal build flag * update that --- .github/workflows/tests.yml | 2 +- SConscript | 2 +- SConstruct | 8 +++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d0be50f68..d9bb1065b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -42,7 +42,7 @@ jobs: - name: C++ tests run: | $RUN "export ${{ matrix.backend }}=1 && \ - scons --test ${{ matrix.flags }} -j$(nproc) && \ + scons ${{ matrix.flags }} -j$(nproc) && \ messaging/test_runner && \ visionipc/test_runner" - name: python tests diff --git a/SConscript b/SConscript index 192e42d0e..a5d9d6bea 100644 --- a/SConscript +++ b/SConscript @@ -69,7 +69,7 @@ else: envCython.Program('visionipc/visionipc_pyx.so', 'visionipc/visionipc_pyx.pyx', LIBS=vipc_libs, FRAMEWORKS=vipc_frameworks) -if GetOption('test'): +if GetOption('extras'): env.Program('messaging/test_runner', ['messaging/test_runner.cc', 'messaging/msgq_tests.cc'], LIBS=[messaging_lib, common]) env.Program('visionipc/test_runner', ['visionipc/test_runner.cc', 'visionipc/visionipc_tests.cc'], diff --git a/SConstruct b/SConstruct index 46c2352bd..bf2a53702 100644 --- a/SConstruct +++ b/SConstruct @@ -21,9 +21,11 @@ libpath = [ '/opt/homebrew/lib', ] -AddOption('--test', - action='store_true', - help='build test files') +AddOption('--minimal', + action='store_false', + dest='extras', + default=True, + help='the minimum build. no tests, tools, etc.') AddOption('--asan', action='store_true', From cdd76b2972b74e8e735588a58cee56833f45e20b Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Thu, 31 Aug 2023 14:38:04 -0700 Subject: [PATCH 04/54] pre-commit: autoupdate hooks (#525) * Update pre-commit hook versions * Update test_visionipc.py --------- Co-authored-by: adeebshihadeh --- .pre-commit-config.yaml | 2 +- visionipc/tests/test_visionipc.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b49cd63d0..83aa3e0e7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: hooks: - id: mypy - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.285 + rev: v0.0.286 hooks: - id: ruff - repo: local diff --git a/visionipc/tests/test_visionipc.py b/visionipc/tests/test_visionipc.py index 1c00219ec..7bf075d7f 100755 --- a/visionipc/tests/test_visionipc.py +++ b/visionipc/tests/test_visionipc.py @@ -33,7 +33,7 @@ def test_connect(self): self.assertTrue(self.client.is_connected) def test_available_streams(self): - for k in range(0, 4): + for k in range(4): stream_types = set(random.choices([x.value for x in VisionStreamType], k=k)) self.setup_vipc("camerad", *stream_types) available_streams = VisionIpcClient.available_streams("camerad", True) From a283061ab7ff09ff53fc77a5db6b7e254f365619 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Tue, 5 Sep 2023 09:51:52 -0700 Subject: [PATCH 05/54] pre-commit: autoupdate hooks (#528) Co-authored-by: adeebshihadeh --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 83aa3e0e7..5060f0404 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: hooks: - id: mypy - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.286 + rev: v0.0.287 hooks: - id: ruff - repo: local From 4291784b4d372782c95279e9fe7741e38633ca5e Mon Sep 17 00:00:00 2001 From: Justin Newberry Date: Tue, 5 Sep 2023 11:02:17 -0700 Subject: [PATCH 06/54] Messaging: DT Param on wait_for_readers_to_update (#529) * dt param * same as the others --- messaging/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/messaging/__init__.py b/messaging/__init__.py index b671a3ef7..fd9a48772 100644 --- a/messaging/__init__.py +++ b/messaging/__init__.py @@ -287,8 +287,7 @@ def send(self, s: str, dat: Union[bytes, capnp.lib.capnp._DynamicStructBuilder]) dat = dat.to_bytes() self.sock[s].send(dat) - def wait_for_readers_to_update(self, s: str, timeout: int) -> bool: - dt = 0.05 + def wait_for_readers_to_update(self, s: str, timeout: int, dt: float = 0.05) -> bool: for _ in range(int(timeout*(1./dt))): if self.sock[s].all_readers_updated(): return True From d469732b3b4c37aaa21fa37682a20f2aba9ab91c Mon Sep 17 00:00:00 2001 From: Justin Newberry Date: Tue, 5 Sep 2023 15:35:01 -0700 Subject: [PATCH 07/54] Precommit: ensure executable bit is set (#530) precommit ensure executable --- .pre-commit-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5060f0404..1e31583e2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,6 +4,8 @@ repos: hooks: - id: check-ast - id: check-yaml + - id: check-executables-have-shebangs + - id: check-shebang-scripts-are-executable - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.5.1 hooks: From 7de568b65922b1b7a5cb9a9a391e3e03394500f7 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sun, 10 Sep 2023 16:14:14 -0700 Subject: [PATCH 08/54] deprecate INDI and LQR tuning --- car.capnp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/car.capnp b/car.capnp index 1039e5cbe..6ae29d702 100644 --- a/car.capnp +++ b/car.capnp @@ -458,8 +458,8 @@ struct CarParams { lateralParams @48 :LateralParams; lateralTuning :union { pid @26 :LateralPIDTuning; - indi @27 :LateralINDITuning; - lqr @40 :LateralLQRTuning; + indiDEPRECATED @27 :LateralINDITuning; + lqrDEPRECATED @40 :LateralLQRTuning; torque @67 :LateralTorqueTuning; } From 4b334f6f10877e4a666b23983de2d27934ebf3b1 Mon Sep 17 00:00:00 2001 From: Vivek Aithal Date: Mon, 11 Sep 2023 16:04:55 -0700 Subject: [PATCH 09/54] Alerts: More understandable locationd, paramsd alerts (#531) * add new alerts * rename old alerts, add new alerts --- car.capnp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/car.capnp b/car.capnp index 6ae29d702..29949a148 100644 --- a/car.capnp +++ b/car.capnp @@ -63,7 +63,6 @@ struct CarEvent @0x9b1657f34caf3ad3 { driverUnresponsive @45; belowSteerSpeed @46; lowBattery @48; - vehicleModelInvalid @50; accFaulted @51; sensorDataInvalid @52; commIssue @53; @@ -106,7 +105,6 @@ struct CarEvent @0x9b1657f34caf3ad3 { roadCameraError @100; driverCameraError @101; wideRoadCameraError @102; - localizerMalfunction @103; highCpuUsage @105; cruiseMismatch @106; lkasDisabled @107; @@ -115,6 +113,10 @@ struct CarEvent @0x9b1657f34caf3ad3 { resumeBlocked @113; steerTimeLimit @115; vehicleSensorsInvalid @116; + locationdTemporaryError @103; + locationdPermanentError @118; + paramsdTemporaryError @50; + paramsdPermanentError @119; radarCanErrorDEPRECATED @15; communityFeatureDisallowedDEPRECATED @62; From e342bc48886e4d69bb73d9159c58e1f019bfe59e Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Tue, 12 Sep 2023 08:30:48 -0700 Subject: [PATCH 10/54] pre-commit: autoupdate hooks (#532) Co-authored-by: adeebshihadeh --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1e31583e2..5d653b7b6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: hooks: - id: mypy - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.287 + rev: v0.0.288 hooks: - id: ruff - repo: local From ff3dbaacb296eaffeeae3324f91920e635722a61 Mon Sep 17 00:00:00 2001 From: JJ <103335846+computerscienceiscool@users.noreply.github.com> Date: Fri, 15 Sep 2023 10:44:30 -0700 Subject: [PATCH 11/54] Update README.md (#533) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 737534e36..e3326aab0 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ All `Events` have a `logMonoTime` and a `valid`. Then a big union defines the pa ### Maintaining backwards-compatibility -When making changes to the messaging spec you want to maintain backwards-compatability, such that old logs can +When making changes to the messaging spec you want to maintain backwards-compatibility, such that old logs can be parsed with a new version of cereal. Adding structs and adding members to structs is generally safe, most other things are not. Read more details [here](https://capnproto.org/language.html). From 25cfa642e600f56d83c392d045e35dd34e7adf4c Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Tue, 19 Sep 2023 16:16:39 -0700 Subject: [PATCH 12/54] pre-commit: autoupdate hooks (#535) Update pre-commit hook versions Co-authored-by: adeebshihadeh --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5d653b7b6..8280179bf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: hooks: - id: mypy - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.288 + rev: v0.0.290 hooks: - id: ruff - repo: local From aed84aeedf8fa7fe2a6bae700ccaf9667d3bdbb0 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 21 Sep 2023 18:01:44 -0700 Subject: [PATCH 13/54] services: capitalize constant service list (#534) * rename services * keep more unique name * type this --- messaging/__init__.py | 4 ++-- messaging/tests/test_messaging.py | 4 ++-- messaging/tests/test_services.py | 8 ++++---- services.py | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/messaging/__init__.py b/messaging/__init__.py index fd9a48772..57675212f 100644 --- a/messaging/__init__.py +++ b/messaging/__init__.py @@ -11,7 +11,7 @@ from collections import deque from cereal import log -from cereal.services import service_list +from cereal.services import SERVICE_LIST assert MultiplePublishersError assert MessagingError @@ -180,7 +180,7 @@ def __init__(self, services: List[str], poll: Optional[List[str]] = None, if addr is not None: p = self.poller if s not in self.non_polled_services else None self.sock[s] = sub_sock(s, poller=p, addr=addr, conflate=True) - self.freq[s] = service_list[s].frequency + self.freq[s] = SERVICE_LIST[s].frequency try: data = new_message(s) diff --git a/messaging/tests/test_messaging.py b/messaging/tests/test_messaging.py index 063351b75..f9c85e0dc 100755 --- a/messaging/tests/test_messaging.py +++ b/messaging/tests/test_messaging.py @@ -11,9 +11,9 @@ from cereal import log, car import cereal.messaging as messaging -from cereal.services import service_list +from cereal.services import SERVICE_LIST -events = [evt for evt in log.Event.schema.union_fields if evt in service_list.keys()] +events = [evt for evt in log.Event.schema.union_fields if evt in SERVICE_LIST.keys()] def random_sock(): return random.choice(events) diff --git a/messaging/tests/test_services.py b/messaging/tests/test_services.py index 6452b5b6e..f26bdbc5f 100755 --- a/messaging/tests/test_services.py +++ b/messaging/tests/test_services.py @@ -6,21 +6,21 @@ from parameterized import parameterized import cereal.services as services -from cereal.services import service_list, RESERVED_PORT, STARTING_PORT +from cereal.services import SERVICE_LIST, RESERVED_PORT, STARTING_PORT class TestServices(unittest.TestCase): - @parameterized.expand(service_list.keys()) + @parameterized.expand(SERVICE_LIST.keys()) def test_services(self, s): - service = service_list[s] + service = SERVICE_LIST[s] self.assertTrue(service.port != RESERVED_PORT) self.assertTrue(service.port >= STARTING_PORT) self.assertTrue(service.frequency <= 104) def test_no_duplicate_port(self): ports: Dict[int, str] = {} - for name, service in service_list.items(): + for name, service in SERVICE_LIST.items(): self.assertFalse(service.port in ports.keys(), f"duplicate port {service.port}") ports[service.port] = name diff --git a/services.py b/services.py index 4e18d5971..e9402f5cb 100755 --- a/services.py +++ b/services.py @@ -18,7 +18,7 @@ def __init__(self, port: int, should_log: bool, frequency: float, decimation: Op self.decimation = decimation -services = { +services: dict[str, tuple] = { # service: (should_log, frequency, qlog decimation (optional)) # note: the "EncodeIdx" packets will still be in the log "gyroscope": (True, 104., 104), @@ -96,7 +96,7 @@ def __init__(self, port: int, should_log: bool, frequency: float, decimation: Op "livestreamRoadEncodeData": (False, 20.), "livestreamDriverEncodeData": (False, 20.), } -service_list = {name: Service(new_port(idx), *vals) for # type: ignore +SERVICE_LIST = {name: Service(new_port(idx), *vals) for idx, (name, vals) in enumerate(services.items())} @@ -111,7 +111,7 @@ def build_header(): h += "struct service { std::string name; int port; bool should_log; int frequency; int decimation; };\n" h += "static std::map services = {\n" - for k, v in service_list.items(): + for k, v in SERVICE_LIST.items(): should_log = "true" if v.should_log else "false" decimation = -1 if v.decimation is None else v.decimation h += ' { "%s", {"%s", %d, %s, %d, %d}},\n' % \ From 6bd65e7c7f2692d15a9b6cc97bad9345f9089c4d Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 21 Sep 2023 18:27:48 -0700 Subject: [PATCH 14/54] Bump pandaStates to 10hz (#536) * 10 Hz * constant * fix * revert * fix peripheralState freq * revert peripheral --- services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services.py b/services.py index e9402f5cb..915398afb 100755 --- a/services.py +++ b/services.py @@ -33,7 +33,7 @@ def __init__(self, port: int, should_log: bool, frequency: float, decimation: Op "deviceState": (True, 2., 1), "can": (True, 100., 1223), # decimation gives ~5 msgs in a full segment "controlsState": (True, 100., 10), - "pandaStates": (True, 2., 1), + "pandaStates": (True, 10., 1), "peripheralState": (True, 2., 1), "radarState": (True, 20., 5), "roadEncodeIdx": (False, 20., 1), From 457255840c2bf42435392542193e45ac21fd659a Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Tue, 26 Sep 2023 14:10:03 -0700 Subject: [PATCH 15/54] pre-commit: autoupdate hooks (#538) Update pre-commit hook versions Co-authored-by: adeebshihadeh --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8280179bf..9c92fef35 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: hooks: - id: mypy - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.290 + rev: v0.0.291 hooks: - id: ruff - repo: local From b1a1afebb8533a96ff0e8efbba7c10eb47df2af0 Mon Sep 17 00:00:00 2001 From: Mitchell Goff Date: Wed, 27 Sep 2023 16:38:06 -0700 Subject: [PATCH 16/54] Added reserved fields for body controls (#537) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added bodyReserved0-2 text fields * Added bodyReserved0 to services list * add more bodyReserved to servcies py * bodyReserved -> customReservedText * :Text -> :Data --------- Co-authored-by: Kacper Rączy --- log.capnp | 4 ++++ services.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/log.capnp b/log.capnp index 70efde241..4a59ca8d3 100644 --- a/log.capnp +++ b/log.capnp @@ -2265,6 +2265,10 @@ struct Event { livestreamWideRoadEncodeData @121 :EncodeData; livestreamDriverEncodeData @122 :EncodeData; + customReservedRawData0 @124 :Data; + customReservedRawData1 @125 :Data; + customReservedRawData2 @126 :Data; + # *********** Custom: reserved for forks *********** customReserved0 @107 :Custom.CustomReserved0; customReserved1 @108 :Custom.CustomReserved1; diff --git a/services.py b/services.py index 915398afb..9c08e6352 100755 --- a/services.py +++ b/services.py @@ -95,6 +95,9 @@ def __init__(self, port: int, should_log: bool, frequency: float, decimation: Op "livestreamWideRoadEncodeData": (False, 20.), "livestreamRoadEncodeData": (False, 20.), "livestreamDriverEncodeData": (False, 20.), + "customReservedRawData0": (True, 0.), + "customReservedRawData1": (True, 0.), + "customReservedRawData2": (True, 0.), } SERVICE_LIST = {name: Service(new_port(idx), *vals) for idx, (name, vals) in enumerate(services.items())} From acb24ff8c21d343f05a6eacb061d137144a67a58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20R=C4=85czy?= Date: Mon, 2 Oct 2023 15:03:39 -0700 Subject: [PATCH 17/54] zmq: assert the same PID (#540) Enforce the same PID in ZMQ pub sockets --- messaging/impl_zmq.cc | 10 ++++++++-- messaging/impl_zmq.h | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/messaging/impl_zmq.cc b/messaging/impl_zmq.cc index aa95d94fb..7da9df1b8 100644 --- a/messaging/impl_zmq.cc +++ b/messaging/impl_zmq.cc @@ -3,6 +3,7 @@ #include #include #include +#include #include "cereal/services.h" #include "cereal/messaging/impl_zmq.h" @@ -108,14 +109,19 @@ int ZMQPubSocket::connect(Context *context, std::string endpoint, bool check_end full_endpoint += endpoint; } + // ZMQ pub sockets cannot be shared between processes, so we need to ensure pid stays the same + pid = getpid(); + return zmq_bind(sock, full_endpoint.c_str()); } -int ZMQPubSocket::sendMessage(Message *message){ +int ZMQPubSocket::sendMessage(Message *message) { + assert(pid == getpid()); return zmq_send(sock, message->getData(), message->getSize(), ZMQ_DONTWAIT); } -int ZMQPubSocket::send(char *data, size_t size){ +int ZMQPubSocket::send(char *data, size_t size) { + assert(pid == getpid()); return zmq_send(sock, data, size, ZMQ_DONTWAIT); } diff --git a/messaging/impl_zmq.h b/messaging/impl_zmq.h index 34864f633..903875f63 100644 --- a/messaging/impl_zmq.h +++ b/messaging/impl_zmq.h @@ -46,6 +46,7 @@ class ZMQPubSocket : public PubSocket { private: void * sock; std::string full_endpoint; + int pid = -1; public: int connect(Context *context, std::string endpoint, bool check_endpoint=true); int sendMessage(Message *message); From 4a07275487aa0280badd315f6c46b19fecb3fdbc Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Tue, 3 Oct 2023 11:34:38 -0700 Subject: [PATCH 18/54] pre-commit: autoupdate hooks (#541) Update pre-commit hook versions Co-authored-by: adeebshihadeh --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9c92fef35..e91d3ae9f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: hooks: - id: mypy - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.291 + rev: v0.0.292 hooks: - id: ruff - repo: local @@ -39,7 +39,7 @@ repos: - --linelength=240 - --filter=-build,-legal,-readability,-runtime,-whitespace,+build/include_subdir,+build/forward_decl,+build/include_what_you_use,+build/deprecated,+whitespace/comma,+whitespace/line_length,+whitespace/empty_if_body,+whitespace/empty_loop_body,+whitespace/empty_conditional_body,+whitespace/forcolon,+whitespace/parens,+whitespace/semicolon,+whitespace/tab,+readability/braces - repo: https://github.com/codespell-project/codespell - rev: v2.2.5 + rev: v2.2.6 hooks: - id: codespell args: From 24a522f6ba47aba12a9baea53020a8323673c79c Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sun, 15 Oct 2023 14:55:54 -0700 Subject: [PATCH 19/54] add initData.wallTimeNanos (#544) --- log.capnp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/log.capnp b/log.capnp index 4a59ca8d3..9777a3c32 100644 --- a/log.capnp +++ b/log.capnp @@ -46,6 +46,8 @@ struct InitData { commands @19 :Map(Text, Data); + wallTimeNanos @20 :UInt64; + enum DeviceType { unknown @0; neo @1; From f5b3915d14c0350f76d763c7d0fc0d6cc2c7aa8f Mon Sep 17 00:00:00 2001 From: Dean Lee Date: Tue, 17 Oct 2023 00:01:03 +0800 Subject: [PATCH 20/54] vipc_client: remove `init_msgq` (#545) remove init_msgq --- visionipc/visionipc_client.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/visionipc/visionipc_client.h b/visionipc/visionipc_client.h index 8dff9b2e0..d67734d32 100644 --- a/visionipc/visionipc_client.h +++ b/visionipc/visionipc_client.h @@ -19,8 +19,6 @@ class VisionIpcClient { cl_device_id device_id = nullptr; cl_context ctx = nullptr; - void init_msgq(bool conflate); - public: bool connected = false; VisionStreamType type; From bc382148d0d418ad90e0008195da67aced30e208 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Thu, 19 Oct 2023 11:29:26 -0700 Subject: [PATCH 21/54] pre-commit: autoupdate hooks (#542) Update pre-commit hook versions Co-authored-by: adeebshihadeh --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e91d3ae9f..41735d278 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,17 +1,17 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-ast - id: check-yaml - id: check-executables-have-shebangs - id: check-shebang-scripts-are-executable - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.5.1 + rev: v1.6.0 hooks: - id: mypy - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.292 + rev: v0.1.0 hooks: - id: ruff - repo: local From c83ed644404d97110d58bd970f11af09ffc00a4c Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Tue, 24 Oct 2023 14:37:55 -0700 Subject: [PATCH 22/54] pre-commit: autoupdate hooks (#547) Update pre-commit hook versions Co-authored-by: adeebshihadeh --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 41735d278..ef08cd040 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,11 +7,11 @@ repos: - id: check-executables-have-shebangs - id: check-shebang-scripts-are-executable - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.6.0 + rev: v1.6.1 hooks: - id: mypy - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.0 + rev: v0.1.1 hooks: - id: ruff - repo: local From 1b43e697e5b02ff3248bad75ccc7076c9b29c50d Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Tue, 31 Oct 2023 11:34:20 -0700 Subject: [PATCH 23/54] pre-commit: autoupdate hooks (#549) Update pre-commit hook versions Co-authored-by: adeebshihadeh --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ef08cd040..ff4024930 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: hooks: - id: mypy - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.1 + rev: v0.1.3 hooks: - id: ruff - repo: local From 416c3d531c90ce16498d782bf383625a857ee74c Mon Sep 17 00:00:00 2001 From: YassineYousfi Date: Thu, 2 Nov 2023 13:36:25 -0700 Subject: [PATCH 24/54] NLP driving - new model outputs (#550) add lat planner solution to modelv2 struct --- log.capnp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/log.capnp b/log.capnp index 9777a3c32..12e25b14f 100644 --- a/log.capnp +++ b/log.capnp @@ -874,6 +874,8 @@ struct ModelDataV2 { navEnabled @22 :Bool; locationMonoTime @24 :UInt64; + # e2e lateral planner + lateralPlannerSolution @25: LateralPlannerSolution; struct LeadDataV2 { prob @0 :Float32; # probability that car is your lead at time t @@ -940,6 +942,18 @@ struct ModelDataV2 { transStd @2 :List(Float32); # std m/s in device frame rotStd @3 :List(Float32); # std rad/s in device frame } + + struct LateralPlannerSolution { + x @0 :List(Float32); + y @1 :List(Float32); + yaw @2 :List(Float32); + yawRate @3 :List(Float32); + xStd @4 :List(Float32); + yStd @5 :List(Float32); + yawStd @6 :List(Float32); + yawRateStd @7 :List(Float32); + } + } struct EncodeIndex { From 3acaeee6ded8e4fca7e3ff3e7464f62b7a553105 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Tue, 7 Nov 2023 10:58:38 -0800 Subject: [PATCH 25/54] pre-commit: autoupdate hooks (#551) Update pre-commit hook versions Co-authored-by: adeebshihadeh --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ff4024930..057e80e88 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: hooks: - id: mypy - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.3 + rev: v0.1.4 hooks: - id: ruff - repo: local From 5b8413efb11590bfba90e77dd41186304db8fbe7 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 18 Nov 2023 22:54:03 -0800 Subject: [PATCH 26/54] pre-commit: autoupdate hooks (#553) Update pre-commit hook versions Co-authored-by: adeebshihadeh --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 057e80e88..232f81ad7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,11 +7,11 @@ repos: - id: check-executables-have-shebangs - id: check-shebang-scripts-are-executable - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.6.1 + rev: v1.7.0 hooks: - id: mypy - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.4 + rev: v0.1.5 hooks: - id: ruff - repo: local From 2cb2bfb0154874775e56715ecf81f0e6b00538e9 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Tue, 21 Nov 2023 15:10:09 -0800 Subject: [PATCH 27/54] pre-commit: autoupdate hooks (#554) Update pre-commit hook versions Co-authored-by: adeebshihadeh --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 232f81ad7..b1d2dfa17 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: hooks: - id: mypy - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.5 + rev: v0.1.6 hooks: - id: ruff - repo: local From 204a1177633394dec12a83153ba13a20f59025cf Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Tue, 28 Nov 2023 14:21:42 -0800 Subject: [PATCH 28/54] pre-commit: autoupdate hooks (#555) Update pre-commit hook versions Co-authored-by: adeebshihadeh --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b1d2dfa17..ea3ee3008 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: - id: check-executables-have-shebangs - id: check-shebang-scripts-are-executable - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.7.0 + rev: v1.7.1 hooks: - id: mypy - repo: https://github.com/astral-sh/ruff-pre-commit From 3ba99bbbea6c7c96a3c8af840a543dcd4434e47b Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 1 Dec 2023 21:35:53 -0800 Subject: [PATCH 29/54] carEvents -> onroadEvents (#556) --- log.capnp | 2 +- services.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/log.capnp b/log.capnp index 12e25b14f..419182b44 100644 --- a/log.capnp +++ b/log.capnp @@ -2226,7 +2226,7 @@ struct Event { liveTorqueParameters @94 :LiveTorqueParametersData; cameraOdometry @63 :CameraOdometry; thumbnail @66: Thumbnail; - carEvents @68: List(Car.CarEvent); + events @68: List(Car.CarEvent); carParams @69: Car.CarParams; driverMonitoringState @71: DriverMonitoringState; liveLocationKalman @72 :LiveLocationKalman; diff --git a/services.py b/services.py index 9c08e6352..6436e3181 100755 --- a/services.py +++ b/services.py @@ -60,7 +60,7 @@ def __init__(self, port: int, should_log: bool, frequency: float, decimation: Op "cameraOdometry": (True, 20., 5), "lateralPlan": (True, 20., 5), "thumbnail": (True, 0.2, 1), - "carEvents": (True, 1., 1), + "onroadEvents": (True, 1., 1), "carParams": (True, 0.02, 1), "roadCameraState": (True, 20., 20), "driverCameraState": (True, 20., 20), From 65b34b4d152bcea6d49b821bf1da7189f6086b0f Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 1 Dec 2023 22:35:50 -0800 Subject: [PATCH 30/54] onroadEvents --- log.capnp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/log.capnp b/log.capnp index 419182b44..11d7fc72f 100644 --- a/log.capnp +++ b/log.capnp @@ -2226,7 +2226,7 @@ struct Event { liveTorqueParameters @94 :LiveTorqueParametersData; cameraOdometry @63 :CameraOdometry; thumbnail @66: Thumbnail; - events @68: List(Car.CarEvent); + onroadEvents @68: List(Car.CarEvent); carParams @69: Car.CarParams; driverMonitoringState @71: DriverMonitoringState; liveLocationKalman @72 :LiveLocationKalman; From 60d1ee0490a33830168b30a1e92f8d4e43cfbb15 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 1 Dec 2023 22:36:37 -0800 Subject: [PATCH 31/54] SubMaster: ensure msg is always a reader (#557) * SubMaster: ensure msg is always a reader * fix --- messaging/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/messaging/__init__.py b/messaging/__init__.py index 57675212f..b09db307d 100644 --- a/messaging/__init__.py +++ b/messaging/__init__.py @@ -187,7 +187,7 @@ def __init__(self, services: List[str], poll: Optional[List[str]] = None, except capnp.lib.capnp.KjException: # pylint: disable=c-extension-no-member data = new_message(s, 0) # lists - self.data[s] = getattr(data, s) + self.data[s] = getattr(data.as_reader(), s) self.logMonoTime[s] = 0 self.valid[s] = data.valid From 3038eacad1e0c5f727f7bd903a91167e460ae8fb Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 2 Dec 2023 10:18:11 -0800 Subject: [PATCH 32/54] new message defaults to invalid (#558) --- messaging/__init__.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/messaging/__init__.py b/messaging/__init__.py index b09db307d..cc149047d 100644 --- a/messaging/__init__.py +++ b/messaging/__init__.py @@ -41,15 +41,13 @@ def log_from_bytes(dat: bytes) -> capnp.lib.capnp._DynamicStructReader: return msg -def new_message(service: Optional[str] = None, size: Optional[int] = None) -> capnp.lib.capnp._DynamicStructBuilder: +def new_message(service: str, size: Optional[int] = None) -> capnp.lib.capnp._DynamicStructBuilder: dat = log.Event.new_message() dat.logMonoTime = int(time.monotonic() * 1e9) - dat.valid = True - if service is not None: - if size is None: - dat.init(service) - else: - dat.init(service, size) + if size is None: + dat.init(service) + else: + dat.init(service, size) return dat From 8cbd294489e07fc1dc8ee9e7abc46861718843cb Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 2 Dec 2023 10:52:02 -0800 Subject: [PATCH 33/54] explicitly set to invalid --- messaging/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/messaging/__init__.py b/messaging/__init__.py index cc149047d..139f9d3c1 100644 --- a/messaging/__init__.py +++ b/messaging/__init__.py @@ -43,6 +43,7 @@ def log_from_bytes(dat: bytes) -> capnp.lib.capnp._DynamicStructReader: def new_message(service: str, size: Optional[int] = None) -> capnp.lib.capnp._DynamicStructBuilder: dat = log.Event.new_message() + dat.valid = False dat.logMonoTime = int(time.monotonic() * 1e9) if size is None: dat.init(service) From c1c0ea2c7a793ed12f3e6b512547616e7b2a390d Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 2 Dec 2023 11:03:07 -0800 Subject: [PATCH 34/54] update the test --- messaging/tests/test_messaging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/messaging/tests/test_messaging.py b/messaging/tests/test_messaging.py index f9c85e0dc..ee1344de7 100755 --- a/messaging/tests/test_messaging.py +++ b/messaging/tests/test_messaging.py @@ -123,7 +123,7 @@ def test_new_message(self, evt): except capnp.lib.capnp.KjException: msg = messaging.new_message(evt, random.randrange(200)) self.assertLess(time.monotonic() - msg.logMonoTime, 0.1) - self.assertTrue(msg.valid) + self.assertFalse(msg.valid) self.assertEqual(evt, msg.which()) @parameterized.expand(events) From 8fae7b4de9acf985ed7572f55f72d97b9decf430 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 2 Dec 2023 22:15:48 -0800 Subject: [PATCH 35/54] SubMaster: add ignore_valid arg --- messaging/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/messaging/__init__.py b/messaging/__init__.py index 139f9d3c1..1348bf0b9 100644 --- a/messaging/__init__.py +++ b/messaging/__init__.py @@ -153,7 +153,7 @@ def recv_one_retry(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader: class SubMaster: def __init__(self, services: List[str], poll: Optional[List[str]] = None, ignore_alive: Optional[List[str]] = None, ignore_avg_freq: Optional[List[str]] = None, - addr: str = "127.0.0.1"): + ignore_valid: Optional[List[str]] = None, addr: str = "127.0.0.1"): self.frame = -1 self.updated = {s: False for s in services} self.rcv_time = {s: 0. for s in services} @@ -173,6 +173,7 @@ def __init__(self, services: List[str], poll: Optional[List[str]] = None, self.ignore_average_freq = [] if ignore_avg_freq is None else ignore_avg_freq self.ignore_alive = [] if ignore_alive is None else ignore_alive + self.ignore_valid = [] if ignore_valid is None else ignore_valid self.simulation = bool(int(os.getenv("SIMULATION", "0"))) for s in services: @@ -265,7 +266,7 @@ def all_freq_ok(self, service_list=None) -> bool: def all_valid(self, service_list=None) -> bool: if service_list is None: # check all service_list = self.valid.keys() - return all(self.valid[s] for s in service_list) + return all(self.valid[s] for s in service_list if s not in self.ignore_valid) def all_checks(self, service_list=None) -> bool: if service_list is None: # check all From 51fd6728f1091d6a8c9753a85fad761d6c42c5c6 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 2 Dec 2023 22:35:51 -0800 Subject: [PATCH 36/54] not ready for this one yet --- messaging/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/messaging/__init__.py b/messaging/__init__.py index 1348bf0b9..041fa8878 100644 --- a/messaging/__init__.py +++ b/messaging/__init__.py @@ -189,7 +189,8 @@ def __init__(self, services: List[str], poll: Optional[List[str]] = None, self.data[s] = getattr(data.as_reader(), s) self.logMonoTime[s] = 0 - self.valid[s] = data.valid + # TODO: this should default to False + self.valid[s] = True def __getitem__(self, s: str) -> capnp.lib.capnp._DynamicStructReader: return self.data[s] From 033dad2b370dac411479de19381f6e0253e51e01 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 2 Dec 2023 22:45:54 -0800 Subject: [PATCH 37/54] add kwargs for new_message --- messaging/__init__.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/messaging/__init__.py b/messaging/__init__.py index 041fa8878..6a4740366 100644 --- a/messaging/__init__.py +++ b/messaging/__init__.py @@ -41,14 +41,18 @@ def log_from_bytes(dat: bytes) -> capnp.lib.capnp._DynamicStructReader: return msg -def new_message(service: str, size: Optional[int] = None) -> capnp.lib.capnp._DynamicStructBuilder: - dat = log.Event.new_message() - dat.valid = False - dat.logMonoTime = int(time.monotonic() * 1e9) - if size is None: - dat.init(service) - else: - dat.init(service, size) +def new_message(service: Optional[str], size: Optional[int] = None, **kwargs) -> capnp.lib.capnp._DynamicStructBuilder: + args = { + 'valid': False, + 'logMonoTime': int(time.monotonic() * 1e9), + **kwargs + } + dat = log.Event.new_message(**args) + if service is not None: + if size is None: + dat.init(service) + else: + dat.init(service, size) return dat From 4fb8352484c4ef074bee775975c0d1d1e8f57a78 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sun, 3 Dec 2023 14:05:10 -0800 Subject: [PATCH 38/54] add carParams.passive --- car.capnp | 1 + 1 file changed, 1 insertion(+) diff --git a/car.capnp b/car.capnp index 29949a148..e6476fb8e 100644 --- a/car.capnp +++ b/car.capnp @@ -484,6 +484,7 @@ struct CarParams { openpilotLongitudinalControl @37 :Bool; # is openpilot doing the longitudinal control? carVin @38 :Text; # VIN number queried during fingerprinting dashcamOnly @41: Bool; + passive @73: Bool; # is openpilot in control? transmissionType @43 :TransmissionType; carFw @44 :List(CarFw); From a3a6e4969e58875f7cdfc9e6cc6b1af3ee2392b5 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 8 Dec 2023 16:33:27 -0800 Subject: [PATCH 39/54] add os04c10 --- log.capnp | 1 + 1 file changed, 1 insertion(+) diff --git a/log.capnp b/log.capnp index 11d7fc72f..b88edae95 100644 --- a/log.capnp +++ b/log.capnp @@ -166,6 +166,7 @@ struct FrameData { unknown @0; ar0231 @1; ox03c10 @2; + os04c10 @3; } frameLengthDEPRECATED @3 :Int32; From d11688a90a1cebacb3fe00dcfbba5f27551b2d89 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 9 Dec 2023 12:52:05 -0800 Subject: [PATCH 40/54] pre-commit: autoupdate hooks (#559) Update pre-commit hook versions Co-authored-by: adeebshihadeh --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ea3ee3008..68691b298 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: hooks: - id: mypy - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.6 + rev: v0.1.7 hooks: - id: ruff - repo: local From 8ae5c51a6515c171f3a82c9511a18d79a9e55c07 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 14 Dec 2023 16:37:40 -0800 Subject: [PATCH 41/54] deprecate gas interceptor detected (#563) --- log.capnp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/log.capnp b/log.capnp index b88edae95..d5f147706 100644 --- a/log.capnp +++ b/log.capnp @@ -409,7 +409,6 @@ struct DeviceState @0xa4d8b5af2aa492eb { struct PandaState @0xa7649e2575e4591e { ignitionLine @2 :Bool; - gasInterceptorDetected @4 :Bool; rxBufferOverflow @7 :UInt32; txBufferOverflow @8 :UInt32; gmlanSendErrs @9 :UInt32; @@ -542,6 +541,7 @@ struct PandaState @0xa7649e2575e4591e { } } + gasInterceptorDetectedDEPRECATED @4 :Bool; startedSignalDetectedDEPRECATED @5 :Bool; hasGpsDEPRECATED @6 :Bool; fanSpeedRpmDEPRECATED @11 :UInt16; From 9bce87fac76e25514667e22ea419b698cef47531 Mon Sep 17 00:00:00 2001 From: Dean Lee Date: Tue, 19 Dec 2023 14:27:12 +0800 Subject: [PATCH 42/54] visionipc: new helper function `get_ipc_path` (#564) new helper function ipc_get_path --- visionipc/visionipc_client.cc | 13 ++++--------- visionipc/visionipc_client.h | 3 --- visionipc/visionipc_server.cc | 20 +++++++++++--------- visionipc/visionipc_server.h | 2 +- 4 files changed, 16 insertions(+), 22 deletions(-) diff --git a/visionipc/visionipc_client.cc b/visionipc/visionipc_client.cc index 169afbb36..e3c6d0d1c 100644 --- a/visionipc/visionipc_client.cc +++ b/visionipc/visionipc_client.cc @@ -3,24 +3,19 @@ #include #include +#include #include "cereal/visionipc/ipc.h" #include "cereal/visionipc/visionipc_client.h" #include "cereal/visionipc/visionipc_server.h" #include "cereal/logger/logger.h" static int connect_to_vipc_server(const std::string &name, bool blocking) { - char* prefix = std::getenv("OPENPILOT_PREFIX"); - std::string path = "/tmp/"; - if (prefix) { - path = path + std::string(prefix) + "_"; - } - path = path + "visionipc_" + name; - - int socket_fd = ipc_connect(path.c_str()); + const std::string ipc_path = get_ipc_path(name); + int socket_fd = ipc_connect(ipc_path.c_str()); while (socket_fd < 0 && blocking) { std::cout << "VisionIpcClient connecting" << std::endl; std::this_thread::sleep_for(std::chrono::milliseconds(100)); - socket_fd = ipc_connect(path.c_str()); + socket_fd = ipc_connect(ipc_path.c_str()); } return socket_fd; } diff --git a/visionipc/visionipc_client.h b/visionipc/visionipc_client.h index d67734d32..970bac3a7 100644 --- a/visionipc/visionipc_client.h +++ b/visionipc/visionipc_client.h @@ -2,11 +2,8 @@ #include #include -#include -#include #include "cereal/messaging/messaging.h" -#include "cereal/visionipc/visionipc.h" #include "cereal/visionipc/visionbuf.h" class VisionIpcClient { diff --git a/visionipc/visionipc_server.cc b/visionipc/visionipc_server.cc index c51e13ea1..da9d11f91 100644 --- a/visionipc/visionipc_server.cc +++ b/visionipc/visionipc_server.cc @@ -22,6 +22,14 @@ std::string get_endpoint_name(std::string name, VisionStreamType type){ } } +std::string get_ipc_path(const std::string& name) { + std::string path = "/tmp/"; + if (char* prefix = std::getenv("OPENPILOT_PREFIX")) { + path += std::string(prefix) + "_"; + } + return path + "visionipc_" + name; +} + VisionIpcServer::VisionIpcServer(std::string name, cl_device_id device_id, cl_context ctx) : name(name), device_id(device_id), ctx(ctx) { msg_ctx = Context::create(); @@ -83,14 +91,8 @@ void VisionIpcServer::start_listener(){ void VisionIpcServer::listener(){ std::cout << "Starting listener for: " << name << std::endl; - char* prefix = std::getenv("OPENPILOT_PREFIX"); - std::string path = "/tmp/"; - if (prefix) { - path = path + std::string(prefix) + "_"; - } - path = path + "visionipc_" + name; - - int sock = ipc_bind(path.c_str()); + const std::string ipc_path = get_ipc_path(name); + int sock = ipc_bind(ipc_path.c_str()); assert(sock >= 0); while (!should_exit){ @@ -160,7 +162,7 @@ void VisionIpcServer::listener(){ std::cout << "Stopping listener for: " << name << std::endl; close(sock); - unlink(path.c_str()); + unlink(ipc_path.c_str()); } diff --git a/visionipc/visionipc_server.h b/visionipc/visionipc_server.h index 6ad86f146..c494b1fcf 100644 --- a/visionipc/visionipc_server.h +++ b/visionipc/visionipc_server.h @@ -6,10 +6,10 @@ #include #include "cereal/messaging/messaging.h" -#include "cereal/visionipc/visionipc.h" #include "cereal/visionipc/visionbuf.h" std::string get_endpoint_name(std::string name, VisionStreamType type); +std::string get_ipc_path(const std::string &name); class VisionIpcServer { private: From bceb8b942d3e622c2476e197102950efc4fe0bfd Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Thu, 21 Dec 2023 21:46:37 -0800 Subject: [PATCH 43/54] pre-commit: autoupdate hooks (#565) Update pre-commit hook versions Co-authored-by: adeebshihadeh --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 68691b298..8a1774120 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: hooks: - id: mypy - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.7 + rev: v0.1.8 hooks: - id: ruff - repo: local From e29625c30bb2a4a34ce21134d0e5a91b2d4fa1b1 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Tue, 26 Dec 2023 20:30:26 -0800 Subject: [PATCH 44/54] pre-commit: autoupdate hooks (#566) Update pre-commit hook versions Co-authored-by: adeebshihadeh --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8a1774120..2ff884418 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,11 +7,11 @@ repos: - id: check-executables-have-shebangs - id: check-shebang-scripts-are-executable - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.7.1 + rev: v1.8.0 hooks: - id: mypy - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.8 + rev: v0.1.9 hooks: - id: ruff - repo: local From ea3bb8d0105360800fd61df303ef23819539c668 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sun, 7 Jan 2024 21:57:16 -0800 Subject: [PATCH 45/54] add camera request id (#567) --- log.capnp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/log.capnp b/log.capnp index d5f147706..55beda223 100644 --- a/log.capnp +++ b/log.capnp @@ -130,8 +130,8 @@ struct InitData { struct FrameData { frameId @0 :UInt32; - encodeId @1 :UInt32; # DEPRECATED frameIdSensor @25 :UInt32; + requestId @28 :UInt32; frameType @7 :FrameType; @@ -169,6 +169,7 @@ struct FrameData { os04c10 @3; } + encodeIdDEPRECATED @1 :UInt32; frameLengthDEPRECATED @3 :Int32; globalGainDEPRECATED @5 :Int32; androidCaptureResultDEPRECATED @9 :AndroidCaptureResult; From d81d86e7cd83d1eb40314964a4d194231381d557 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sun, 7 Jan 2024 22:12:48 -0800 Subject: [PATCH 46/54] comment was a lie --- log.capnp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/log.capnp b/log.capnp index 55beda223..e601e8203 100644 --- a/log.capnp +++ b/log.capnp @@ -132,6 +132,7 @@ struct FrameData { frameId @0 :UInt32; frameIdSensor @25 :UInt32; requestId @28 :UInt32; + encodeId @1 :UInt32; frameType @7 :FrameType; @@ -169,7 +170,6 @@ struct FrameData { os04c10 @3; } - encodeIdDEPRECATED @1 :UInt32; frameLengthDEPRECATED @3 :Int32; globalGainDEPRECATED @5 :Int32; androidCaptureResultDEPRECATED @9 :AndroidCaptureResult; From 20b65eeb1f6c580cdd7d63e53639f4fc48bc2f56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Harald=20Sch=C3=A4fer?= Date: Wed, 17 Jan 2024 15:25:24 -0800 Subject: [PATCH 47/54] Deprecate curv rate (#571) --- log.capnp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/log.capnp b/log.capnp index e601e8203..1f27e0698 100644 --- a/log.capnp +++ b/log.capnp @@ -668,7 +668,6 @@ struct ControlsState @0x97ff69c53601abf1 { aTarget @35 :Float32; curvature @37 :Float32; # path curvature from vehicle model desiredCurvature @61 :Float32; # lag adjusted curvatures used by lateral controllers - desiredCurvatureRate @62 :Float32; forceDecel @51 :Bool; # UI alerts @@ -826,6 +825,7 @@ struct ControlsState @0x97ff69c53601abf1 { steerOverrideDEPRECATED @20 :Bool; steeringAngleDesiredDegDEPRECATED @29 :Float32; canMonoTimesDEPRECATED @21 :List(UInt64); + desiredCurvatureRateDEPRECATED @62 :Float32; } # All SI units and in device frame From fd1843960f4ba55bb2857593857d05ca463ac377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Harald=20Sch=C3=A4fer?= Date: Sun, 21 Jan 2024 12:10:24 -0800 Subject: [PATCH 48/54] Delete lateralPlan (#573) * Add action * Add lane change state * planner error dead * Deperacte mote * Fix bugs * Fails the replay test * Typo --- car.capnp | 2 +- log.capnp | 34 ++++++++++++++++++++++++++++++++-- services.py | 1 - 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/car.capnp b/car.capnp index e6476fb8e..af91986d4 100644 --- a/car.capnp +++ b/car.capnp @@ -51,7 +51,6 @@ struct CarEvent @0x9b1657f34caf3ad3 { parkBrake @29; manualRestart @30; lowSpeedLockout @31; - plannerError @32; joystickDebug @34; steerTempUnavailableSilent @35; resumeRequired @36; @@ -142,6 +141,7 @@ struct CarEvent @0x9b1657f34caf3ad3 { startupFuzzyFingerprintDEPRECATED @97; noTargetDEPRECATED @25; brakeUnavailableDEPRECATED @2; + plannerErrorDEPRECATED @32; } } diff --git a/log.capnp b/log.capnp index 1f27e0698..1be5c878b 100644 --- a/log.capnp +++ b/log.capnp @@ -299,6 +299,29 @@ struct GpsLocationData { } } +enum Desire { + none @0; + turnLeft @1; + turnRight @2; + laneChangeLeft @3; + laneChangeRight @4; + keepLeft @5; + keepRight @6; +} + +enum LaneChangeState { + off @0; + preLaneChange @1; + laneChangeStarting @2; + laneChangeFinishing @3; +} + +enum LaneChangeDirection { + none @0; + left @1; + right @2; +} + struct CanData { address @0 :UInt32; busTime @1 :UInt16; @@ -877,7 +900,8 @@ struct ModelDataV2 { locationMonoTime @24 :UInt64; # e2e lateral planner - lateralPlannerSolution @25: LateralPlannerSolution; + lateralPlannerSolutionDEPRECATED @25: LateralPlannerSolution; + action @26: Action; struct LeadDataV2 { prob @0 :Float32; # probability that car is your lead at time t @@ -915,6 +939,9 @@ struct ModelDataV2 { desireState @5 :List(Float32); disengagePredictions @6 :DisengagePredictions; hardBrakePredicted @7 :Bool; + laneChangeState @8 :LaneChangeState; + laneChangeDirection @9 :LaneChangeDirection; + # deprecated brakeDisengageProbDEPRECATED @2 :Float32; @@ -956,6 +983,9 @@ struct ModelDataV2 { yawRateStd @7 :List(Float32); } + struct Action { + desiredCurvature @0 :Float32; + } } struct EncodeIndex { @@ -2216,7 +2246,6 @@ struct Event { carState @22 :Car.CarState; carControl @23 :Car.CarControl; longitudinalPlan @24 :LongitudinalPlan; - lateralPlan @64 :LateralPlan; uiPlan @106 :UiPlan; ubloxGnss @34 :UbloxGnss; ubloxRaw @39 :Data; @@ -2337,5 +2366,6 @@ struct Event { pandaStateDEPRECATED @12 :PandaState; driverStateDEPRECATED @59 :DriverStateDEPRECATED; sensorEventsDEPRECATED @11 :List(SensorEventData); + lateralPlan @64 :LateralPlan; } } diff --git a/services.py b/services.py index 6436e3181..8ab8f5eeb 100755 --- a/services.py +++ b/services.py @@ -58,7 +58,6 @@ def __init__(self, port: int, should_log: bool, frequency: float, decimation: Op "liveLocationKalman": (True, 20., 5), "liveParameters": (True, 20., 5), "cameraOdometry": (True, 20., 5), - "lateralPlan": (True, 20., 5), "thumbnail": (True, 0.2, 1), "onroadEvents": (True, 1., 1), "carParams": (True, 0.02, 1), From 775e7f84da5eb283d604e94f9bf4582ffa741c22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Harald=20Sch=C3=A4fer?= Date: Sun, 21 Jan 2024 12:29:45 -0800 Subject: [PATCH 49/54] Deprecate lat planner (#574) --- log.capnp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/log.capnp b/log.capnp index 1be5c878b..ea5da7496 100644 --- a/log.capnp +++ b/log.capnp @@ -2366,6 +2366,6 @@ struct Event { pandaStateDEPRECATED @12 :PandaState; driverStateDEPRECATED @59 :DriverStateDEPRECATED; sensorEventsDEPRECATED @11 :List(SensorEventData); - lateralPlan @64 :LateralPlan; + lateralPlanDEPRECATED @64 :LateralPlan; } } From 8ae59093eabdcd662621bd1c862a0dd19f0b9adb Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Tue, 23 Jan 2024 19:01:44 -0800 Subject: [PATCH 50/54] add quatro --- log.capnp | 1 + 1 file changed, 1 insertion(+) diff --git a/log.capnp b/log.capnp index ea5da7496..9a2014193 100644 --- a/log.capnp +++ b/log.capnp @@ -518,6 +518,7 @@ struct PandaState @0xa7649e2575e4591e { redPanda @7; redPandaV2 @8; tres @9; + quatro @9; } enum HarnessStatus { From 16edae3d50d6d7003acf6079a844085eebd21bd5 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Tue, 23 Jan 2024 19:09:00 -0800 Subject: [PATCH 51/54] bump ordinal --- log.capnp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/log.capnp b/log.capnp index 9a2014193..84605477a 100644 --- a/log.capnp +++ b/log.capnp @@ -518,7 +518,7 @@ struct PandaState @0xa7649e2575e4591e { redPanda @7; redPandaV2 @8; tres @9; - quatro @9; + quatro @10; } enum HarnessStatus { From a6ade85c9dd6652fde547b9e089a297f67606dcf Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Wed, 24 Jan 2024 13:06:40 -0800 Subject: [PATCH 52/54] spelling --- log.capnp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/log.capnp b/log.capnp index 84605477a..9a1835e42 100644 --- a/log.capnp +++ b/log.capnp @@ -518,7 +518,7 @@ struct PandaState @0xa7649e2575e4591e { redPanda @7; redPandaV2 @8; tres @9; - quatro @10; + cuatro @10; } enum HarnessStatus { From c54369f8ad4e0bcb18c96feb4334755c6f65e8f1 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Thu, 1 Feb 2024 21:24:05 -0800 Subject: [PATCH 53/54] add frame width + height to encodeData (#578) * add frame width + height to encodeData * fix * lol --- log.capnp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/log.capnp b/log.capnp index 9a1835e42..276cb85e5 100644 --- a/log.capnp +++ b/log.capnp @@ -2200,6 +2200,8 @@ struct EncodeData { data @1 :Data; header @2 :Data; unixTimestampNanos @3 :UInt64; + width @4 :UInt32; + height @5 :UInt32; } struct UserFlag { From 80e1e55f0dd71cea7f596e8b80c7c33865b689f3 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sat, 3 Feb 2024 01:07:53 -0600 Subject: [PATCH 54/54] deprecate curvature steer control type (#579) deprecate curvature --- car.capnp | 3 ++- log.capnp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/car.capnp b/car.capnp index af91986d4..2560a6722 100644 --- a/car.capnp +++ b/car.capnp @@ -601,7 +601,8 @@ struct CarParams { enum SteerControlType { torque @0; angle @1; - curvature @2; + + curvatureDEPRECATED @2; } enum TransmissionType { diff --git a/log.capnp b/log.capnp index 276cb85e5..9ad6b6247 100644 --- a/log.capnp +++ b/log.capnp @@ -713,8 +713,8 @@ struct ControlsState @0x97ff69c53601abf1 { angleState @58 :LateralAngleState; debugState @59 :LateralDebugState; torqueState @60 :LateralTorqueState; - curvatureState @65 :LateralCurvatureState; + curvatureStateDEPRECATED @65 :LateralCurvatureState; lqrStateDEPRECATED @55 :LateralLQRState; }