From 67d104cb16b636e057aeb965b4a12024c41f7490 Mon Sep 17 00:00:00 2001 From: DevTekVE Date: Fri, 13 Oct 2023 18:29:36 +0200 Subject: [PATCH] Merge with upstream (#3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * pre-commit: autoupdate hooks (#532) Co-authored-by: adeebshihadeh * Update README.md (#533) * pre-commit: autoupdate hooks (#535) Update pre-commit hook versions Co-authored-by: adeebshihadeh * services: capitalize constant service list (#534) * rename services * keep more unique name * type this * Bump pandaStates to 10hz (#536) * 10 Hz * constant * fix * revert * fix peripheralState freq * revert peripheral * Bringing sunnypilot * Refactor service list and deprecate certain fields Refactored service list naming convention in messaging/init.py and services.py for consistency, changing `service_list` to `SERVICE_LIST`. Additionally, fields as `laneWidth`, `lProb`, `rProb`, `dProb` in `LateralPlan` struct and `brakeLights` in `CarEvent.EventName` have been marked as DEPRECATED as part of an ongoing effort for codebase clean-up. Changes also include updating the frequency of `pandaStates` service and commenting out a debug statement in msgq.cc, enhancing overall system performance. * pre-commit: autoupdate hooks (#538) Update pre-commit hook versions Co-authored-by: adeebshihadeh * Added reserved fields for body controls (#537) * Added bodyReserved0-2 text fields * Added bodyReserved0 to services list * add more bodyReserved to servcies py * bodyReserved -> customReservedText * :Text -> :Data --------- Co-authored-by: Kacper Rączy * zmq: assert the same PID (#540) Enforce the same PID in ZMQ pub sockets * pre-commit: autoupdate hooks (#541) Update pre-commit hook versions Co-authored-by: adeebshihadeh * Bringing sunnypilot * Refactor service list and deprecate certain fields Refactored service list naming convention in messaging/init.py and services.py for consistency, changing `service_list` to `SERVICE_LIST`. Additionally, fields as `laneWidth`, `lProb`, `rProb`, `dProb` in `LateralPlan` struct and `brakeLights` in `CarEvent.EventName` have been marked as DEPRECATED as part of an ongoing effort for codebase clean-up. Changes also include updating the frequency of `pandaStates` service and commenting out a debug statement in msgq.cc, enhancing overall system performance. * Bringing sunnypilot's cereal changes aligned * Converging cereal changes * accidentally duplicated some literals * Update log.capnp Co-authored-by: Jason Wen --------- Co-authored-by: Adeeb Shihadeh Co-authored-by: adeebshihadeh Co-authored-by: JJ <103335846+computerscienceiscool@users.noreply.github.com> Co-authored-by: Shane Smiskol Co-authored-by: Miguel Fernandez Co-authored-by: Mitchell Goff Co-authored-by: Kacper Rączy Co-authored-by: Jason Wen --- .idea/.gitignore | 8 ++++++ .pre-commit-config.yaml | 41 ++++++++++++++++++++++++++++++- README.md | 2 +- car.capnp | 2 +- log.capnp | 4 +++ messaging/__init__.py | 4 +-- messaging/impl_zmq.cc | 10 ++++++-- messaging/impl_zmq.h | 1 + messaging/tests/test_messaging.py | 4 +-- messaging/tests/test_services.py | 8 +++--- services.py | 11 ++++++--- 11 files changed, 78 insertions(+), 17 deletions(-) create mode 100644 .idea/.gitignore diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 000000000..13566b81b --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 306a7140c..e91d3ae9f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,45 @@ repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-ast + - id: check-yaml + - id: check-executables-have-shebangs + - id: check-shebang-scripts-are-executable +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.5.1 + hooks: + - id: mypy +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.0.292 + hooks: + - id: ruff +- repo: local + hooks: + - id: cppcheck + name: cppcheck + entry: cppcheck + language: system + types: [c++] + exclude: '^(messaging/msgq_tests.cc|messaging/test_runner.cc)' + args: + - --error-exitcode=1 + - --inline-suppr + - --language=c++ + - --force + - --quiet + - -j4 +- repo: https://github.com/cpplint/cpplint + rev: 1.6.1 + hooks: + - id: cpplint + args: + - --quiet + - --counting=detailed + - --linelength=240 + - --filter=-build,-legal,-readability,-runtime,-whitespace,+build/include_subdir,+build/forward_decl,+build/include_what_you_use,+build/deprecated,+whitespace/comma,+whitespace/line_length,+whitespace/empty_if_body,+whitespace/empty_loop_body,+whitespace/empty_conditional_body,+whitespace/forcolon,+whitespace/parens,+whitespace/semicolon,+whitespace/tab,+readability/braces - repo: https://github.com/codespell-project/codespell - rev: v2.2.5 + rev: v2.2.6 hooks: - id: codespell args: diff --git a/README.md b/README.md index 737534e36..e3326aab0 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ All `Events` have a `logMonoTime` and a `valid`. Then a big union defines the pa ### Maintaining backwards-compatibility -When making changes to the messaging spec you want to maintain backwards-compatability, such that old logs can +When making changes to the messaging spec you want to maintain backwards-compatibility, such that old logs can be parsed with a new version of cereal. Adding structs and adding members to structs is generally safe, most other things are not. Read more details [here](https://capnproto.org/language.html). diff --git a/car.capnp b/car.capnp index cc1173837..0e4831001 100644 --- a/car.capnp +++ b/car.capnp @@ -312,7 +312,7 @@ struct CarState { # deprecated errorsDEPRECATED @0 :List(CarEvent.EventName); - brakeLights @19 :Bool; + brakeLightsDEPRECATED @19 :Bool; steeringRateLimitedDEPRECATED @29 :Bool; canMonoTimesDEPRECATED @12: List(UInt64); } diff --git a/log.capnp b/log.capnp index f63700584..68f36e3fb 100644 --- a/log.capnp +++ b/log.capnp @@ -2265,6 +2265,10 @@ struct Event { livestreamWideRoadEncodeData @121 :EncodeData; livestreamDriverEncodeData @122 :EncodeData; + customReservedRawData0 @124 :Data; + customReservedRawData1 @125 :Data; + customReservedRawData2 @126 :Data; + # *********** Custom: reserved for forks *********** controlsStateSP @107 :Custom.ControlsStateSP; longitudinalPlanSP @108 :Custom.LongitudinalPlanSP; diff --git a/messaging/__init__.py b/messaging/__init__.py index fd9a48772..57675212f 100644 --- a/messaging/__init__.py +++ b/messaging/__init__.py @@ -11,7 +11,7 @@ from collections import deque from cereal import log -from cereal.services import service_list +from cereal.services import SERVICE_LIST assert MultiplePublishersError assert MessagingError @@ -180,7 +180,7 @@ def __init__(self, services: List[str], poll: Optional[List[str]] = None, if addr is not None: p = self.poller if s not in self.non_polled_services else None self.sock[s] = sub_sock(s, poller=p, addr=addr, conflate=True) - self.freq[s] = service_list[s].frequency + self.freq[s] = SERVICE_LIST[s].frequency try: data = new_message(s) diff --git a/messaging/impl_zmq.cc b/messaging/impl_zmq.cc index aa95d94fb..7da9df1b8 100644 --- a/messaging/impl_zmq.cc +++ b/messaging/impl_zmq.cc @@ -3,6 +3,7 @@ #include #include #include +#include #include "cereal/services.h" #include "cereal/messaging/impl_zmq.h" @@ -108,14 +109,19 @@ int ZMQPubSocket::connect(Context *context, std::string endpoint, bool check_end full_endpoint += endpoint; } + // ZMQ pub sockets cannot be shared between processes, so we need to ensure pid stays the same + pid = getpid(); + return zmq_bind(sock, full_endpoint.c_str()); } -int ZMQPubSocket::sendMessage(Message *message){ +int ZMQPubSocket::sendMessage(Message *message) { + assert(pid == getpid()); return zmq_send(sock, message->getData(), message->getSize(), ZMQ_DONTWAIT); } -int ZMQPubSocket::send(char *data, size_t size){ +int ZMQPubSocket::send(char *data, size_t size) { + assert(pid == getpid()); return zmq_send(sock, data, size, ZMQ_DONTWAIT); } diff --git a/messaging/impl_zmq.h b/messaging/impl_zmq.h index 34864f633..903875f63 100644 --- a/messaging/impl_zmq.h +++ b/messaging/impl_zmq.h @@ -46,6 +46,7 @@ class ZMQPubSocket : public PubSocket { private: void * sock; std::string full_endpoint; + int pid = -1; public: int connect(Context *context, std::string endpoint, bool check_endpoint=true); int sendMessage(Message *message); diff --git a/messaging/tests/test_messaging.py b/messaging/tests/test_messaging.py index 063351b75..f9c85e0dc 100755 --- a/messaging/tests/test_messaging.py +++ b/messaging/tests/test_messaging.py @@ -11,9 +11,9 @@ from cereal import log, car import cereal.messaging as messaging -from cereal.services import service_list +from cereal.services import SERVICE_LIST -events = [evt for evt in log.Event.schema.union_fields if evt in service_list.keys()] +events = [evt for evt in log.Event.schema.union_fields if evt in SERVICE_LIST.keys()] def random_sock(): return random.choice(events) diff --git a/messaging/tests/test_services.py b/messaging/tests/test_services.py index 6452b5b6e..f26bdbc5f 100755 --- a/messaging/tests/test_services.py +++ b/messaging/tests/test_services.py @@ -6,21 +6,21 @@ from parameterized import parameterized import cereal.services as services -from cereal.services import service_list, RESERVED_PORT, STARTING_PORT +from cereal.services import SERVICE_LIST, RESERVED_PORT, STARTING_PORT class TestServices(unittest.TestCase): - @parameterized.expand(service_list.keys()) + @parameterized.expand(SERVICE_LIST.keys()) def test_services(self, s): - service = service_list[s] + service = SERVICE_LIST[s] self.assertTrue(service.port != RESERVED_PORT) self.assertTrue(service.port >= STARTING_PORT) self.assertTrue(service.frequency <= 104) def test_no_duplicate_port(self): ports: Dict[int, str] = {} - for name, service in service_list.items(): + for name, service in SERVICE_LIST.items(): self.assertFalse(service.port in ports.keys(), f"duplicate port {service.port}") ports[service.port] = name diff --git a/services.py b/services.py index 8cb26d02a..8e7bcd16b 100755 --- a/services.py +++ b/services.py @@ -18,7 +18,7 @@ def __init__(self, port: int, should_log: bool, frequency: float, decimation: Op self.decimation = decimation -services = { +services: dict[str, tuple] = { # service: (should_log, frequency, qlog decimation (optional)) # note: the "EncodeIdx" packets will still be in the log "gyroscope": (True, 104., 104), @@ -33,7 +33,7 @@ def __init__(self, port: int, should_log: bool, frequency: float, decimation: Op "deviceState": (True, 2., 1), "can": (True, 100., 1223), # decimation gives ~5 msgs in a full segment "controlsState": (True, 100., 10), - "pandaStates": (True, 2., 1), + "pandaStates": (True, 10., 1), "peripheralState": (True, 2., 1), "radarState": (True, 20., 5), "roadEncodeIdx": (False, 20., 1), @@ -102,8 +102,11 @@ def __init__(self, port: int, should_log: bool, frequency: float, decimation: Op "livestreamWideRoadEncodeData": (False, 20.), "livestreamRoadEncodeData": (False, 20.), "livestreamDriverEncodeData": (False, 20.), + "customReservedRawData0": (True, 0.), + "customReservedRawData1": (True, 0.), + "customReservedRawData2": (True, 0.), } -service_list = {name: Service(new_port(idx), *vals) for # type: ignore +SERVICE_LIST = {name: Service(new_port(idx), *vals) for idx, (name, vals) in enumerate(services.items())} @@ -118,7 +121,7 @@ def build_header(): h += "struct service { std::string name; int port; bool should_log; int frequency; int decimation; };\n" h += "static std::map services = {\n" - for k, v in service_list.items(): + for k, v in SERVICE_LIST.items(): should_log = "true" if v.should_log else "false" decimation = -1 if v.decimation is None else v.decimation h += ' { "%s", {"%s", %d, %s, %d, %d}},\n' % \