Merge branch 'master-upstream'

.github/workflows/repo.yml

@@ -2,17 +2,17 @@ name: repo
 
 on:
   schedule:
-    - cron: "0 15 * * 2"
+    - cron: "0 15 1 * *"
   workflow_dispatch:
 
 jobs:
   pre-commit-autoupdate:
     name: pre-commit autoupdate
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     container:
       image: ghcr.io/commaai/opendbc:latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: pre-commit autoupdate
       run: |
         git config --global --add safe.directory '*'

.github/workflows/tests.yml

@@ -4,34 +4,31 @@ on: [push, pull_request]
 
 env:
   RUN: docker run -v $GITHUB_WORKSPACE:/project/opendbc -w /project/opendbc -e PYTHONWARNINGS="error,default::DeprecationWarning" --shm-size 1G --rm opendbc /bin/bash -c
-  BUILD: |
-      docker pull $(grep -ioP '(?<=^from)\s+\S+' Dockerfile) || true
-      docker pull ghcr.io/commaai/opendbc:latest || true
-      docker build --cache-from ghcr.io/commaai/opendbc:latest -t opendbc -f Dockerfile .
+  BUILD: docker buildx build --pull --load --cache-to type=inline --cache-from type=registry,ref=ghcr.io/commaai/opendbc:latest -t opendbc -f Dockerfile .
   PYTHONWARNINGS: error
 
 jobs:
   unit-tests:
     name: unit tests
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     #strategy:
     #  fail-fast: false
     #  matrix:
     #    run: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build Docker image
       run: eval "$BUILD"
     - name: Build opendbc
       run: ${{ env.RUN }} "cd ../ && scons -j$(nproc) --minimal"
     - name: Unit tests
-      run: ${{ env.RUN }} "python -m unittest discover ."
+      run: ${{ env.RUN }} "pytest"
 
   static-analysis:
     name: static analysis
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build Docker image
       run: eval "$BUILD"
     - name: Build opendbc
@@ -44,10 +41,10 @@ jobs:
 
   docker-push:
     name: docker push
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/opendbc'
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build Docker image
       run: eval "$BUILD"
     - name: Push to dockerhub

.pre-commit-config.yaml

@@ -17,15 +17,15 @@ repos:
         - --check-hidden
         - --builtins clear,rare,informal,usage,code,names,en-GB_to_en-US
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.7.0
+    rev: v1.9.0
     hooks:
     -   id: mypy
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.5
+    rev: v0.3.2
     hooks:
     - id: ruff
 -   repo: https://github.com/MarcoGorelli/cython-lint
-    rev: v0.15.0
+    rev: v0.16.0
     hooks:
     -   id: cython-lint
     -   id: double-quote-cython-strings
@@ -48,6 +48,7 @@ repos:
         args:
         - --error-exitcode=1
         - --language=c++
+        - --inline-suppr
         - --force
         - --quiet
         - -j4

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:20.04
+FROM ubuntu:24.04
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -26,21 +26,18 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     ocl-icd-opencl-dev \
     opencl-headers \
     tk-dev \
-    python-openssl \
+    python3-pip \
+    python3-dev \
+    python3-openssl \
+    python-is-python3 \
     xz-utils \
     zlib1g-dev \
     cmake \
   && rm -rf /var/lib/apt/lists/*
 
-RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
-ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}"
-RUN pyenv install 3.11.4
-RUN pyenv global 3.11.4
-RUN pyenv rehash
-
 COPY requirements.txt /tmp/
-RUN pip install --no-cache-dir -r /tmp/requirements.txt
-RUN pip install --no-cache-dir pre-commit==2.15.0 pylint==2.17.4
+RUN pip3 install --break-system-packages --no-cache-dir -r /tmp/requirements.txt
+RUN pip3 install --break-system-packages --no-cache-dir pre-commit==2.15.0 pylint==2.17.4
 
 ENV PYTHONPATH=/project
 
@@ -49,7 +46,7 @@ RUN git config --global --add safe.directory '*'
 WORKDIR /project
 RUN git clone https://github.com/commaai/cereal.git /project/cereal && \
     cd /project/cereal && \
-    git checkout 82bca3a9714b73c05414fdf848b6016a0ffac17d && \
+    git checkout 861144c136c91f70dcbc652c2ffe99f57440ad47 && \
     rm -rf .git && \
     scons -j$(nproc) --minimal
 

SConstruct

@@ -39,6 +39,7 @@ env = Environment(
     "-Wunused",
     "-Werror",
     "-Wshadow",
+    "-Wno-vla-cxx-extension",
   ] + ccflags_asan,
   LDFLAGS=ldflags_asan,
   LINKFLAGS=ldflags_asan,

acura_ilx_2016_can_generated.dbc

@@ -1,7 +1,7 @@
 CM_ "AUTOGENERATED FILE, DO NOT EDIT";
 
 
-CM_ "Imported file _comma.dbc starts here";
+CM_ "Imported file _community.dbc starts here";
 BO_ 512 GAS_COMMAND: 6 EON
  SG_ GAS_COMMAND : 7|16@0+ (0.253984064,-83.3) [0|1] "" INTERCEPTOR
  SG_ GAS_COMMAND2 : 23|16@0+ (0.126992032,-83.3) [0|1] "" INTERCEPTOR

acura_rdx_2018_can_generated.dbc

@@ -1,7 +1,7 @@
 CM_ "AUTOGENERATED FILE, DO NOT EDIT";
 
 
-CM_ "Imported file _comma.dbc starts here";
+CM_ "Imported file _community.dbc starts here";
 BO_ 512 GAS_COMMAND: 6 EON
  SG_ GAS_COMMAND : 7|16@0+ (0.253984064,-83.3) [0|1] "" INTERCEPTOR
  SG_ GAS_COMMAND2 : 23|16@0+ (0.126992032,-83.3) [0|1] "" INTERCEPTOR

can/SConscript

@@ -22,3 +22,7 @@ packer = lenv.Program('packer_pyx.so', 'packer_pyx.pyx')
 
 lenv.Depends(parser, libdbc)
 lenv.Depends(packer, libdbc)
+
+opendbc_python = Alias("opendbc_python", [parser, packer])
+
+Export('opendbc_python')

can/common.cc

@@ -102,11 +102,15 @@ void gen_crc_lookup_table_16(uint16_t poly, uint16_t crc_lut[]) {
   }
 }
 
-void init_crc_lookup_tables() {
-  // At init time, set up static lookup tables for fast CRC computation.
-  gen_crc_lookup_table_8(0x2F, crc8_lut_8h2f);    // CRC-8 8H2F/AUTOSAR for Volkswagen
-  gen_crc_lookup_table_16(0x1021, crc16_lut_xmodem);    // CRC-16 XMODEM for HKG CAN FD
-}
+// Initializes CRC lookup tables at module initialization
+struct CrcInitializer {
+  CrcInitializer() {
+    gen_crc_lookup_table_8(0x2F, crc8_lut_8h2f);    // CRC-8 8H2F/AUTOSAR for Volkswagen
+    gen_crc_lookup_table_16(0x1021, crc16_lut_xmodem);    // CRC-16 XMODEM for HKG CAN FD
+  }
+};
+
+static CrcInitializer crcInitializer;
 
 unsigned int volkswagen_mqb_checksum(uint32_t address, const Signal &sig, const std::vector<uint8_t> &d) {
   // Volkswagen uses standard CRC8 8H2F/AUTOSAR, but they compute it with

can/common.h

@@ -13,6 +13,7 @@
 #include "cereal/gen/cpp/log.capnp.h"
 #endif
 
+#include "opendbc/can/logger.h"
 #include "opendbc/can/common_dbc.h"
 
 #define INFO printf
@@ -23,8 +24,6 @@
 #define MAX_BAD_COUNTER 5
 #define CAN_INVALID_CNT 5
 
-void init_crc_lookup_tables();
-
 // Car specific functions
 unsigned int honda_checksum(uint32_t address, const Signal &sig, const std::vector<uint8_t> &d);
 unsigned int toyota_checksum(uint32_t address, const Signal &sig, const std::vector<uint8_t> &d);
@@ -92,11 +91,10 @@ class CANPacker {
 private:
   const DBC *dbc = NULL;
   std::map<std::pair<uint32_t, std::string>, Signal> signal_lookup;
-  std::map<uint32_t, Msg> message_lookup;
   std::map<uint32_t, uint32_t> counters;
 
 public:
   CANPacker(const std::string& dbc_name);
   std::vector<uint8_t> pack(uint32_t address, const std::vector<SignalPackValue> &values);
-  Msg* lookup_message(uint32_t address);
+  const Msg* lookup_message(uint32_t address);
 };

can/common.pxd

@@ -6,6 +6,7 @@ from libcpp cimport bool
 from libcpp.pair cimport pair
 from libcpp.string cimport string
 from libcpp.vector cimport vector
+from libcpp.unordered_map cimport unordered_map
 
 
 ctypedef unsigned int (*calc_checksum_type)(uint32_t, const Signal&, const vector[uint8_t] &)
@@ -48,6 +49,8 @@ cdef extern from "common_dbc.h":
     string name
     vector[Msg] msgs
     vector[Val] vals
+    unordered_map[uint32_t, const Msg*] addr_to_msg
+    unordered_map[string, const Msg*] name_to_msg
 
   cdef struct SignalValue:
     uint32_t address

can/common_dbc.h

@@ -1,8 +1,8 @@
 #pragma once
 
-#include <cstddef>
 #include <cstdint>
 #include <string>
+#include <unordered_map>
 #include <vector>
 
 struct SignalPackValue {
@@ -59,6 +59,8 @@ struct DBC {
   std::string name;
   std::vector<Msg> msgs;
   std::vector<Val> vals;
+  std::unordered_map<uint32_t, const Msg*> addr_to_msg;
+  std::unordered_map<std::string, const Msg*> name_to_msg;
 };
 
 typedef struct ChecksumState {

can/dbc.cc

@@ -200,6 +200,8 @@ DBC* dbc_parse_from_stream(const std::string &dbc_name, std::istream &stream, Ch
 
   for (auto& m : dbc->msgs) {
     m.sigs = signals[m.address];
+    dbc->addr_to_msg[m.address] = &m;
+    dbc->name_to_msg[m.name] = &m;
   }
   for (auto& v : dbc->vals) {
     v.sigs = signals[v.address];

can/logger.h

@@ -0,0 +1,27 @@
+#pragma once
+
+#ifdef SWAGLOG
+// cppcheck-suppress preprocessorErrorDirective
+#include SWAGLOG
+#else
+
+#define CLOUDLOG_DEBUG 10
+#define CLOUDLOG_INFO 20
+#define CLOUDLOG_WARNING 30
+#define CLOUDLOG_ERROR 40
+#define CLOUDLOG_CRITICAL 50
+
+#define cloudlog(lvl, fmt, ...) printf(fmt "\n", ## __VA_ARGS__)
+#define cloudlog_rl(burst, millis, lvl, fmt, ...) printf(fmt "\n", ##__VA_ARGS__)
+
+#define LOGD(fmt, ...) cloudlog(CLOUDLOG_DEBUG, fmt, ## __VA_ARGS__)
+#define LOG(fmt, ...) cloudlog(CLOUDLOG_INFO, fmt, ## __VA_ARGS__)
+#define LOGW(fmt, ...) cloudlog(CLOUDLOG_WARNING, fmt, ## __VA_ARGS__)
+#define LOGE(fmt, ...) cloudlog(CLOUDLOG_ERROR, fmt, ## __VA_ARGS__)
+
+#define LOGD_100(fmt, ...) cloudlog_rl(2, 100, CLOUDLOG_DEBUG, fmt, ## __VA_ARGS__)
+#define LOG_100(fmt, ...) cloudlog_rl(2, 100, CLOUDLOG_INFO, fmt, ## __VA_ARGS__)
+#define LOGW_100(fmt, ...) cloudlog_rl(2, 100, CLOUDLOG_WARNING, fmt, ## __VA_ARGS__)
+#define LOGE_100(fmt, ...) cloudlog_rl(2, 100, CLOUDLOG_ERROR, fmt, ## __VA_ARGS__)
+
+#endif

can/packer.cc

@@ -33,24 +33,28 @@ CANPacker::CANPacker(const std::string& dbc_name) {
   assert(dbc);
 
   for (const auto& msg : dbc->msgs) {
-    message_lookup[msg.address] = msg;
     for (const auto& sig : msg.sigs) {
       signal_lookup[std::make_pair(msg.address, sig.name)] = sig;
     }
   }
-  init_crc_lookup_tables();
 }
 
 std::vector<uint8_t> CANPacker::pack(uint32_t address, const std::vector<SignalPackValue> &signals) {
-  std::vector<uint8_t> ret(message_lookup[address].size, 0);
+  auto msg_it = dbc->addr_to_msg.find(address);
+  if (msg_it == dbc->addr_to_msg.end()) {
+    LOGE("undefined address %d", address);
+    return {};
+  }
+
+  std::vector<uint8_t> ret(msg_it->second->size, 0);
 
   // set all values for all given signal/value pairs
   bool counter_set = false;
   for (const auto& sigval : signals) {
     auto sig_it = signal_lookup.find(std::make_pair(address, sigval.name));
     if (sig_it == signal_lookup.end()) {
       // TODO: do something more here. invalid flag like CANParser?
-      WARN("undefined signal %s - %d\n", sigval.name.c_str(), address);
+      LOGE("undefined signal %s - %d\n", sigval.name.c_str(), address);
       continue;
     }
     const auto &sig = sig_it->second;
@@ -61,9 +65,9 @@ std::vector<uint8_t> CANPacker::pack(uint32_t address, const std::vector<SignalP
     }
     set_value(ret, sig, ival);
 
-    counter_set = counter_set || (sigval.name == "COUNTER");
-    if (counter_set) {
+    if (sigval.name == "COUNTER") {
       counters[address] = sigval.value;
+      counter_set = true;
     }
   }
 
@@ -93,6 +97,6 @@ std::vector<uint8_t> CANPacker::pack(uint32_t address, const std::vector<SignalP
 }
 
 // This function has a definition in common.h and is used in PlotJuggler
-Msg* CANPacker::lookup_message(uint32_t address) {
-  return &message_lookup[address];
+const Msg* CANPacker::lookup_message(uint32_t address) {
+  return dbc->addr_to_msg.at(address);
 }