Devel (#6)

* Squashed 'cereal/' changes from b8382bb..01942b8

01942b8 add TODO
b74a456 don't hardcode the lists
ed5a4bf add face stds
396a2bb add can error counter to controlsState
c6b5c73 Switch default to msgq (commaai#21)
a457ffa Fix indentation in readme.md
a1fc8c7 explicitly mention Python for syntax colouring (commaai#20)
19e2393 Fix expected for cameraOdometry and liveCalibration
e7d2f97 Add radar comm issue error
db64cd4 Reserve safety commaai#21 for VAG PQ35/PQ46/NMS (commaai#19)
79d638d separate honda safety models between Bosch Giraffe and Bosch Nidec
2614a65 better name
b6b84cd add longitudinal
78f5934 Add canRxErrs to health
6758899 qlog liveCalibration
df80b87 add more stuff to fw log in CarParams
a87805a fix doxs
4746b20 got doxed
21cf3f5 build on mac
31ac47c Add carUnrecognized event

git-subtree-dir: cereal
git-subtree-split: 01942b8

* Remove old panda subtree

* Squashed 'panda/' content from commit 3b35621

git-subtree-dir: panda
git-subtree-split: 3b35621

* Squashed 'opendbc/' changes from 4f82d01..5081966

5081966 One more fix
fa5dc68 Fix honda dbc files after steer torque addition
e4dfb2f update honda steering signals (commaai#208)
53fc448 added generator test (commaai#207)
cb27d6e Honda Nidec: add new ACC_HUD signals to all other cars other than the CIVIC
6e6779f build on mac AND linux, also gotta not use these python files
79dbfc1 build on mac

git-subtree-dir: opendbc
git-subtree-split: 5081966

* openpilot v0.7.1 release

* registration: send up both imeis

* hotfix panda#426

* Panda signature needs to be bytes

* update offroad apk: fix pairing

* apk lib: Grant offroad access to TelephonyManager

* update frame apk

* catch exception in uploader.py

* update common/android.py

* update athenad.py

Co-authored-by: Willem Melching <willem.melching@gmail.com>
Co-authored-by: Andy <andyh2@me.com>

Author: 3 people

.gitignore

@@ -3,11 +3,14 @@ venv/
 .tags
 .ipynb_checkpoints
 .idea
+.overlay_init
+.overlay_consistent
 .sconsign.dblite
 .vscode
 model2.png
 a.out
 
+*.dylib
 *.DSYM
 *.d
 *.pyc
@@ -27,6 +30,7 @@ a.out
 config.json
 clcache
 
+persist
 board/obj/
 selfdrive/boardd/boardd
 selfdrive/logcatd/logcatd
@@ -51,4 +55,5 @@ panda_jungle
 
 .coverage*
 htmlcov
+pandaextra
 

Dockerfile.openpilot

@@ -17,12 +17,14 @@ RUN apt-get update && apt-get install -y \
     libffi-dev \
     libglew-dev \
     libgles2-mesa-dev \
+    libglfw3-dev \
     libglib2.0-0 \
     liblzma-dev \
     libmysqlclient-dev \
     libomp-dev \
     libopencv-dev \
     libssl-dev \
+    libsqlite3-dev \
     libtool \
     libusb-1.0-0-dev \
     libzmq5-dev \

Pipfile

@@ -8,71 +8,54 @@ opencv-python= "==3.4.2.17"
 PyQt5 = "*"
 ipython = "*"
 networkx = "==2.3"
-azure-common = "==1.1.23"
+azure-core = "==1.1.1"
+azure-common = "==1.1.24"
 azure-nspkg = "==3.0.2"
 azure-storage-blob = "==2.1.0"
 azure-storage-common = "==2.1.0"
 azure-storage-nspkg = "==3.1.0"
-bincopy = "*"
-bleach = "*"
 boto = "*"
 "boto3" = "*"
-celery = "*"
 control = "*"
 datadog = "*"
-decorator = "*"
 dlib = "*"
-dominate = "*"
 elasticsearch = "*"
-fasteners = "*"
 future = "*"
 futures = "*"
-gevent = "*"
 pycocotools = {git = "https://github.com/cocodataset/cocoapi.git",subdirectory = "PythonAPI"}
 gunicorn = "*"
 "h5py" = "*"
 hexdump = "*"
 "html5lib" = "*"
 imageio = "*"
-intervaltree = "*"
 ipykernel = "*"
 joblib = "*"
 json-logging-py = "*"
 jupyter = "*"
 libarchive = "*"
 lru-dict = "*"
-lxml = "*"
 "mpld3" = "*"
 msgpack-python = "*"
 nbstripout = "*"
-nose-parameterized = "*"
 numpy = "*"
 osmium = "*"
 pbr = "*"
 percache = "*"
 pprofile = "*"
-psutil = "*"
 pycurl = "*"
 git-pylint-commit-hook = "*"
 pymongo = "*"
 "pynmea2" = "*"
 pypolyline = "*"
-pysendfile = "*"
 python-logstash = "*"
-pyvcd = "*"
 redis = "*"
-redlock = "*"
 "s2sphere" = "*"
 scikit-image = "*"
 "subprocess32" = "*"
-supervisor = "*"
 tenacity = "*"
 tensorflow-gpu = ""
-utm = "*"
-"v4l2" = "*"
 PyJWT = "==1.4.1"
 PyMySQL = "==0.9.2"
-Theano = "*"
 Werkzeug = "*"
 "backports.lzma" = "*"
 Flask-Cors = "*"
@@ -84,23 +67,20 @@ PyNaCl = "*"
 reverse_geocoder = "*"
 Shapely = "*"
 SQLAlchemy = "*"
-uWSGI = "*"
 scipy = "*"
 fastcluster = "*"
 backports-abc = "*"
 pygame = "*"
 simplejson = "*"
 python-logstash-async = "*"
 seaborn = "*"
-tensorflow-estimator = "*"
 pyproj = "*"
 mock = "*"
-blinker = "*"
-gast = "==0.2.2"
 matplotlib = "*"
 dictdiffer = "*"
 aenum = "*"
 coverage = "*"
+azure-cli-core = "*"
 
 [packages]
 overpy = {git = "https://github.com/commaai/python-overpy.git",ref = "f86529af402d4642e1faeb146671c40284007323"}
@@ -144,6 +124,5 @@ pillow = "*"
 scons = "*"
 cysignals = "*"
 
-
 [requires]
 python_version = "3.7.3"

Pipfile.lock

@@ -57,7 +57,7 @@ openpilot should preserve all other vehicle's stock features, including, but are
 Supported Hardware
 ------
 
-At the moment, openpilot supports the [EON DevKit](https://comma.ai/shop/products/eon-dashcam-devkit). A [car harness](https://comma.ai/shop/products/car-harness) is recommended to connect the EON to the car. In the future, we'd like to support other platforms as well.
+At the moment, openpilot supports the [EON DevKit](https://comma.ai/shop/products/eon-dashcam-devkit) and the [comma two](https://comma.ai/shop/products/comma-two-devkit). A [car harness](https://comma.ai/shop/products/car-harness) is recommended to connect the EON or comma two to the car. In the future, we'd like to support other platforms as well, like gaming PCs.
 
 Supported Cars
 ------
@@ -159,6 +159,8 @@ Limitations of openpilot ALC and LDW
 
 openpilot ALC and openpilot LDW do not automatically drive the vehicle or reduce the amount of attention that must be paid to operate your vehicle. The driver must always keep control of the steering wheel and be ready to correct the openpilot ALC action at all times.
 
+While changing lanes, openpilot is not capable of looking next to you or checking your blind spot. Only nudge the wheel to initiate a lane change after you have confirmed it's safe to do so.
+
 Many factors can impact the performance of openpilot ALC and openpilot LDW, causing them to be unable to function as intended. These include, but are not limited to:
 
 * Poor visibility (heavy rain, snow, fog, etc.) or weather conditions that may interfere with sensor operation.

README.md

@@ -1,3 +1,12 @@
+Version 0.7.1 (2020-01-20)
+========================
+ * comma two support!
+ * Lane Change Assist above 45 mph!
+ * Replace zmq with custom messaging library, msgq!
+ * Supercombo model: calibration and driving models are combined for better lead estimate
+ * More robust updater thanks to jyoung8607! Requires NEOS update
+ * Improve low speed ACC tuning
+
 Version 0.7 (2019-12-13)
 ========================
  * Move to SCons build system!

RELEASES.md

@@ -46,6 +46,7 @@ else:
     "#phonelibs/capnp-cpp/include",
     "#phonelibs/capnp-c/include",
     "#phonelibs/zmq/x64/include",
+    "#external/tensorflow/include",
   ]
   libpath = [
     "#phonelibs/capnp-cpp/x64/lib",
@@ -55,13 +56,15 @@ else:
     "#phonelibs/zmq/x64/lib",
     "#phonelibs/libyuv/x64/lib",
     "#external/zmq/lib",
+    "#external/tensorflow/lib",
     "#cereal",
     "#selfdrive/common",
     "/usr/lib",
     "/usr/local/lib",
   ]
 
   rpath = ["phonelibs/capnp-cpp/x64/lib",
+           "external/tensorflow/lib",
            "cereal",
            "selfdrive/common"]
 
@@ -201,11 +204,13 @@ SConscript(['selfdrive/controls/lib/longitudinal_mpc/SConscript'])
 SConscript(['selfdrive/boardd/SConscript'])
 SConscript(['selfdrive/proclogd/SConscript'])
 
+SConscript(['selfdrive/ui/SConscript'])
+SConscript(['selfdrive/loggerd/SConscript'])
+
 if arch == "aarch64":
   SConscript(['selfdrive/logcatd/SConscript'])
-  SConscript(['selfdrive/ui/SConscript'])
   SConscript(['selfdrive/sensord/SConscript'])
-  SConscript(['selfdrive/loggerd/SConscript'])
+  SConscript(['selfdrive/clocksd/SConscript'])
 
 SConscript(['selfdrive/locationd/SConscript'])
 

SConstruct

@@ -10,5 +10,5 @@ libmessaging.*
 libmessaging_shared.*
 services.h
 .sconsign.dblite
-libcereal_shared.so
+libcereal_shared.*
 

apk/ai.comma.plus.frame.apk

@@ -0,0 +1,42 @@
+What is cereal?
+----
+
+cereal is both a messaging spec for robotics systems as well as generic high performance IPC pub sub messaging with a single publisher and multiple subscribers.
+
+Imagine this use case:
+* A sensor process reads gyro measurements directly from an IMU and publishes a sensorEvents packet
+* A calibration process subscribes to the sensorEvents packet to use the IMU
+* A localization process subscribes to the sensorEvents packet to use the IMU also
+
+
+Messaging Spec
+----
+
+You'll find the message types in [log.capnp](log.capnp). It uses [Cap'n proto](https://capnproto.org/capnp-tool.html) and defines one struct called Event.
+
+All Events have a logMonoTime and a valid. Then a big union defines the packet type.
+
+
+Pub Sub Backends
+----
+
+cereal supports two backends, one based on [zmq](https://zeromq.org/), the other called msgq, a custom pub sub based on shared memory that doesn't require the bytes to pass through the kernel.
+
+Example
+---
+```python
+import cereal.messaging as messaging
+
+# in subscriber
+sm = messaging.SubMaster(['sensorEvents'])
+while 1:
+  sm.update()
+  print(sm['sensorEvents'])
+
+# in publisher
+pm = messaging.PubMaster(['sensorEvents'])
+dat = messaging.new_message()
+dat.init('sensorEvents', 1)
+dat.sensorEvents[0] = {"gyro": {"v": [0.1, -0.1, 0.1]}}
+pm.send('sensorEvents', dat)
+```

apk/ai.comma.plus.offroad.apk

@@ -29,7 +29,7 @@ cereal_objects = env.SharedObject([
   ])
 
 env.Library('cereal', cereal_objects)
-env.SharedLibrary('cereal_shared', cereal_objects)
+env.SharedLibrary('cereal_shared', cereal_objects, LIBS=["capnp_c"])
 
 cereal_dir = Dir('.')
 services_h = env.Command(
@@ -49,7 +49,7 @@ Depends('messaging/impl_zmq.cc', services_h)
 
 # note, this rebuilds the deps shared, zmq is statically linked to make APK happy
 # TODO: get APK to load system zmq to remove the static link
-shared_lib_shared_lib = [zmq, 'm', 'stdc++'] + ["gnustl_shared"] if arch == "aarch64" else []
+shared_lib_shared_lib = [zmq, 'm', 'stdc++'] + ["gnustl_shared"] if arch == "aarch64" else [zmq]
 env.SharedLibrary('messaging_shared', messaging_objects, LIBS=shared_lib_shared_lib)
 
 env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=[messaging_lib, 'zmq'])

cereal/.gitignore

@@ -88,6 +88,8 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     lowMemory @63;
     stockAeb @64;
     ldw @65;
+    carUnrecognized @66;
+    radarCommIssue @67;
   }
 }
 
@@ -410,11 +412,11 @@ struct CarParams {
 
   enum SafetyModel {
     silent @0;
-    honda @1;
+    hondaNidec @1;
     toyota @2;
     elm327 @3;
     gm @4;
-    hondaBosch @5;
+    hondaBoschGiraffe @5;
     ford @6;
     cadillac @7;
     hyundai @8;
@@ -428,7 +430,9 @@ struct CarParams {
     toyotaIpas @16;
     allOutput @17;
     gmAscm @18;
-    noOutput @19;  # like silent but with silent CAN TXs
+    noOutput @19;  # like silent but without silent CAN TXs
+    hondaBoschHarness @20;
+    volkswagenPq @21;
   }
 
   enum SteerControlType {
@@ -444,13 +448,21 @@ struct CarParams {
 
   struct CarFw {
     ecu @0 :Ecu;
-    fwVersion @1 :Text;
+    fwVersion @1 :Data;
+    address @2: UInt32;
+    subAddress @3: UInt8;
   }
 
   enum Ecu {
     eps @0;
     esp @1;
     fwdRadar @2;
     fwdCamera @3;
+    engine @4;
+    unknown @5;
+
+    # Toyota only
+    dsu @6;
+    apgs @7;
   }
 }

cereal/README.md

@@ -310,6 +310,7 @@ struct HealthData {
   hasGps @6 :Bool;
   canSendErrs @7 :UInt32;
   canFwdErrs @8 :UInt32;
+  canRxErrs @19 :UInt32;
   gmlanSendErrs @9 :UInt32;
   hwType @10 :HwType;
   fanSpeedRpm @11 :UInt16;
@@ -484,6 +485,7 @@ struct ControlsState @0x97ff69c53601abf1 {
   decelForTurn @47 :Bool;
 
   decelForModel @54 :Bool;
+  canErrorCounter @57 :UInt32;
 
   lateralControlState :union {
     indiState @52 :LateralINDIState;
@@ -575,6 +577,7 @@ struct ModelData {
   leadFuture @7 :LeadData;
   speed @8 :List(Float32);
   meta @10 :MetaData;
+  longitudinal @11 :LongitudinalData;
 
   struct PathData {
     points @0 :List(Float32);
@@ -605,13 +608,19 @@ struct ModelData {
     yuvCorrection @5 :List(Float32);
     inputTransform @6 :List(Float32);
   }
+
   struct MetaData {
     engagedProb @0 :Float32;
     desirePrediction @1 :List(Float32);
     brakeDisengageProb @2 :Float32;
     gasDisengageProb @3 :Float32;
     steerOverrideProb @4 :Float32;
   }
+
+  struct LongitudinalData {
+    speeds @0 :List(Float32);
+    accelerations @1 :List(Float32);
+  }
 }
 
 struct CalibrationFeatures {
@@ -1757,6 +1766,8 @@ struct DriverMonitoring {
   leftBlinkProb @8 :Float32;
   rightBlinkProb @9 :Float32;
   irPwrDEPRECATED @10 :Float32;
+  faceOrientationStd @11 :List(Float32);
+  facePositionStd @12 :List(Float32);
 }
 
 struct Boot {