Devel (#6)

* Squashed 'cereal/' changes from b8382bb..01942b8 01942b8 add TODO b74a456 don't hardcode the lists ed5a4bf add face stds 396a2bb add can error counter to controlsState c6b5c73 Switch default to msgq (commaai#21) a457ffa Fix indentation in readme.md a1fc8c7 explicitly mention Python for syntax colouring (commaai#20) 19e2393 Fix expected for cameraOdometry and liveCalibration e7d2f97 Add radar comm issue error db64cd4 Reserve safety commaai#21 for VAG PQ35/PQ46/NMS (commaai#19) 79d638d separate honda safety models between Bosch Giraffe and Bosch Nidec 2614a65 better name b6b84cd add longitudinal 78f5934 Add canRxErrs to health 6758899 qlog liveCalibration df80b87 add more stuff to fw log in CarParams a87805a fix doxs 4746b20 got doxed 21cf3f5 build on mac 31ac47c Add carUnrecognized event git-subtree-dir: cereal git-subtree-split: 01942b8 * Remove old panda subtree * Squashed 'panda/' content from commit 3b35621 git-subtree-dir: panda git-subtree-split: 3b35621 * Squashed 'opendbc/' changes from 4f82d01e..5081966f 5081966f One more fix fa5dc680 Fix honda dbc files after steer torque addition e4dfb2fa update honda steering signals (commaai#208) 53fc4487 added generator test (commaai#207) cb27d6e3 Honda Nidec: add new ACC_HUD signals to all other cars other than the CIVIC 6e6779f9 build on mac AND linux, also gotta not use these python files 79dbfc1a build on mac git-subtree-dir: opendbc git-subtree-split: 5081966f106e4ed385d756220cfd3dc502e737d9 * openpilot v0.7.1 release * registration: send up both imeis * hotfix panda#426 * Panda signature needs to be bytes * update offroad apk: fix pairing * apk lib: Grant offroad access to TelephonyManager * update frame apk * catch exception in uploader.py * update common/android.py * update athenad.py Co-authored-by: Willem Melching <willem.melching@gmail.com> Co-authored-by: Andy <andyh2@me.com>
ntstrub · Jan 25, 2020 · 1c19479 · 1c19479
1 parent a2ae18d
commit 1c19479
Show file tree

Hide file tree

Showing 200 changed files with 5,867 additions and 4,154 deletions.
diff --git a/.gitignore b/.gitignore
@@ -3,11 +3,14 @@ venv/
 .tags
 .ipynb_checkpoints
 .idea
+.overlay_init
+.overlay_consistent
 .sconsign.dblite
 .vscode
 model2.png
 a.out
 
+*.dylib
 *.DSYM
 *.d
 *.pyc
@@ -27,6 +30,7 @@ a.out
 config.json
 clcache
 
+persist
 board/obj/
 selfdrive/boardd/boardd
 selfdrive/logcatd/logcatd
@@ -51,4 +55,5 @@ panda_jungle
 
 .coverage*
 htmlcov
+pandaextra
 
diff --git a/Dockerfile.openpilot b/Dockerfile.openpilot
@@ -17,12 +17,14 @@ RUN apt-get update && apt-get install -y \
     libffi-dev \
     libglew-dev \
     libgles2-mesa-dev \
+    libglfw3-dev \
     libglib2.0-0 \
     liblzma-dev \
     libmysqlclient-dev \
     libomp-dev \
     libopencv-dev \
     libssl-dev \
+    libsqlite3-dev \
     libtool \
     libusb-1.0-0-dev \
     libzmq5-dev \

diff --git a/Pipfile b/Pipfile
@@ -8,71 +8,54 @@ opencv-python= "==3.4.2.17"
 PyQt5 = "*"
 ipython = "*"
 networkx = "==2.3"
-azure-common = "==1.1.23"
+azure-core = "==1.1.1"
+azure-common = "==1.1.24"
 azure-nspkg = "==3.0.2"
 azure-storage-blob = "==2.1.0"
 azure-storage-common = "==2.1.0"
 azure-storage-nspkg = "==3.1.0"
-bincopy = "*"
-bleach = "*"
 boto = "*"
 "boto3" = "*"
-celery = "*"
 control = "*"
 datadog = "*"
-decorator = "*"
 dlib = "*"
-dominate = "*"
 elasticsearch = "*"
-fasteners = "*"
 future = "*"
 futures = "*"
-gevent = "*"
 pycocotools = {git = "https://github.com/cocodataset/cocoapi.git",subdirectory = "PythonAPI"}
 gunicorn = "*"
 "h5py" = "*"
 hexdump = "*"
 "html5lib" = "*"
 imageio = "*"
-intervaltree = "*"
 ipykernel = "*"
 joblib = "*"
 json-logging-py = "*"
 jupyter = "*"
 libarchive = "*"
 lru-dict = "*"
-lxml = "*"
 "mpld3" = "*"
 msgpack-python = "*"
 nbstripout = "*"
-nose-parameterized = "*"
 numpy = "*"
 osmium = "*"
 pbr = "*"
 percache = "*"
 pprofile = "*"
-psutil = "*"
 pycurl = "*"
 git-pylint-commit-hook = "*"
 pymongo = "*"
 "pynmea2" = "*"
 pypolyline = "*"
-pysendfile = "*"
 python-logstash = "*"
-pyvcd = "*"
 redis = "*"
-redlock = "*"
 "s2sphere" = "*"
 scikit-image = "*"
 "subprocess32" = "*"
-supervisor = "*"
 tenacity = "*"
 tensorflow-gpu = ""
-utm = "*"
-"v4l2" = "*"
 PyJWT = "==1.4.1"
 PyMySQL = "==0.9.2"
-Theano = "*"
 Werkzeug = "*"
 "backports.lzma" = "*"
 Flask-Cors = "*"
@@ -84,23 +67,20 @@ PyNaCl = "*"
 reverse_geocoder = "*"
 Shapely = "*"
 SQLAlchemy = "*"
-uWSGI = "*"
 scipy = "*"
 fastcluster = "*"
 backports-abc = "*"
 pygame = "*"
 simplejson = "*"
 python-logstash-async = "*"
 seaborn = "*"
-tensorflow-estimator = "*"
 pyproj = "*"
 mock = "*"
-blinker = "*"
-gast = "==0.2.2"
 matplotlib = "*"
 dictdiffer = "*"
 aenum = "*"
 coverage = "*"
+azure-cli-core = "*"
 
 [packages]
 overpy = {git = "https://github.com/commaai/python-overpy.git",ref = "f86529af402d4642e1faeb146671c40284007323"}
@@ -144,6 +124,5 @@ pillow = "*"
 scons = "*"
 cysignals = "*"
 
-
 [requires]
 python_version = "3.7.3"
diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/README.md b/README.md
@@ -57,7 +57,7 @@ openpilot should preserve all other vehicle's stock features, including, but are
 Supported Hardware
 ------
 
-At the moment, openpilot supports the [EON DevKit](https://comma.ai/shop/products/eon-dashcam-devkit). A [car harness](https://comma.ai/shop/products/car-harness) is recommended to connect the EON to the car. In the future, we'd like to support other platforms as well.
+At the moment, openpilot supports the [EON DevKit](https://comma.ai/shop/products/eon-dashcam-devkit) and the [comma two](https://comma.ai/shop/products/comma-two-devkit). A [car harness](https://comma.ai/shop/products/car-harness) is recommended to connect the EON or comma two to the car. In the future, we'd like to support other platforms as well, like gaming PCs.
 
 Supported Cars
 ------
@@ -159,6 +159,8 @@ Limitations of openpilot ALC and LDW
 
 openpilot ALC and openpilot LDW do not automatically drive the vehicle or reduce the amount of attention that must be paid to operate your vehicle. The driver must always keep control of the steering wheel and be ready to correct the openpilot ALC action at all times.
 
+While changing lanes, openpilot is not capable of looking next to you or checking your blind spot. Only nudge the wheel to initiate a lane change after you have confirmed it's safe to do so.
+
 Many factors can impact the performance of openpilot ALC and openpilot LDW, causing them to be unable to function as intended. These include, but are not limited to:
 
 * Poor visibility (heavy rain, snow, fog, etc.) or weather conditions that may interfere with sensor operation.

diff --git a/RELEASES.md b/RELEASES.md
@@ -1,3 +1,12 @@
+Version 0.7.1 (2020-01-20)
+========================
+ * comma two support!
+ * Lane Change Assist above 45 mph!
+ * Replace zmq with custom messaging library, msgq!
+ * Supercombo model: calibration and driving models are combined for better lead estimate
+ * More robust updater thanks to jyoung8607! Requires NEOS update
+ * Improve low speed ACC tuning
+
 Version 0.7 (2019-12-13)
 ========================
  * Move to SCons build system!

diff --git a/SConstruct b/SConstruct
@@ -46,6 +46,7 @@ else:
     "#phonelibs/capnp-cpp/include",
     "#phonelibs/capnp-c/include",
     "#phonelibs/zmq/x64/include",
+    "#external/tensorflow/include",
   ]
   libpath = [
     "#phonelibs/capnp-cpp/x64/lib",
@@ -55,13 +56,15 @@ else:
     "#phonelibs/zmq/x64/lib",
     "#phonelibs/libyuv/x64/lib",
     "#external/zmq/lib",
+    "#external/tensorflow/lib",
     "#cereal",
     "#selfdrive/common",
     "/usr/lib",
     "/usr/local/lib",
   ]
 
   rpath = ["phonelibs/capnp-cpp/x64/lib",
+           "external/tensorflow/lib",
            "cereal",
            "selfdrive/common"]
 
@@ -201,11 +204,13 @@ SConscript(['selfdrive/controls/lib/longitudinal_mpc/SConscript'])
 SConscript(['selfdrive/boardd/SConscript'])
 SConscript(['selfdrive/proclogd/SConscript'])
 
+SConscript(['selfdrive/ui/SConscript'])
+SConscript(['selfdrive/loggerd/SConscript'])
+
 if arch == "aarch64":
   SConscript(['selfdrive/logcatd/SConscript'])
-  SConscript(['selfdrive/ui/SConscript'])
   SConscript(['selfdrive/sensord/SConscript'])
-  SConscript(['selfdrive/loggerd/SConscript'])
+  SConscript(['selfdrive/clocksd/SConscript'])
 
 SConscript(['selfdrive/locationd/SConscript'])
 

diff --git a/apk/ai.comma.plus.frame.apk b/apk/ai.comma.plus.frame.apk
diff --git a/apk/ai.comma.plus.offroad.apk b/apk/ai.comma.plus.offroad.apk
diff --git a/cereal/.gitignore b/cereal/.gitignore
@@ -10,5 +10,5 @@ libmessaging.*
 libmessaging_shared.*
 services.h
 .sconsign.dblite
-libcereal_shared.so
+libcereal_shared.*
 
diff --git a/cereal/README.md b/cereal/README.md
@@ -0,0 +1,42 @@
+What is cereal?
+----
+
+cereal is both a messaging spec for robotics systems as well as generic high performance IPC pub sub messaging with a single publisher and multiple subscribers.
+
+Imagine this use case:
+* A sensor process reads gyro measurements directly from an IMU and publishes a sensorEvents packet
+* A calibration process subscribes to the sensorEvents packet to use the IMU
+* A localization process subscribes to the sensorEvents packet to use the IMU also
+
+
+Messaging Spec
+----
+
+You'll find the message types in [log.capnp](log.capnp). It uses [Cap'n proto](https://capnproto.org/capnp-tool.html) and defines one struct called Event.
+
+All Events have a logMonoTime and a valid. Then a big union defines the packet type.
+
+
+Pub Sub Backends
+----
+
+cereal supports two backends, one based on [zmq](https://zeromq.org/), the other called msgq, a custom pub sub based on shared memory that doesn't require the bytes to pass through the kernel.
+
+Example
+---
+```python
+import cereal.messaging as messaging
+
+# in subscriber
+sm = messaging.SubMaster(['sensorEvents'])
+while 1:
+  sm.update()
+  print(sm['sensorEvents'])
+
+# in publisher
+pm = messaging.PubMaster(['sensorEvents'])
+dat = messaging.new_message()
+dat.init('sensorEvents', 1)
+dat.sensorEvents[0] = {"gyro": {"v": [0.1, -0.1, 0.1]}}
+pm.send('sensorEvents', dat)
+```
diff --git a/cereal/SConscript b/cereal/SConscript
@@ -29,7 +29,7 @@ cereal_objects = env.SharedObject([
   ])
 
 env.Library('cereal', cereal_objects)
-env.SharedLibrary('cereal_shared', cereal_objects)
+env.SharedLibrary('cereal_shared', cereal_objects, LIBS=["capnp_c"])
 
 cereal_dir = Dir('.')
 services_h = env.Command(
@@ -49,7 +49,7 @@ Depends('messaging/impl_zmq.cc', services_h)
 
 # note, this rebuilds the deps shared, zmq is statically linked to make APK happy
 # TODO: get APK to load system zmq to remove the static link
-shared_lib_shared_lib = [zmq, 'm', 'stdc++'] + ["gnustl_shared"] if arch == "aarch64" else []
+shared_lib_shared_lib = [zmq, 'm', 'stdc++'] + ["gnustl_shared"] if arch == "aarch64" else [zmq]
 env.SharedLibrary('messaging_shared', messaging_objects, LIBS=shared_lib_shared_lib)
 
 env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=[messaging_lib, 'zmq'])

diff --git a/cereal/car.capnp b/cereal/car.capnp
@@ -88,6 +88,8 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     lowMemory @63;
     stockAeb @64;
     ldw @65;
+    carUnrecognized @66;
+    radarCommIssue @67;
   }
 }
 
@@ -410,11 +412,11 @@ struct CarParams {
 
   enum SafetyModel {
     silent @0;
-    honda @1;
+    hondaNidec @1;
     toyota @2;
     elm327 @3;
     gm @4;
-    hondaBosch @5;
+    hondaBoschGiraffe @5;
     ford @6;
     cadillac @7;
     hyundai @8;
@@ -428,7 +430,9 @@ struct CarParams {
     toyotaIpas @16;
     allOutput @17;
     gmAscm @18;
-    noOutput @19;  # like silent but with silent CAN TXs
+    noOutput @19;  # like silent but without silent CAN TXs
+    hondaBoschHarness @20;
+    volkswagenPq @21;
   }
 
   enum SteerControlType {
@@ -444,13 +448,21 @@ struct CarParams {
 
   struct CarFw {
     ecu @0 :Ecu;
-    fwVersion @1 :Text;
+    fwVersion @1 :Data;
+    address @2: UInt32;
+    subAddress @3: UInt8;
   }
 
   enum Ecu {
     eps @0;
     esp @1;
     fwdRadar @2;
     fwdCamera @3;
+    engine @4;
+    unknown @5;
+
+    # Toyota only
+    dsu @6;
+    apgs @7;
   }
 }
diff --git a/cereal/log.capnp b/cereal/log.capnp
@@ -310,6 +310,7 @@ struct HealthData {
   hasGps @6 :Bool;
   canSendErrs @7 :UInt32;
   canFwdErrs @8 :UInt32;
+  canRxErrs @19 :UInt32;
   gmlanSendErrs @9 :UInt32;
   hwType @10 :HwType;
   fanSpeedRpm @11 :UInt16;
@@ -484,6 +485,7 @@ struct ControlsState @0x97ff69c53601abf1 {
   decelForTurn @47 :Bool;
 
   decelForModel @54 :Bool;
+  canErrorCounter @57 :UInt32;
 
   lateralControlState :union {
     indiState @52 :LateralINDIState;
@@ -575,6 +577,7 @@ struct ModelData {
   leadFuture @7 :LeadData;
   speed @8 :List(Float32);
   meta @10 :MetaData;
+  longitudinal @11 :LongitudinalData;
 
   struct PathData {
     points @0 :List(Float32);
@@ -605,13 +608,19 @@ struct ModelData {
     yuvCorrection @5 :List(Float32);
     inputTransform @6 :List(Float32);
   }
+
   struct MetaData {
     engagedProb @0 :Float32;
     desirePrediction @1 :List(Float32);
     brakeDisengageProb @2 :Float32;
     gasDisengageProb @3 :Float32;
     steerOverrideProb @4 :Float32;
   }
+
+  struct LongitudinalData {
+    speeds @0 :List(Float32);
+    accelerations @1 :List(Float32);
+  }
 }
 
 struct CalibrationFeatures {
@@ -1757,6 +1766,8 @@ struct DriverMonitoring {
   leftBlinkProb @8 :Float32;
   rightBlinkProb @9 :Float32;
   irPwrDEPRECATED @10 :Float32;
+  faceOrientationStd @11 :List(Float32);
+  facePositionStd @12 :List(Float32);
 }
 
 struct Boot {