Squashed 'cereal/' changes from 12aad06..3ad1ac0

3ad1ac0 enable almost all flake8 checks
58878bf whitespace fix
b25a7d6 fix flake8 complaint about too many blank lines
76eb23e two spaces before inline comment
286b7e5 capitalize docker command
0adfc7e add canErrorPersistent event
c8be73d use github url instead of gitlab mirror
8e2d212 add pre-commit static analysis (#48)
f27222f add gpsOK flag to liveLocationKalman
4bb1eb8 submaster always conflates
30838d4 C++ implementation of SubMaster and PubMaster (#42)
c1a6d75 Fix potential segfault in MSGQPubSocket::connect (#45)
67fae6a Use ZMQ on MacOS (#46)
01cdf83 add default values for backwards compat
c96381b add OK flags to locationd output
d589d5e add white panda deprecation events
856c981 mark unused car events as deprecated
4f68db8 remove unnecessary new event type
9073b9b Library cleanup (#43)
7a786d9 move remaining alerts to car events
d6f10a4 add alert event type

git-subtree-dir: cereal
git-subtree-split: 3ad1ac0

.github/workflows/tests.yml

@@ -11,6 +11,9 @@ jobs:
     - uses: actions/checkout@v2
     - name: Build docker image
       run: docker build -t cereal .
+    - name: Static analysis
+      run: |
+        docker run cereal bash -c "git init && git add -A && pre-commit run --all"
     - name: Unit Tests
       run: |
         docker run cereal bash -c "scons --test --asan -j$(nproc) && messaging/test_runner"
@@ -20,4 +23,3 @@ jobs:
     - name: Test MSGQ
       run: |
         docker run cereal bash -c "MSGQ=1 python -m unittest discover ."
-

.gitignore

@@ -11,4 +11,4 @@ libmessaging_shared.*
 services.h
 .sconsign.dblite
 libcereal_shared.*
-
+.mypy_cache/

.pre-commit-config.yaml

@@ -0,0 +1,28 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: master
+    hooks:
+    -   id: check-ast
+    -   id: check-json
+    -   id: check-xml
+    -   id: check-yaml
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: master
+    hooks:
+    -   id: mypy
+-   repo: https://github.com/PyCQA/flake8
+    rev: master
+    hooks:
+    -   id: flake8
+        args:
+        - --ignore=E111,E114,E121,E124,E302,E501
+        - --statistics
+-   repo: local
+    hooks:
+    -   id: pylint
+        name: pylint
+        entry: pylint
+        language: system
+        types: [python]
+        args:
+        - --disable=R,C,W

Dockerfile

@@ -1,19 +1,18 @@
-from ubuntu:16.04
+FROM ubuntu:16.04
 
-RUN apt-get update && apt-get install -y libzmq3-dev clang wget git autoconf libtool curl make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python-openssl
+RUN apt-get update && apt-get install -y libzmq3-dev capnproto libcapnp-dev clang wget git autoconf libtool curl make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python-openssl
 
 RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
 ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}"
 RUN pyenv install 3.7.3
 RUN pyenv global 3.7.3
 RUN pyenv rehash
-RUN pip3 install pyyaml==5.1.2 Cython==0.29.14 scons==3.1.1 pycapnp==0.6.4
+RUN pip3 install pyyaml==5.1.2 Cython==0.29.14 scons==3.1.1 pycapnp==0.6.4 pre-commit==2.4.0 pylint==2.5.2
 
 WORKDIR /project/cereal
-COPY install_capnp.sh .
-RUN ./install_capnp.sh
 
 ENV PYTHONPATH=/project
 
 COPY . .
+RUN rm -rf .git
 RUN scons -c && scons -j$(nproc)

SConscript

@@ -20,6 +20,7 @@ if shutil.which('capnpc-java'):
 cereal_objects = env.SharedObject([
     'gen/cpp/car.capnp.c++',
     'gen/cpp/log.capnp.c++',
+    'messaging/socketmaster.cc',
   ])
 
 env.Library('cereal', cereal_objects)
@@ -43,8 +44,10 @@ Depends('messaging/impl_zmq.cc', services_h)
 
 # note, this rebuilds the deps shared, zmq is statically linked to make APK happy
 # TODO: get APK to load system zmq to remove the static link
-shared_lib_shared_lib = [zmq, 'm', 'stdc++'] + ["gnustl_shared"] if arch == "aarch64" else [zmq]
-env.SharedLibrary('messaging_shared', messaging_objects, LIBS=shared_lib_shared_lib)
+if arch == "aarch64":
+  zmq_static = FindFile("libzmq.a", "/usr/lib")
+  shared_lib_shared_lib = [zmq_static, 'm', 'stdc++', "gnustl_shared"]
+  env.SharedLibrary('messaging_shared', messaging_objects, LIBS=shared_lib_shared_lib)
 
 env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=[messaging_lib, 'zmq'])
 Depends('messaging/bridge.cc', services_h)

__init__.py

@@ -1,3 +1,4 @@
+# pylint: skip-file
 import os
 import capnp
 

car.capnp

@@ -13,12 +13,12 @@ struct CarEvent @0x9b1657f34caf3ad3 {
   name @0 :EventName;
   enable @1 :Bool;
   noEntry @2 :Bool;
-  warning @3 :Bool;
+  warning @3 :Bool;   # alerts presented only when  enabled or soft disabling
   userDisable @4 :Bool;
   softDisable @5 :Bool;
   immediateDisable @6 :Bool;
   preEnable @7 :Bool;
-  permanent @8 :Bool;
+  permanent @8 :Bool; # alerts presented regardless of openpilot state
 
   enum EventName @0xbaa8c5d505f727de {
     # TODO: copy from error list
@@ -38,7 +38,7 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     pedalPressed @13;
     cruiseDisabled @14;
     radarCanError @15;
-    dataNeeded @16;
+    dataNeededDEPRECATED @16;
     speedTooLow @17;
     outOfSpace @18;
     overheat @19;
@@ -55,23 +55,23 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     manualRestart @30;
     lowSpeedLockout @31;
     plannerError @32;
-    ipasOverride @33;
+    ipasOverrideDEPRECATED @33;
     debugAlert @34;
     steerTempUnavailableMute @35;
     resumeRequired @36;
     preDriverDistracted @37;
     promptDriverDistracted @38;
     driverDistracted @39;
-    geofence @40;
-    driverMonitorOn @41;
-    driverMonitorOff @42;
+    geofenceDEPRECATED @40;
+    driverMonitorOnDEPRECATED @41;
+    driverMonitorOffDEPRECATED @42;
     preDriverUnresponsive @43;
     promptDriverUnresponsive @44;
     driverUnresponsive @45;
     belowSteerSpeed @46;
-    calibrationProgress @47;
+    calibrationProgressDEPRECATED @47;
     lowBattery @48;
-    invalidGiraffeHonda @49;
+    invalidGiraffeHondaDEPRECATED @49;
     vehicleModelInvalid @50;
     controlsFailed @51;
     sensorDataInvalid @52;
@@ -93,10 +93,19 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     driverMonitorLowAcc @68;
     invalidLkasSetting @69;
     speedTooHigh @70;
-    laneChangeBlocked @71;
+    laneChangeBlockedDEPRECATED @71;
     relayMalfunction @72;
     gasPressed @73;
     stockFcw @74;
+    startup @75;
+    startupNoCar @76;
+    startupNoControl @77;
+    startupMaster @78;
+    fcw @79;
+    steerSaturated @80;
+    whitePandaUnsupported @81;
+    startupWhitePanda @82;
+    canErrorPersistent @83;
   }
 }
 

install_capnp.sh

@@ -10,31 +10,3 @@ CXXFLAGS="-fPIC" ./configure
 
 make -j$(nproc)
 make install
-
-# manually build binaries statically
-g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -DCAPNP_INCLUDE_DIR=\"/usr/local/include\" -pthread -O2 -DNDEBUG -pthread -pthread -o .libs/capnp src/capnp/compiler/module-loader.o src/capnp/compiler/capnp.o  ./.libs/libcapnpc.a ./.libs/libcapnp.a ./.libs/libkj.a -lpthread -pthread
-
-g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -DCAPNP_INCLUDE_DIR=\"/usr/local/include\" -pthread -O2 -DNDEBUG -pthread -pthread -o .libs/capnpc-c++ src/capnp/compiler/capnpc-c++.o  ./.libs/libcapnp.a ./.libs/libkj.a -lpthread -pthread
-
-g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -DCAPNP_INCLUDE_DIR=\"/usr/local/include\" -pthread -O2 -DNDEBUG -pthread -pthread -o .libs/capnpc-capnp src/capnp/compiler/capnpc-capnp.o  ./.libs/libcapnp.a ./.libs/libkj.a -lpthread -pthread
-
-cp .libs/capnp /usr/local/bin/
-cp .libs/capnpc-c++ /usr/local/bin/
-cp .libs/capnpc-capnp /usr/local/bin/
-cp .libs/*.a /usr/local/lib
-
-cd /tmp
-echo "Installing c-capnp"
-git clone https://github.com/commaai/c-capnproto.git
-cd c-capnproto
-git submodule update --init --recursive
-autoreconf -f -i -s
-CXXFLAGS="-fPIC" ./configure
-make -j$(nproc)
-make install
-
-# manually build binaries statically
-gcc -fPIC -o .libs/capnpc-c compiler/capnpc-c.o compiler/schema.capnp.o compiler/str.o  ./.libs/libcapnp_c.a
-
-cp .libs/capnpc-c /usr/local/bin/
-cp .libs/*.a /usr/local/lib

log.capnp

@@ -855,6 +855,9 @@ struct LiveLocationKalman {
   gpsTimeOfWeek @14 :Float64;
   status @15 :Status;
   unixTimestampMillis @16 :Int64;
+  inputsOK @17 :Bool = true;
+  posenetOK @18 :Bool = true;
+  gpsOK @19 :Bool = true;
 
   enum Status {
     uninitialized @0;

messaging/__init__.py

@@ -3,12 +3,12 @@
 from .messaging_pyx import MultiplePublishersError, MessagingError  # pylint: disable=no-name-in-module, import-error
 import capnp
 
-assert MultiplePublishersError
-assert MessagingError
-
 from cereal import log
 from cereal.services import service_list
 
+assert MultiplePublishersError
+assert MessagingError
+
 # sec_since_boot is faster, but allow to run standalone too
 try:
   from common.realtime import sec_since_boot
@@ -73,7 +73,7 @@ def drain_sock(sock, wait_for_one=False):
     else:
       dat = sock.receive(non_blocking=True)
 
-    if dat is None: # Timeout hit
+    if dat is None:  # Timeout hit
       break
 
     dat = log.Event.from_bytes(dat)
@@ -93,7 +93,7 @@ def recv_sock(sock, wait=False):
     else:
       rcv = sock.receive(non_blocking=True)
 
-    if rcv is None: # Timeout hit
+    if rcv is None:  # Timeout hit
       break
 
     dat = rcv
@@ -133,10 +133,10 @@ class SubMaster():
   def __init__(self, services, ignore_alive=None, addr="127.0.0.1"):
     self.poller = Poller()
     self.frame = -1
-    self.updated = {s : False for s in services}
-    self.rcv_time = {s : 0. for s in services}
-    self.rcv_frame = {s : 0 for s in services}
-    self.alive = {s : False for s in services}
+    self.updated = {s: False for s in services}
+    self.rcv_time = {s: 0. for s in services}
+    self.rcv_frame = {s: 0 for s in services}
+    self.alive = {s: False for s in services}
     self.sock = {}
     self.freq = {}
     self.data = {}
@@ -155,7 +155,7 @@ def __init__(self, services, ignore_alive=None, addr="127.0.0.1"):
 
       try:
         data = new_message(s)
-      except capnp.lib.capnp.KjException:
+      except capnp.lib.capnp.KjException:  # pylint: disable=c-extension-no-member
         # lists
         data = new_message(s, 0)
 

messaging/demo.py

@@ -1,6 +1,6 @@
 import time
 
-from messaging_pyx import Context, Poller, SubSocket, PubSocket # pylint: disable=no-name-in-module, import-error
+from messaging_pyx import Context, Poller, SubSocket, PubSocket  # pylint: disable=no-name-in-module, import-error
 
 MSGS = 1e5
 
@@ -12,7 +12,6 @@
   sub_sock.connect(c, "controlsState")
   pub_sock.connect(c, "controlsState")
 
-
   poller = Poller()
   poller.registerSocket(sub_sock)
 

messaging/impl_msgq.cc

@@ -143,7 +143,11 @@ int MSGQPubSocket::connect(Context *context, std::string endpoint){
   assert(context);
 
   q = new msgq_queue_t;
-  msgq_new_queue(q, endpoint.c_str(), DEFAULT_SEGMENT_SIZE);
+  int r = msgq_new_queue(q, endpoint.c_str(), DEFAULT_SEGMENT_SIZE);
+  if (r != 0){
+    return r;
+  }
+
   msgq_init_publisher(q);
 
   return 0;

messaging/messaging.cc

@@ -2,9 +2,15 @@
 #include "impl_zmq.hpp"
 #include "impl_msgq.hpp"
 
+#ifdef __APPLE__
+const bool MUST_USE_ZMQ = true;
+#else
+const bool MUST_USE_ZMQ = false;
+#endif
+
 Context * Context::create(){
   Context * c;
-  if (std::getenv("ZMQ")){
+  if (std::getenv("ZMQ") || MUST_USE_ZMQ){
     c = new ZMQContext();
   } else {
     c = new MSGQContext();
@@ -14,7 +20,7 @@ Context * Context::create(){
 
 SubSocket * SubSocket::create(){
   SubSocket * s;
-  if (std::getenv("ZMQ")){
+  if (std::getenv("ZMQ") || MUST_USE_ZMQ){
     s = new ZMQSubSocket();
   } else {
     s = new MSGQSubSocket();
@@ -60,7 +66,7 @@ SubSocket * SubSocket::create(Context * context, std::string endpoint, std::stri
 
 PubSocket * PubSocket::create(){
   PubSocket * s;
-  if (std::getenv("ZMQ")){
+  if (std::getenv("ZMQ") || MUST_USE_ZMQ){
     s = new ZMQPubSocket();
   } else {
     s = new MSGQPubSocket();
@@ -82,7 +88,7 @@ PubSocket * PubSocket::create(Context * context, std::string endpoint){
 
 Poller * Poller::create(){
   Poller * p;
-  if (std::getenv("ZMQ")){
+  if (std::getenv("ZMQ") || MUST_USE_ZMQ){
     p = new ZMQPoller();
   } else {
     p = new MSGQPoller();

messaging/messaging.hpp

@@ -1,7 +1,10 @@
 #pragma once
 #include <cstddef>
-#include <vector>
+#include <map>
 #include <string>
+#include <vector>
+#include <capnp/serialize.h>
+#include "../gen/cpp/log.capnp.h"
 
 #define MSG_MULTIPLE_PUBLISHERS 100
 
@@ -54,3 +57,36 @@ class Poller {
   static Poller * create(std::vector<SubSocket*> sockets);
   virtual ~Poller(){};
 };
+
+class SubMaster {
+ public:
+  SubMaster(const std::initializer_list<const char *> &service_list,
+            const char *address = nullptr, const std::initializer_list<const char *> &ignore_alive = {});
+  int update(int timeout = 1000);
+  inline bool allAlive(const std::initializer_list<const char *> &service_list = {}) { return all_(service_list, false, true); }
+  inline bool allValid(const std::initializer_list<const char *> &service_list = {}) { return all_(service_list, true, false); }
+  inline bool allAliveAndValid(const std::initializer_list<const char *> &service_list = {}) { return all_(service_list, true, true); }
+  bool updated(const char *name) const;
+  void drain();
+  cereal::Event::Reader &operator[](const char *name);
+  ~SubMaster();
+
+ private:
+  bool all_(const std::initializer_list<const char *> &service_list, bool valid, bool alive);
+  Poller *poller_ = nullptr;
+  uint64_t frame_ = 0;
+  struct SubMessage;
+  std::map<SubSocket *, SubMessage *> messages_;
+  std::map<std::string, SubMessage *> services_;
+};
+
+class PubMaster {
+ public:
+  PubMaster(const std::initializer_list<const char *> &service_list);
+  inline int send(const char *name, capnp::byte *data, size_t size) { return sockets_.at(name)->send((char *)data, size); }
+  int send(const char *name, capnp::MessageBuilder &msg);
+  ~PubMaster();
+
+ private:
+  std::map<std::string, PubSocket *> sockets_;
+};