Squashed 'cereal/' changes from 12aad06..4bb1eb8

4bb1eb8 submaster always conflates
30838d4 C++ implementation of SubMaster and PubMaster (#42)
c1a6d75 Fix potential segfault in MSGQPubSocket::connect (#45)
67fae6a Use ZMQ on MacOS (#46)
01cdf83 add default values for backwards compat
c96381b add OK flags to locationd output
d589d5e add white panda deprecation events
856c981 mark unused car events as deprecated
4f68db8 remove unnecessary new event type
9073b9b Library cleanup (#43)
7a786d9 move remaining alerts to car events
d6f10a4 add alert event type

git-subtree-dir: cereal
git-subtree-split: 4bb1eb8

Dockerfile

@@ -1,6 +1,6 @@
 from ubuntu:16.04
 
-RUN apt-get update && apt-get install -y libzmq3-dev clang wget git autoconf libtool curl make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python-openssl
+RUN apt-get update && apt-get install -y libzmq3-dev capnproto libcapnp-dev clang wget git autoconf libtool curl make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python-openssl
 
 RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
 ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}"
@@ -10,8 +10,6 @@ RUN pyenv rehash
 RUN pip3 install pyyaml==5.1.2 Cython==0.29.14 scons==3.1.1 pycapnp==0.6.4
 
 WORKDIR /project/cereal
-COPY install_capnp.sh .
-RUN ./install_capnp.sh
 
 ENV PYTHONPATH=/project
 

SConscript

@@ -20,6 +20,7 @@ if shutil.which('capnpc-java'):
 cereal_objects = env.SharedObject([
     'gen/cpp/car.capnp.c++',
     'gen/cpp/log.capnp.c++',
+    'messaging/socketmaster.cc',
   ])
 
 env.Library('cereal', cereal_objects)
@@ -43,8 +44,10 @@ Depends('messaging/impl_zmq.cc', services_h)
 
 # note, this rebuilds the deps shared, zmq is statically linked to make APK happy
 # TODO: get APK to load system zmq to remove the static link
-shared_lib_shared_lib = [zmq, 'm', 'stdc++'] + ["gnustl_shared"] if arch == "aarch64" else [zmq]
-env.SharedLibrary('messaging_shared', messaging_objects, LIBS=shared_lib_shared_lib)
+if arch == "aarch64":
+  zmq_static = FindFile("libzmq.a", "/usr/lib")
+  shared_lib_shared_lib = [zmq_static, 'm', 'stdc++', "gnustl_shared"]
+  env.SharedLibrary('messaging_shared', messaging_objects, LIBS=shared_lib_shared_lib)
 
 env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=[messaging_lib, 'zmq'])
 Depends('messaging/bridge.cc', services_h)

car.capnp

@@ -13,12 +13,12 @@ struct CarEvent @0x9b1657f34caf3ad3 {
   name @0 :EventName;
   enable @1 :Bool;
   noEntry @2 :Bool;
-  warning @3 :Bool;
+  warning @3 :Bool;   # alerts presented only when  enabled or soft disabling
   userDisable @4 :Bool;
   softDisable @5 :Bool;
   immediateDisable @6 :Bool;
   preEnable @7 :Bool;
-  permanent @8 :Bool;
+  permanent @8 :Bool; # alerts presented regardless of openpilot state
 
   enum EventName @0xbaa8c5d505f727de {
     # TODO: copy from error list
@@ -38,7 +38,7 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     pedalPressed @13;
     cruiseDisabled @14;
     radarCanError @15;
-    dataNeeded @16;
+    dataNeededDEPRECATED @16;
     speedTooLow @17;
     outOfSpace @18;
     overheat @19;
@@ -55,23 +55,23 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     manualRestart @30;
     lowSpeedLockout @31;
     plannerError @32;
-    ipasOverride @33;
+    ipasOverrideDEPRECATED @33;
     debugAlert @34;
     steerTempUnavailableMute @35;
     resumeRequired @36;
     preDriverDistracted @37;
     promptDriverDistracted @38;
     driverDistracted @39;
-    geofence @40;
-    driverMonitorOn @41;
-    driverMonitorOff @42;
+    geofenceDEPRECATED @40;
+    driverMonitorOnDEPRECATED @41;
+    driverMonitorOffDEPRECATED @42;
     preDriverUnresponsive @43;
     promptDriverUnresponsive @44;
     driverUnresponsive @45;
     belowSteerSpeed @46;
-    calibrationProgress @47;
+    calibrationProgressDEPRECATED @47;
     lowBattery @48;
-    invalidGiraffeHonda @49;
+    invalidGiraffeHondaDEPRECATED @49;
     vehicleModelInvalid @50;
     controlsFailed @51;
     sensorDataInvalid @52;
@@ -93,10 +93,18 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     driverMonitorLowAcc @68;
     invalidLkasSetting @69;
     speedTooHigh @70;
-    laneChangeBlocked @71;
+    laneChangeBlockedDEPRECATED @71;
     relayMalfunction @72;
     gasPressed @73;
     stockFcw @74;
+    startup @75;
+    startupNoCar @76;
+    startupNoControl @77;
+    startupMaster @78;
+    fcw @79;
+    steerSaturated @80;
+    whitePandaUnsupported @81;
+    startupWhitePanda @82;
   }
 }
 

install_capnp.sh

@@ -10,31 +10,3 @@ CXXFLAGS="-fPIC" ./configure
 
 make -j$(nproc)
 make install
-
-# manually build binaries statically
-g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -DCAPNP_INCLUDE_DIR=\"/usr/local/include\" -pthread -O2 -DNDEBUG -pthread -pthread -o .libs/capnp src/capnp/compiler/module-loader.o src/capnp/compiler/capnp.o  ./.libs/libcapnpc.a ./.libs/libcapnp.a ./.libs/libkj.a -lpthread -pthread
-
-g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -DCAPNP_INCLUDE_DIR=\"/usr/local/include\" -pthread -O2 -DNDEBUG -pthread -pthread -o .libs/capnpc-c++ src/capnp/compiler/capnpc-c++.o  ./.libs/libcapnp.a ./.libs/libkj.a -lpthread -pthread
-
-g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -DCAPNP_INCLUDE_DIR=\"/usr/local/include\" -pthread -O2 -DNDEBUG -pthread -pthread -o .libs/capnpc-capnp src/capnp/compiler/capnpc-capnp.o  ./.libs/libcapnp.a ./.libs/libkj.a -lpthread -pthread
-
-cp .libs/capnp /usr/local/bin/
-cp .libs/capnpc-c++ /usr/local/bin/
-cp .libs/capnpc-capnp /usr/local/bin/
-cp .libs/*.a /usr/local/lib
-
-cd /tmp
-echo "Installing c-capnp"
-git clone https://github.com/commaai/c-capnproto.git
-cd c-capnproto
-git submodule update --init --recursive
-autoreconf -f -i -s
-CXXFLAGS="-fPIC" ./configure
-make -j$(nproc)
-make install
-
-# manually build binaries statically
-gcc -fPIC -o .libs/capnpc-c compiler/capnpc-c.o compiler/schema.capnp.o compiler/str.o  ./.libs/libcapnp_c.a
-
-cp .libs/capnpc-c /usr/local/bin/
-cp .libs/*.a /usr/local/lib

log.capnp

@@ -855,6 +855,8 @@ struct LiveLocationKalman {
   gpsTimeOfWeek @14 :Float64;
   status @15 :Status;
   unixTimestampMillis @16 :Int64;
+  inputsOK @17 :Bool = true;
+  posenetOK @18 :Bool = true;
 
   enum Status {
     uninitialized @0;

messaging/impl_msgq.cc

@@ -143,7 +143,11 @@ int MSGQPubSocket::connect(Context *context, std::string endpoint){
   assert(context);
 
   q = new msgq_queue_t;
-  msgq_new_queue(q, endpoint.c_str(), DEFAULT_SEGMENT_SIZE);
+  int r = msgq_new_queue(q, endpoint.c_str(), DEFAULT_SEGMENT_SIZE);
+  if (r != 0){
+    return r;
+  }
+
   msgq_init_publisher(q);
 
   return 0;

messaging/messaging.cc

@@ -2,9 +2,15 @@
 #include "impl_zmq.hpp"
 #include "impl_msgq.hpp"
 
+#ifdef __APPLE__
+const bool MUST_USE_ZMQ = true;
+#else
+const bool MUST_USE_ZMQ = false;
+#endif
+
 Context * Context::create(){
   Context * c;
-  if (std::getenv("ZMQ")){
+  if (std::getenv("ZMQ") || MUST_USE_ZMQ){
     c = new ZMQContext();
   } else {
     c = new MSGQContext();
@@ -14,7 +20,7 @@ Context * Context::create(){
 
 SubSocket * SubSocket::create(){
   SubSocket * s;
-  if (std::getenv("ZMQ")){
+  if (std::getenv("ZMQ") || MUST_USE_ZMQ){
     s = new ZMQSubSocket();
   } else {
     s = new MSGQSubSocket();
@@ -60,7 +66,7 @@ SubSocket * SubSocket::create(Context * context, std::string endpoint, std::stri
 
 PubSocket * PubSocket::create(){
   PubSocket * s;
-  if (std::getenv("ZMQ")){
+  if (std::getenv("ZMQ") || MUST_USE_ZMQ){
     s = new ZMQPubSocket();
   } else {
     s = new MSGQPubSocket();
@@ -82,7 +88,7 @@ PubSocket * PubSocket::create(Context * context, std::string endpoint){
 
 Poller * Poller::create(){
   Poller * p;
-  if (std::getenv("ZMQ")){
+  if (std::getenv("ZMQ") || MUST_USE_ZMQ){
     p = new ZMQPoller();
   } else {
     p = new MSGQPoller();

messaging/messaging.hpp

@@ -1,7 +1,10 @@
 #pragma once
 #include <cstddef>
-#include <vector>
+#include <map>
 #include <string>
+#include <vector>
+#include <capnp/serialize.h>
+#include "../gen/cpp/log.capnp.h"
 
 #define MSG_MULTIPLE_PUBLISHERS 100
 
@@ -54,3 +57,36 @@ class Poller {
   static Poller * create(std::vector<SubSocket*> sockets);
   virtual ~Poller(){};
 };
+
+class SubMaster {
+ public:
+  SubMaster(const std::initializer_list<const char *> &service_list,
+            const char *address = nullptr, const std::initializer_list<const char *> &ignore_alive = {});
+  int update(int timeout = 1000);
+  inline bool allAlive(const std::initializer_list<const char *> &service_list = {}) { return all_(service_list, false, true); }
+  inline bool allValid(const std::initializer_list<const char *> &service_list = {}) { return all_(service_list, true, false); }
+  inline bool allAliveAndValid(const std::initializer_list<const char *> &service_list = {}) { return all_(service_list, true, true); }
+  bool updated(const char *name) const;
+  void drain();
+  cereal::Event::Reader &operator[](const char *name);
+  ~SubMaster();
+
+ private:
+  bool all_(const std::initializer_list<const char *> &service_list, bool valid, bool alive);
+  Poller *poller_ = nullptr;
+  uint64_t frame_ = 0;
+  struct SubMessage;
+  std::map<SubSocket *, SubMessage *> messages_;
+  std::map<std::string, SubMessage *> services_;
+};
+
+class PubMaster {
+ public:
+  PubMaster(const std::initializer_list<const char *> &service_list);
+  inline int send(const char *name, capnp::byte *data, size_t size) { return sockets_.at(name)->send((char *)data, size); }
+  int send(const char *name, capnp::MessageBuilder &msg);
+  ~PubMaster();
+
+ private:
+  std::map<std::string, PubSocket *> sockets_;
+};

messaging/messaging_pyx_setup.py

@@ -30,7 +30,7 @@ def get_ext_filename(self, ext_name):
 
 
 sourcefiles = ['messaging_pyx.pyx']
-extra_compile_args = ["-std=c++11"]
+extra_compile_args = ["-std=c++14"]
 libraries = ['zmq']
 ARCH = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()  # pylint: disable=unexpected-keyword-arg
 

messaging/msgq.cc

@@ -90,8 +90,10 @@ int msgq_new_queue(msgq_queue_t * q, const char * path, size_t size){
   auto fd = open(full_path, O_RDWR | O_CREAT, 0777);
   delete[] full_path;
 
-  if (fd < 0)
+  if (fd < 0) {
+    std::cout << "Warning, could not open: " << full_path << std::endl;
     return -1;
+  }
 
   int rc = ftruncate(fd, size + sizeof(msgq_header_t));
   if (rc < 0)