From 23bbf525789a73b098409057d960f38a7afe5280 Mon Sep 17 00:00:00 2001 From: Vehicle Researcher Date: Sun, 31 May 2020 13:22:40 -0700 Subject: [PATCH] Squashed 'cereal/' changes from 12aad06e1..286b7e58b 286b7e58b capitalize docker command 0adfc7e77 add canErrorPersistent event c8be73d10 use github url instead of gitlab mirror 8e2d212a3 add pre-commit static analysis (#48) f27222f8f add gpsOK flag to liveLocationKalman 4bb1eb826 submaster always conflates 30838d40a C++ implementation of SubMaster and PubMaster (#42) c1a6d75d1 Fix potential segfault in MSGQPubSocket::connect (#45) 67fae6afc Use ZMQ on MacOS (#46) 01cdf832c add default values for backwards compat c96381b0d add OK flags to locationd output d589d5e3d add white panda deprecation events 856c9812d mark unused car events as deprecated 4f68db8f6 remove unnecessary new event type 9073b9b1b Library cleanup (#43) 7a786d9ce move remaining alerts to car events d6f10a4b9 add alert event type git-subtree-dir: cereal git-subtree-split: 286b7e58b5e7ede697370acc77fc7cca21d69764 --- .github/workflows/tests.yml | 4 +- .gitignore | 2 +- .pre-commit-config.yaml | 27 ++++++ Dockerfile | 9 +- SConscript | 7 +- __init__.py | 1 + car.capnp | 29 ++++-- install_capnp.sh | 28 ------ log.capnp | 3 + messaging/__init__.py | 2 +- messaging/impl_msgq.cc | 6 +- messaging/messaging.cc | 14 ++- messaging/messaging.hpp | 38 +++++++- messaging/messaging_pyx_setup.py | 2 +- messaging/msgq.cc | 4 +- messaging/socketmaster.cc | 162 +++++++++++++++++++++++++++++++ 16 files changed, 282 insertions(+), 56 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 messaging/socketmaster.cc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3b8f6ca8e03b87..51b8364793d112 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,6 +11,9 @@ jobs: - uses: actions/checkout@v2 - name: Build docker image run: docker build -t cereal . + - name: Static analysis + run: | + docker run cereal bash -c "git init && git add -A && pre-commit run --all" - name: Unit Tests run: | docker run cereal bash -c "scons --test --asan -j$(nproc) && messaging/test_runner" @@ -20,4 +23,3 @@ jobs: - name: Test MSGQ run: | docker run cereal bash -c "MSGQ=1 python -m unittest discover ." - diff --git a/.gitignore b/.gitignore index e90f8c750f9b26..9e2fec81dc159e 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,4 @@ libmessaging_shared.* services.h .sconsign.dblite libcereal_shared.* - +.mypy_cache/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000000000..602d9d8ee634e1 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,27 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: master + hooks: + - id: check-ast + - id: check-json + - id: check-xml + - id: check-yaml +- repo: https://github.com/pre-commit/mirrors-mypy + rev: master + hooks: + - id: mypy +- repo: https://github.com/PyCQA/flake8 + rev: master + hooks: + - id: flake8 + args: + - --select=F +- repo: local + hooks: + - id: pylint + name: pylint + entry: pylint + language: system + types: [python] + args: + - --disable=R,C,W diff --git a/Dockerfile b/Dockerfile index f681dce68ff4b5..f4b8f2d69b8c64 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,18 @@ -from ubuntu:16.04 +FROM ubuntu:16.04 -RUN apt-get update && apt-get install -y libzmq3-dev clang wget git autoconf libtool curl make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python-openssl +RUN apt-get update && apt-get install -y libzmq3-dev capnproto libcapnp-dev clang wget git autoconf libtool curl make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python-openssl RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}" RUN pyenv install 3.7.3 RUN pyenv global 3.7.3 RUN pyenv rehash -RUN pip3 install pyyaml==5.1.2 Cython==0.29.14 scons==3.1.1 pycapnp==0.6.4 +RUN pip3 install pyyaml==5.1.2 Cython==0.29.14 scons==3.1.1 pycapnp==0.6.4 pre-commit==2.4.0 pylint==2.5.2 WORKDIR /project/cereal -COPY install_capnp.sh . -RUN ./install_capnp.sh ENV PYTHONPATH=/project COPY . . +RUN rm -rf .git RUN scons -c && scons -j$(nproc) diff --git a/SConscript b/SConscript index ed63c1717b797f..a724dfa6b938a4 100644 --- a/SConscript +++ b/SConscript @@ -20,6 +20,7 @@ if shutil.which('capnpc-java'): cereal_objects = env.SharedObject([ 'gen/cpp/car.capnp.c++', 'gen/cpp/log.capnp.c++', + 'messaging/socketmaster.cc', ]) env.Library('cereal', cereal_objects) @@ -43,8 +44,10 @@ Depends('messaging/impl_zmq.cc', services_h) # note, this rebuilds the deps shared, zmq is statically linked to make APK happy # TODO: get APK to load system zmq to remove the static link -shared_lib_shared_lib = [zmq, 'm', 'stdc++'] + ["gnustl_shared"] if arch == "aarch64" else [zmq] -env.SharedLibrary('messaging_shared', messaging_objects, LIBS=shared_lib_shared_lib) +if arch == "aarch64": + zmq_static = FindFile("libzmq.a", "/usr/lib") + shared_lib_shared_lib = [zmq_static, 'm', 'stdc++', "gnustl_shared"] + env.SharedLibrary('messaging_shared', messaging_objects, LIBS=shared_lib_shared_lib) env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=[messaging_lib, 'zmq']) Depends('messaging/bridge.cc', services_h) diff --git a/__init__.py b/__init__.py index 2d3b48526b2c34..88a181cbc51da4 100644 --- a/__init__.py +++ b/__init__.py @@ -1,3 +1,4 @@ +# pylint: skip-file import os import capnp diff --git a/car.capnp b/car.capnp index 3ef7117384c311..c4be41b0737001 100644 --- a/car.capnp +++ b/car.capnp @@ -13,12 +13,12 @@ struct CarEvent @0x9b1657f34caf3ad3 { name @0 :EventName; enable @1 :Bool; noEntry @2 :Bool; - warning @3 :Bool; + warning @3 :Bool; # alerts presented only when enabled or soft disabling userDisable @4 :Bool; softDisable @5 :Bool; immediateDisable @6 :Bool; preEnable @7 :Bool; - permanent @8 :Bool; + permanent @8 :Bool; # alerts presented regardless of openpilot state enum EventName @0xbaa8c5d505f727de { # TODO: copy from error list @@ -38,7 +38,7 @@ struct CarEvent @0x9b1657f34caf3ad3 { pedalPressed @13; cruiseDisabled @14; radarCanError @15; - dataNeeded @16; + dataNeededDEPRECATED @16; speedTooLow @17; outOfSpace @18; overheat @19; @@ -55,23 +55,23 @@ struct CarEvent @0x9b1657f34caf3ad3 { manualRestart @30; lowSpeedLockout @31; plannerError @32; - ipasOverride @33; + ipasOverrideDEPRECATED @33; debugAlert @34; steerTempUnavailableMute @35; resumeRequired @36; preDriverDistracted @37; promptDriverDistracted @38; driverDistracted @39; - geofence @40; - driverMonitorOn @41; - driverMonitorOff @42; + geofenceDEPRECATED @40; + driverMonitorOnDEPRECATED @41; + driverMonitorOffDEPRECATED @42; preDriverUnresponsive @43; promptDriverUnresponsive @44; driverUnresponsive @45; belowSteerSpeed @46; - calibrationProgress @47; + calibrationProgressDEPRECATED @47; lowBattery @48; - invalidGiraffeHonda @49; + invalidGiraffeHondaDEPRECATED @49; vehicleModelInvalid @50; controlsFailed @51; sensorDataInvalid @52; @@ -93,10 +93,19 @@ struct CarEvent @0x9b1657f34caf3ad3 { driverMonitorLowAcc @68; invalidLkasSetting @69; speedTooHigh @70; - laneChangeBlocked @71; + laneChangeBlockedDEPRECATED @71; relayMalfunction @72; gasPressed @73; stockFcw @74; + startup @75; + startupNoCar @76; + startupNoControl @77; + startupMaster @78; + fcw @79; + steerSaturated @80; + whitePandaUnsupported @81; + startupWhitePanda @82; + canErrorPersistent @83; } } diff --git a/install_capnp.sh b/install_capnp.sh index cc570b60ffc884..2d2d43d6141ce9 100755 --- a/install_capnp.sh +++ b/install_capnp.sh @@ -10,31 +10,3 @@ CXXFLAGS="-fPIC" ./configure make -j$(nproc) make install - -# manually build binaries statically -g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -DCAPNP_INCLUDE_DIR=\"/usr/local/include\" -pthread -O2 -DNDEBUG -pthread -pthread -o .libs/capnp src/capnp/compiler/module-loader.o src/capnp/compiler/capnp.o ./.libs/libcapnpc.a ./.libs/libcapnp.a ./.libs/libkj.a -lpthread -pthread - -g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -DCAPNP_INCLUDE_DIR=\"/usr/local/include\" -pthread -O2 -DNDEBUG -pthread -pthread -o .libs/capnpc-c++ src/capnp/compiler/capnpc-c++.o ./.libs/libcapnp.a ./.libs/libkj.a -lpthread -pthread - -g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -DCAPNP_INCLUDE_DIR=\"/usr/local/include\" -pthread -O2 -DNDEBUG -pthread -pthread -o .libs/capnpc-capnp src/capnp/compiler/capnpc-capnp.o ./.libs/libcapnp.a ./.libs/libkj.a -lpthread -pthread - -cp .libs/capnp /usr/local/bin/ -cp .libs/capnpc-c++ /usr/local/bin/ -cp .libs/capnpc-capnp /usr/local/bin/ -cp .libs/*.a /usr/local/lib - -cd /tmp -echo "Installing c-capnp" -git clone https://github.com/commaai/c-capnproto.git -cd c-capnproto -git submodule update --init --recursive -autoreconf -f -i -s -CXXFLAGS="-fPIC" ./configure -make -j$(nproc) -make install - -# manually build binaries statically -gcc -fPIC -o .libs/capnpc-c compiler/capnpc-c.o compiler/schema.capnp.o compiler/str.o ./.libs/libcapnp_c.a - -cp .libs/capnpc-c /usr/local/bin/ -cp .libs/*.a /usr/local/lib diff --git a/log.capnp b/log.capnp index 3744c46126f7fb..4c64ee3b7bd920 100644 --- a/log.capnp +++ b/log.capnp @@ -855,6 +855,9 @@ struct LiveLocationKalman { gpsTimeOfWeek @14 :Float64; status @15 :Status; unixTimestampMillis @16 :Int64; + inputsOK @17 :Bool = true; + posenetOK @18 :Bool = true; + gpsOK @19 :Bool = true; enum Status { uninitialized @0; diff --git a/messaging/__init__.py b/messaging/__init__.py index b53d6fb7ab6247..31b9147eec5d8f 100644 --- a/messaging/__init__.py +++ b/messaging/__init__.py @@ -155,7 +155,7 @@ def __init__(self, services, ignore_alive=None, addr="127.0.0.1"): try: data = new_message(s) - except capnp.lib.capnp.KjException: + except capnp.lib.capnp.KjException: # pylint: disable=c-extension-no-member # lists data = new_message(s, 0) diff --git a/messaging/impl_msgq.cc b/messaging/impl_msgq.cc index d37b8c986d7237..0a51d12c17ff4e 100644 --- a/messaging/impl_msgq.cc +++ b/messaging/impl_msgq.cc @@ -143,7 +143,11 @@ int MSGQPubSocket::connect(Context *context, std::string endpoint){ assert(context); q = new msgq_queue_t; - msgq_new_queue(q, endpoint.c_str(), DEFAULT_SEGMENT_SIZE); + int r = msgq_new_queue(q, endpoint.c_str(), DEFAULT_SEGMENT_SIZE); + if (r != 0){ + return r; + } + msgq_init_publisher(q); return 0; diff --git a/messaging/messaging.cc b/messaging/messaging.cc index 1a9f860cd35666..9e2ae409d1dc08 100644 --- a/messaging/messaging.cc +++ b/messaging/messaging.cc @@ -2,9 +2,15 @@ #include "impl_zmq.hpp" #include "impl_msgq.hpp" +#ifdef __APPLE__ +const bool MUST_USE_ZMQ = true; +#else +const bool MUST_USE_ZMQ = false; +#endif + Context * Context::create(){ Context * c; - if (std::getenv("ZMQ")){ + if (std::getenv("ZMQ") || MUST_USE_ZMQ){ c = new ZMQContext(); } else { c = new MSGQContext(); @@ -14,7 +20,7 @@ Context * Context::create(){ SubSocket * SubSocket::create(){ SubSocket * s; - if (std::getenv("ZMQ")){ + if (std::getenv("ZMQ") || MUST_USE_ZMQ){ s = new ZMQSubSocket(); } else { s = new MSGQSubSocket(); @@ -60,7 +66,7 @@ SubSocket * SubSocket::create(Context * context, std::string endpoint, std::stri PubSocket * PubSocket::create(){ PubSocket * s; - if (std::getenv("ZMQ")){ + if (std::getenv("ZMQ") || MUST_USE_ZMQ){ s = new ZMQPubSocket(); } else { s = new MSGQPubSocket(); @@ -82,7 +88,7 @@ PubSocket * PubSocket::create(Context * context, std::string endpoint){ Poller * Poller::create(){ Poller * p; - if (std::getenv("ZMQ")){ + if (std::getenv("ZMQ") || MUST_USE_ZMQ){ p = new ZMQPoller(); } else { p = new MSGQPoller(); diff --git a/messaging/messaging.hpp b/messaging/messaging.hpp index 79968c97986f01..20bb35213c02ae 100644 --- a/messaging/messaging.hpp +++ b/messaging/messaging.hpp @@ -1,7 +1,10 @@ #pragma once #include -#include +#include #include +#include +#include +#include "../gen/cpp/log.capnp.h" #define MSG_MULTIPLE_PUBLISHERS 100 @@ -54,3 +57,36 @@ class Poller { static Poller * create(std::vector sockets); virtual ~Poller(){}; }; + +class SubMaster { + public: + SubMaster(const std::initializer_list &service_list, + const char *address = nullptr, const std::initializer_list &ignore_alive = {}); + int update(int timeout = 1000); + inline bool allAlive(const std::initializer_list &service_list = {}) { return all_(service_list, false, true); } + inline bool allValid(const std::initializer_list &service_list = {}) { return all_(service_list, true, false); } + inline bool allAliveAndValid(const std::initializer_list &service_list = {}) { return all_(service_list, true, true); } + bool updated(const char *name) const; + void drain(); + cereal::Event::Reader &operator[](const char *name); + ~SubMaster(); + + private: + bool all_(const std::initializer_list &service_list, bool valid, bool alive); + Poller *poller_ = nullptr; + uint64_t frame_ = 0; + struct SubMessage; + std::map messages_; + std::map services_; +}; + +class PubMaster { + public: + PubMaster(const std::initializer_list &service_list); + inline int send(const char *name, capnp::byte *data, size_t size) { return sockets_.at(name)->send((char *)data, size); } + int send(const char *name, capnp::MessageBuilder &msg); + ~PubMaster(); + + private: + std::map sockets_; +}; diff --git a/messaging/messaging_pyx_setup.py b/messaging/messaging_pyx_setup.py index f5962d0c2b2310..3e1c41ce15f58b 100644 --- a/messaging/messaging_pyx_setup.py +++ b/messaging/messaging_pyx_setup.py @@ -30,7 +30,7 @@ def get_ext_filename(self, ext_name): sourcefiles = ['messaging_pyx.pyx'] -extra_compile_args = ["-std=c++11"] +extra_compile_args = ["-std=c++14"] libraries = ['zmq'] ARCH = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip() # pylint: disable=unexpected-keyword-arg diff --git a/messaging/msgq.cc b/messaging/msgq.cc index 4ccd13df44527a..7c6210ead16c22 100644 --- a/messaging/msgq.cc +++ b/messaging/msgq.cc @@ -90,8 +90,10 @@ int msgq_new_queue(msgq_queue_t * q, const char * path, size_t size){ auto fd = open(full_path, O_RDWR | O_CREAT, 0777); delete[] full_path; - if (fd < 0) + if (fd < 0) { + std::cout << "Warning, could not open: " << full_path << std::endl; return -1; + } int rc = ftruncate(fd, size + sizeof(msgq_header_t)); if (rc < 0) diff --git a/messaging/socketmaster.cc b/messaging/socketmaster.cc new file mode 100644 index 00000000000000..66b7696cf299dc --- /dev/null +++ b/messaging/socketmaster.cc @@ -0,0 +1,162 @@ +#include +#include +#include "messaging.hpp" +#include "services.h" +#ifdef __APPLE__ +#define CLOCK_BOOTTIME CLOCK_MONOTONIC +#endif +static inline uint64_t nanos_since_boot() { + struct timespec t; + clock_gettime(CLOCK_BOOTTIME, &t); + return t.tv_sec * 1000000000ULL + t.tv_nsec; +} +static const service *get_service(const char *name) { + for (const auto &it : services) { + if (strcmp(it.name, name) == 0) return ⁢ + } + return nullptr; +} +static inline bool inList(const std::initializer_list &list, const char *value) { + for (auto &v : list) { + if (strcmp(value, v) == 0) return true; + } + return false; +} + +class MessageContext { + public: + MessageContext() { ctx_ = Context::create(); } + ~MessageContext() { delete ctx_; } + Context *ctx_; +}; +MessageContext ctx; + +struct SubMaster::SubMessage { + std::string name; + SubSocket *socket = nullptr; + int freq = 0; + bool updated = false, alive = false, valid = false, ignore_alive; + uint64_t rcv_time = 0, rcv_frame = 0; + void *allocated_msg_reader = nullptr; + capnp::FlatArrayMessageReader *msg_reader = nullptr; + kj::Array buf; + cereal::Event::Reader event; +}; + +SubMaster::SubMaster(const std::initializer_list &service_list, const char *address, + const std::initializer_list &ignore_alive) { + poller_ = Poller::create(); + for (auto name : service_list) { + const service *serv = get_service(name); + assert(serv != nullptr); + SubSocket *socket = SubSocket::create(ctx.ctx_, name, address ? address : "127.0.0.1", true); + assert(socket != 0); + poller_->registerSocket(socket); + SubMessage *m = new SubMessage{ + .socket = socket, + .freq = serv->frequency, + .ignore_alive = inList(ignore_alive, name), + .allocated_msg_reader = malloc(sizeof(capnp::FlatArrayMessageReader)), + .buf = kj::heapArray(1024)}; + messages_[socket] = m; + services_[name] = m; + } +} + +int SubMaster::update(int timeout) { + if (++frame_ == UINT64_MAX) frame_ = 1; + for (auto &kv : messages_) kv.second->updated = false; + + int updated = 0; + auto sockets = poller_->poll(timeout); + uint64_t current_time = nanos_since_boot(); + for (auto s : sockets) { + Message *msg = s->receive(true); + if (msg == nullptr) continue; + + SubMessage *m = messages_.at(s); + const size_t size = (msg->getSize() / sizeof(capnp::word)) + 1; + if (m->buf.size() < size) { + m->buf = kj::heapArray(size); + } + memcpy(m->buf.begin(), msg->getData(), msg->getSize()); + delete msg; + + if (m->msg_reader) { + m->msg_reader->~FlatArrayMessageReader(); + } + m->msg_reader = new (m->allocated_msg_reader) capnp::FlatArrayMessageReader(kj::ArrayPtr(m->buf.begin(), size)); + m->event = m->msg_reader->getRoot(); + m->updated = true; + m->rcv_time = current_time; + m->rcv_frame = frame_; + m->valid = m->event.getValid(); + + ++updated; + } + + for (auto &kv : messages_) { + SubMessage *m = kv.second; + m->alive = (m->freq <= (1e-5) || ((current_time - m->rcv_time) * (1e-9)) < (10.0 / m->freq)); + } + return updated; +} + +bool SubMaster::all_(const std::initializer_list &service_list, bool valid, bool alive) { + int found = 0; + for (auto &kv : messages_) { + SubMessage *m = kv.second; + if (service_list.size() == 0 || inList(service_list, m->name.c_str())) { + found += (!valid || m->valid) && (!alive || (m->alive && !m->ignore_alive)); + } + } + return service_list.size() == 0 ? found == messages_.size() : found == service_list.size(); +} + +void SubMaster::drain() { + while (true) { + auto polls = poller_->poll(0); + if (polls.size() == 0) + break; + + for (auto sock : polls) { + Message *msg = sock->receive(true); + delete msg; + } + } +} + +bool SubMaster::updated(const char *name) const { return services_.at(name)->updated; } +cereal::Event::Reader &SubMaster::operator[](const char *name) { return services_.at(name)->event; }; + +SubMaster::~SubMaster() { + delete poller_; + for (auto &kv : messages_) { + SubMessage *m = kv.second; + if (m->msg_reader) { + m->msg_reader->~FlatArrayMessageReader(); + } + free(m->allocated_msg_reader); + delete m->socket; + delete m; + } +} + +PubMaster::PubMaster(const std::initializer_list &service_list) { + for (auto name : service_list) { + assert(get_service(name) != nullptr); + PubSocket *socket = PubSocket::create(ctx.ctx_, name); + assert(socket); + sockets_[name] = socket; + } +} + +int PubMaster::send(const char *name, capnp::MessageBuilder &msg) { + auto words = capnp::messageToFlatArray(msg); + auto bytes = words.asBytes(); + return send(name, bytes.begin(), bytes.size()); +} + +PubMaster::~PubMaster() { + for (auto s : sockets_) delete s.second; +}