Squashed 'opendbc/' changes from c0eba096..4f82d01e

4f82d01e gitignore
5cb83454 Honda FCM: diagnostic signals
d309cdce Added linter to opendbc (#203)
d452706f add requirements.txt
ec3b4595 deterministic dependency order
a265d351 Azure pipelines ci (#202)
bce9a2e1 packer depends on libdbc
5d5fdd6a no more python version of libdbc, everything through cython
541705bf move CANDefine to parser code
da25c52a add test for can define
0ba7926b unify can packer and parser
25d88009 consistent naming
a5c640a5 fix linter
be210fef remove obsolete make file
ffd9dca7 opendbc needs cereal
b559f63d remove more make
d0929496 seems to work now
41e80836 don't make
3254d1fc think scons works
eb78f6aa scons sort of working
0ef1e35d fix gitignore
e155e017 Can migration (#199)
3eded83a Honda: correct steering torque sensor sign to be consistent with standard convention (left+)
32f70e2f Fix outback endianness consistency (#196)
a7da471f Update subaru_outback_2015_eyesight.dbc (#195)

git-subtree-dir: opendbc
git-subtree-split: 4f82d01ebc78109888954d9807d320e3c27896fd

.gitignore

@@ -1,2 +1,10 @@
 *.pyc
+*.os
+*.tmp
 .*.swp
+can/*.so
+can/build/
+can/obj/
+can/packer_pyx.cpp
+can/parser_pyx.cpp
+can/packer_impl.cpp

Dockerfile

@@ -0,0 +1,25 @@
+from ubuntu:16.04
+
+RUN apt-get update && apt-get install -y libzmq3-dev clang wget git autoconf libtool curl make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python-openssl
+
+RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
+ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}"
+RUN pyenv install 3.7.3
+RUN pyenv global 3.7.3
+RUN pyenv rehash
+
+COPY requirements.txt /tmp/
+RUN pip install -r /tmp/requirements.txt
+
+ENV PYTHONPATH=/project
+
+# TODO: Add tag to cereal
+RUN git clone https://github.com/commaai/cereal.git /project/cereal
+RUN /project/cereal/install_capnp.sh
+
+WORKDIR /project
+
+COPY SConstruct .
+COPY . /project/opendbc
+
+RUN scons -c && scons -j$(nproc)

README.md

@@ -42,7 +42,7 @@ For example:
     SG_ VEHICLE_SPEED : 7|15@0+ (0.01,0) [0|250] "kph" PCM
     ```
 
-- Signal's size: always use the smallest amount of bits possible. For example, let's say I'm reverse engineering the gas pedal position and I've determined that it's in a 3 bytes message. For 0% pedal position I read a message value of `0x00 0x00 0x00`, while for 100% of pedal position I read `0x64 0x00 0x00`: clearly, the gas pedal position is within the first byte of the message and I might be tempted to define the signal `GAS_POS` as:
+- Signal size: always use the smallest amount of bits possible. For example, let's say I'm reverse engineering the gas pedal position and I've determined that it's in a 3 bytes message. For 0% pedal position I read a message value of `0x00 0x00 0x00`, while for 100% of pedal position I read `0x64 0x00 0x00`: clearly, the gas pedal position is within the first byte of the message and I might be tempted to define the signal `GAS_POS` as:
     ```
     SG_ GAS_POS : 7|8@0+ (1,0) [0|100] "%" PCM
     ```

SConstruct

@@ -0,0 +1,57 @@
+import os
+import subprocess
+
+zmq = 'zmq'
+arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
+
+cereal_dir = Dir('.')
+
+cpppath = [
+    '#',
+    '#cereal',
+    "#cereal/messaging",
+    "#opendbc/can",
+    '/usr/lib/include',
+]
+
+AddOption('--test',
+          action='store_true',
+          help='build test files')
+
+AddOption('--asan',
+          action='store_true',
+          help='turn on ASAN')
+
+ccflags_asan = ["-fsanitize=address", "-fno-omit-frame-pointer"] if GetOption('asan') else []
+ldflags_asan = ["-fsanitize=address"] if GetOption('asan') else []
+
+env = Environment(
+  ENV=os.environ,
+  CC='clang',
+  CXX='clang++',
+  CCFLAGS=[
+    "-g",
+    "-fPIC",
+    "-O2",
+    "-Werror=implicit-function-declaration",
+    "-Werror=incompatible-pointer-types",
+    "-Werror=int-conversion",
+    "-Werror=return-type",
+    "-Werror=format-extra-args",
+  ] + ccflags_asan,
+  LDFLAGS=ldflags_asan,
+  LINKFLAGS=ldflags_asan,
+
+  CFLAGS="-std=gnu11",
+  CXXFLAGS="-std=c++14",
+  CPPPATH=cpppath,
+)
+
+Export('env', 'zmq', 'arch')
+
+cereal = [File('#cereal/libcereal.a')]
+messaging = [File('#cereal/libmessaging.a')]
+Export('cereal', 'messaging')
+
+SConscript(['cereal/SConscript'])
+SConscript(['opendbc/can/SConscript'])

acura_ilx_2016_can_generated.dbc

@@ -271,8 +271,8 @@ BO_ 342 STEERING_SENSORS: 6 EPS
  SG_ CHECKSUM : 43|4@0+ (1,0) [0|3] "" EON
 
 BO_ 399 STEER_STATUS: 7 EPS
- SG_ STEER_TORQUE_SENSOR : 7|16@0- (1,0) [-31000|31000] "tbd" EON
- SG_ STEER_TORQUE_MOTOR : 23|16@0- (1,0) [-31000|31000] "tbd" EON
+ SG_ STEER_TORQUE_SENSOR : 7|16@0- (-1,0) [-31000|31000] "tbd" EON
+ SG_ STEER_TORQUE_MOTOR : 23|16@0- (-1,0) [-31000|31000] "tbd" EON
  SG_ STEER_STATUS : 39|4@0+ (1,0) [0|15] "" EON
  SG_ STEER_CONTROL_ACTIVE : 35|1@0+ (1,0) [0|1] "" EON
  SG_ COUNTER : 53|2@0+ (1,0) [0|3] "" EON

acura_rdx_2018_can_generated.dbc

@@ -259,8 +259,8 @@ BO_ 392 GEARBOX: 6 XXX
  SG_ GEAR : 36|5@0+ (1,0) [0|31] "" EON
 
 BO_ 399 STEER_STATUS: 6 EPS
- SG_ STEER_TORQUE_SENSOR : 7|12@0- (1,0) [-2047.5|2047.5] "tbd" EON
- SG_ STEER_TORQUE_MOTOR : 23|16@0- (1,0) [-31000|31000] "tbd" EON
+ SG_ STEER_TORQUE_SENSOR : 7|12@0- (-1,0) [-2047.5|2047.5] "tbd" EON
+ SG_ STEER_TORQUE_MOTOR : 23|16@0- (-1,0) [-31000|31000] "tbd" EON
  SG_ STEER_STATUS : 35|4@0+ (1,0) [0|15] "" EON
  SG_ STEER_CONTROL_ACTIVE : 36|1@0+ (1,0) [0|1] "" EON
  SG_ COUNTER : 45|2@0+ (1,0) [0|3] "" EON

azure-pipelines.yml

@@ -0,0 +1,16 @@
+pr: none
+
+pool:
+  vmImage: 'ubuntu-16.04'
+steps:
+- script: |
+    set -e
+    docker build -t opendbc .
+  displayName: 'Build'
+- script: |
+    docker run opendbc bash -c "python -m unittest discover opendbc"
+  displayName: 'Unit tests'
+- script: |
+    docker run opendbc bash -c "cd opendbc/can/tests/linter_python; PYTHONPATH=/ ./flake8_opendbc.sh"
+    docker run opendbc bash -c "cd opendbc/can/tests/linter_python; PYTHONPATH=/ ./pylint_opendbc.sh"
+  displayName: 'Python linter'

can/SConscript

@@ -0,0 +1,27 @@
+Import('env', 'cereal')
+
+import os
+from opendbc.can.process_dbc import process
+
+dbcs = []
+for x in sorted(os.listdir('../')):
+  if x.endswith(".dbc"):
+    def compile_dbc(target, source, env):
+      process(source[0].path, target[0].path)
+    in_fn = [os.path.join('../', x), 'dbc_template.cc']
+    out_fn = os.path.join('dbc_out', x.replace(".dbc", ".cc"))
+    dbc = env.Command(out_fn, in_fn, compile_dbc)
+    dbcs.append(dbc)
+
+
+libdbc = env.SharedLibrary('libdbc', ["dbc.cc", "parser.cc", "packer.cc", "common.cc"]+dbcs, LIBS=["capnp", "kj"])
+
+# packer
+env.Command(['packer_pyx.so'],
+  [libdbc, 'packer_pyx.pyx', 'packer_pyx_setup.py'],
+  "cd opendbc/can && python3 packer_pyx_setup.py build_ext --inplace")
+
+# parser
+env.Command(['parser_pyx.so'],
+  [libdbc, cereal, 'parser_pyx_setup.py', 'parser_pyx.pyx', 'common.pxd'],
+  "cd opendbc/can && python3 parser_pyx_setup.py build_ext --inplace")

can/can_define.py

@@ -0,0 +1,2 @@
+from opendbc.can.parser_pyx import CANDefine # pylint: disable=no-name-in-module, import-error
+assert CANDefine

can/common.cc

@@ -0,0 +1,165 @@
+#include "common.h"
+
+unsigned int honda_checksum(unsigned int address, uint64_t d, int l) {
+  d >>= ((8-l)*8); // remove padding
+  d >>= 4; // remove checksum
+
+  int s = 0;
+  while (address) { s += (address & 0xF); address >>= 4; }
+  while (d) { s += (d & 0xF); d >>= 4; }
+  s = 8-s;
+  s &= 0xF;
+
+  return s;
+}
+
+unsigned int toyota_checksum(unsigned int address, uint64_t d, int l) {
+  d >>= ((8-l)*8); // remove padding
+  d >>= 8; // remove checksum
+
+  unsigned int s = l;
+  while (address) { s += address & 0xff; address >>= 8; }
+  while (d) { s += d & 0xff; d >>= 8; }
+
+  return s & 0xFF;
+}
+
+// Static lookup table for fast computation of CRC8 poly 0x2F, aka 8H2F/AUTOSAR
+uint8_t crc8_lut_8h2f[256];
+
+void gen_crc_lookup_table(uint8_t poly, uint8_t crc_lut[]) {
+  uint8_t crc;
+  int i, j;
+
+   for (i = 0; i < 256; i++) {
+    crc = i;
+    for (j = 0; j < 8; j++) {
+      if ((crc & 0x80) != 0)
+        crc = (uint8_t)((crc << 1) ^ poly);
+      else
+        crc <<= 1;
+    }
+    crc_lut[i] = crc;
+  }
+}
+
+void init_crc_lookup_tables() {
+  // At init time, set up static lookup tables for fast CRC computation.
+
+  gen_crc_lookup_table(0x2F, crc8_lut_8h2f);    // CRC-8 8H2F/AUTOSAR for Volkswagen
+}
+
+unsigned int volkswagen_crc(unsigned int address, uint64_t d, int l) {
+  // Volkswagen uses standard CRC8 8H2F/AUTOSAR, but they compute it with
+  // a magic variable padding byte tacked onto the end of the payload.
+  // https://www.autosar.org/fileadmin/user_upload/standards/classic/4-3/AUTOSAR_SWS_CRCLibrary.pdf
+
+  uint8_t *dat = (uint8_t *)&d;
+  uint8_t crc = 0xFF; // Standard init value for CRC8 8H2F/AUTOSAR
+
+  // CRC the payload first, skipping over the first byte where the CRC lives.
+  for (int i = 1; i < l; i++) {
+    crc ^= dat[i];
+    crc = crc8_lut_8h2f[crc];
+  }
+
+  // Look up and apply the magic final CRC padding byte, which permutes by CAN
+  // address, and additionally (for SOME addresses) by the message counter.
+  uint8_t counter = dat[1] & 0x0F;
+  switch(address) {
+    case 0x86:  // LWI_01 Steering Angle
+      crc ^= (uint8_t[]){0x86,0x86,0x86,0x86,0x86,0x86,0x86,0x86,0x86,0x86,0x86,0x86,0x86,0x86,0x86,0x86}[counter];
+      break;
+    case 0x9F:  // EPS_01 Electric Power Steering
+      crc ^= (uint8_t[]){0xF5,0xF5,0xF5,0xF5,0xF5,0xF5,0xF5,0xF5,0xF5,0xF5,0xF5,0xF5,0xF5,0xF5,0xF5,0xF5}[counter];
+      break;
+    case 0xAD:  // Getriebe_11 Automatic Gearbox
+      crc ^= (uint8_t[]){0x3F,0x69,0x39,0xDC,0x94,0xF9,0x14,0x64,0xD8,0x6A,0x34,0xCE,0xA2,0x55,0xB5,0x2C}[counter];
+      break;
+    case 0xFD:  // ESP_21 Electronic Stability Program
+      crc ^= (uint8_t[]){0xB4,0xEF,0xF8,0x49,0x1E,0xE5,0xC2,0xC0,0x97,0x19,0x3C,0xC9,0xF1,0x98,0xD6,0x61}[counter];
+      break;
+    case 0x106: // ESP_05 Electronic Stability Program
+      crc ^= (uint8_t[]){0x07,0x07,0x07,0x07,0x07,0x07,0x07,0x07,0x07,0x07,0x07,0x07,0x07,0x07,0x07,0x07}[counter];
+      break;
+    case 0x117: // ACC_10 Automatic Cruise Control
+      crc ^= (uint8_t[]){0xAC,0xAC,0xAC,0xAC,0xAC,0xAC,0xAC,0xAC,0xAC,0xAC,0xAC,0xAC,0xAC,0xAC,0xAC,0xAC}[counter];
+      break;
+    case 0x122: // ACC_06 Automatic Cruise Control
+      crc ^= (uint8_t[]){0x37,0x7D,0xF3,0xA9,0x18,0x46,0x6D,0x4D,0x3D,0x71,0x92,0x9C,0xE5,0x32,0x10,0xB9}[counter];
+      break;
+    case 0x126: // HCA_01 Heading Control Assist
+      crc ^= (uint8_t[]){0xDA,0xDA,0xDA,0xDA,0xDA,0xDA,0xDA,0xDA,0xDA,0xDA,0xDA,0xDA,0xDA,0xDA,0xDA,0xDA}[counter];
+      break;
+    case 0x12B: // GRA_ACC_01 Steering wheel controls for ACC
+      crc ^= (uint8_t[]){0x6A,0x38,0xB4,0x27,0x22,0xEF,0xE1,0xBB,0xF8,0x80,0x84,0x49,0xC7,0x9E,0x1E,0x2B}[counter];
+      break;
+    case 0x187: // EV_Gearshift "Gear" selection data for EVs with no gearbox
+      crc ^= (uint8_t[]){0x7F,0xED,0x17,0xC2,0x7C,0xEB,0x44,0x21,0x01,0xFA,0xDB,0x15,0x4A,0x6B,0x23,0x05}[counter];
+      break;
+    case 0x30C: // ACC_02 Automatic Cruise Control
+      crc ^= (uint8_t[]){0x0F,0x0F,0x0F,0x0F,0x0F,0x0F,0x0F,0x0F,0x0F,0x0F,0x0F,0x0F,0x0F,0x0F,0x0F,0x0F}[counter];
+      break;
+    case 0x3C0: // Klemmen_Status_01 ignition and starting status
+      crc ^= (uint8_t[]){0xC3,0xC3,0xC3,0xC3,0xC3,0xC3,0xC3,0xC3,0xC3,0xC3,0xC3,0xC3,0xC3,0xC3,0xC3,0xC3}[counter];
+      break;
+    case 0x65D: // ESP_20 Electronic Stability Program
+      crc ^= (uint8_t[]){0xAC,0xB3,0xAB,0xEB,0x7A,0xE1,0x3B,0xF7,0x73,0xBA,0x7C,0x9E,0x06,0x5F,0x02,0xD9}[counter];
+      break;
+    default:    // As-yet undefined CAN message, CRC check expected to fail
+      printf("Attempt to CRC check undefined Volkswagen message 0x%02X\n", address);
+      crc ^= (uint8_t[]){0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}[counter];
+      break;
+  }
+  crc = crc8_lut_8h2f[crc];
+
+  return crc ^ 0xFF; // Return after standard final XOR for CRC8 8H2F/AUTOSAR
+}
+
+
+unsigned int pedal_checksum(uint64_t d, int l) {
+  uint8_t crc = 0xFF;
+  uint8_t poly = 0xD5; // standard crc8
+
+  d >>= ((8-l)*8); // remove padding
+  d >>= 8; // remove checksum
+
+  uint8_t *dat = (uint8_t *)&d;
+
+  int i, j;
+  for (i = 0; i < l - 1; i++) {
+    crc ^= dat[i];
+    for (j = 0; j < 8; j++) {
+      if ((crc & 0x80) != 0) {
+        crc = (uint8_t)((crc << 1) ^ poly);
+      }
+      else {
+        crc <<= 1;
+      }
+    }
+  }
+  return crc;
+}
+
+
+uint64_t read_u64_be(const uint8_t* v) {
+  return (((uint64_t)v[0] << 56)
+          | ((uint64_t)v[1] << 48)
+          | ((uint64_t)v[2] << 40)
+          | ((uint64_t)v[3] << 32)
+          | ((uint64_t)v[4] << 24)
+          | ((uint64_t)v[5] << 16)
+          | ((uint64_t)v[6] << 8)
+          | (uint64_t)v[7]);
+}
+
+uint64_t read_u64_le(const uint8_t* v) {
+  return ((uint64_t)v[0]
+          | ((uint64_t)v[1] << 8)
+          | ((uint64_t)v[2] << 16)
+          | ((uint64_t)v[3] << 24)
+          | ((uint64_t)v[4] << 32)
+          | ((uint64_t)v[5] << 40)
+          | ((uint64_t)v[6] << 48)
+          | ((uint64_t)v[7] << 56));
+}