From d432eb0f16071e94d9f2c1b84dab9a256f8e61ca Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 17 Aug 2024 10:47:05 -0700 Subject: [PATCH 01/13] cleanup dependencies --- .gitignore | 1 + Dockerfile | 14 +------------- pyproject.toml | 25 +++++++++++++++++++++++++ requirements.txt | 16 ---------------- 4 files changed, 27 insertions(+), 29 deletions(-) delete mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index 2f57ce978b..2e54c43c6f 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ .DS_Store .sconsign.dblite .hypothesis +*.egg-info/ opendbc/can/*.so opendbc/can/*.a diff --git a/Dockerfile b/Dockerfile index 7cbf7c522d..ffedeb2353 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,16 +5,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ autoconf \ build-essential \ ca-certificates \ - capnproto \ clang \ cppcheck \ curl \ git \ libtool \ make \ - libbz2-dev \ libffi-dev \ - libcapnp-dev \ liblzma-dev \ libncurses5-dev \ libncursesw5-dev \ @@ -23,25 +20,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libsqlite3-dev \ libzmq3-dev \ llvm \ - ocl-icd-opencl-dev \ - opencl-headers \ - tk-dev \ python3-pip \ python3-dev \ python3-openssl \ - python-is-python3 \ - xz-utils \ zlib1g-dev \ - cmake \ && rm -rf /var/lib/apt/lists/* -COPY requirements.txt /tmp/ -RUN pip3 install --break-system-packages --no-cache-dir -r /tmp/requirements.txt -RUN pip3 install --break-system-packages --no-cache-dir pre-commit==2.15.0 pylint==2.17.4 - WORKDIR /project/opendbc ENV PYTHONPATH=/project/opendbc COPY . . +RUN pip3 install --break-system-packages --no-cache-dir . RUN ls && rm -rf .git && \ scons -c && scons -j$(nproc) \ diff --git a/pyproject.toml b/pyproject.toml index f7925c8162..aa552a8b57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,31 @@ requires-python = ">=3.9" urls = { "homepage" = "https://github.com/commaai/opendbc" } +dependencies = [ + "scons", + "pyyaml", + "numpy", + "Cython", + "crcmod", + "pandacan@git+https://github.com/commaai/panda.git@master", +] + +[project.optional-dependencies] +testing = [ + "ruff", + "pytest", + "pytest-mock", + "pytest-xdist", + "pytest-subtests", + "hypothesis==6.47.*", + "parameterized>=0.8,<0.9", + "pre-commit", +] +docs = [ + "Jinja2", + "natsort", +] + [tool.pytest.ini_options] addopts = "--ignore=panda/ -Werror --strict-config --strict-markers --durations=10 -n auto" python_files = "test_*.py" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index b3c3d9f9ab..0000000000 --- a/requirements.txt +++ /dev/null @@ -1,16 +0,0 @@ -ruff -Cython -Jinja2 -numpy -pycapnp -pyyaml -scons -pytest -pytest-xdist -pytest-subtests -pytest-mock -hypothesis==6.47.* -crcmod -natsort -parameterized>=0.8,<0.9 -git+https://github.com/commaai/panda.git From 5a633fe258e338edf4b67a0ecf4aea37f6d30317 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 17 Aug 2024 10:53:40 -0700 Subject: [PATCH 02/13] all --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ffedeb2353..5be22d7cea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,6 @@ WORKDIR /project/opendbc ENV PYTHONPATH=/project/opendbc COPY . . -RUN pip3 install --break-system-packages --no-cache-dir . +RUN pip3 install --break-system-packages --no-cache-dir .[all] RUN ls && rm -rf .git && \ scons -c && scons -j$(nproc) \ From 80a8a17476e757622d031f183b9140ed543488fc Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 17 Aug 2024 10:59:45 -0700 Subject: [PATCH 03/13] install --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5be22d7cea..3fce940d52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,6 @@ WORKDIR /project/opendbc ENV PYTHONPATH=/project/opendbc COPY . . -RUN pip3 install --break-system-packages --no-cache-dir .[all] +RUN pip3 install --break-system-packages --no-cache-dir .[testing,docs] RUN ls && rm -rf .git && \ scons -c && scons -j$(nproc) \ From 0414b0f3eaa9cdb52cc0749c73cb7e9bb721536e Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 17 Aug 2024 11:05:11 -0700 Subject: [PATCH 04/13] less --- Dockerfile | 15 --------------- SConstruct | 3 +-- opendbc/can/SConscript | 2 +- 3 files changed, 2 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3fce940d52..e762f62634 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,28 +2,13 @@ FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ - autoconf \ build-essential \ - ca-certificates \ clang \ cppcheck \ curl \ git \ - libtool \ - make \ - libffi-dev \ - liblzma-dev \ - libncurses5-dev \ - libncursesw5-dev \ - libreadline-dev \ - libssl-dev \ - libsqlite3-dev \ - libzmq3-dev \ - llvm \ python3-pip \ python3-dev \ - python3-openssl \ - zlib1g-dev \ && rm -rf /var/lib/apt/lists/* WORKDIR /project/opendbc diff --git a/SConstruct b/SConstruct index 2401720f17..ac87bbb9db 100644 --- a/SConstruct +++ b/SConstruct @@ -3,7 +3,6 @@ import subprocess import sysconfig import numpy as np -zmq = 'zmq' arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip() python_path = sysconfig.get_paths()['include'] @@ -52,7 +51,7 @@ env = Environment( ) common = '' -Export('env', 'zmq', 'arch', 'common') +Export('env', 'arch', 'common') envCython = env.Clone() envCython["CPPPATH"] += [np.get_include()] diff --git a/opendbc/can/SConscript b/opendbc/can/SConscript index 1c1629d584..9f9aa291c5 100644 --- a/opendbc/can/SConscript +++ b/opendbc/can/SConscript @@ -6,7 +6,7 @@ envDBC = env.Clone() dbc_file_path = '-DDBC_FILE_PATH=\'"%s"\'' % (envDBC.Dir("../dbc").abspath) envDBC['CXXFLAGS'] += [dbc_file_path] src = ["dbc.cc", "parser.cc", "packer.cc", "common.cc"] -libs = [common, "zmq"] +libs = [common, ] # shared library for openpilot LINKFLAGS = envDBC["LINKFLAGS"] From 26e3d066dd1365e2a9bd0b341ddd1d62145ee489 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 17 Aug 2024 11:14:30 -0700 Subject: [PATCH 05/13] try this --- Dockerfile | 6 +----- SConstruct | 1 + 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index e762f62634..5067a77775 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,10 @@ -FROM ubuntu:24.04 +FROM python:3.12 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ clang \ cppcheck \ - curl \ - git \ - python3-pip \ - python3-dev \ && rm -rf /var/lib/apt/lists/* WORKDIR /project/opendbc diff --git a/SConstruct b/SConstruct index ac87bbb9db..69aa4ec379 100644 --- a/SConstruct +++ b/SConstruct @@ -37,6 +37,7 @@ env = Environment( "-Werror", "-Wshadow", "-Wno-vla-cxx-extension", + "-Wno-unknown-warning-option", # for compatibility across clang versions ] + ccflags_asan, LDFLAGS=ldflags_asan, LINKFLAGS=ldflags_asan, From c186739545384817df00eb1d3c4252d01080f506 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 17 Aug 2024 11:16:30 -0700 Subject: [PATCH 06/13] slimmm --- Dockerfile | 7 ------- SConstruct | 6 +++--- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5067a77775..8ca88b63d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,5 @@ FROM python:3.12 -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - clang \ - cppcheck \ - && rm -rf /var/lib/apt/lists/* - WORKDIR /project/opendbc ENV PYTHONPATH=/project/opendbc diff --git a/SConstruct b/SConstruct index 69aa4ec379..932ff26e8b 100644 --- a/SConstruct +++ b/SConstruct @@ -27,8 +27,8 @@ ldflags_asan = ["-fsanitize=address"] if GetOption('asan') else [] env = Environment( ENV=os.environ, - CC='clang', - CXX='clang++', + CC='gcc', + CXX='g++', CCFLAGS=[ "-g", "-fPIC", @@ -37,7 +37,7 @@ env = Environment( "-Werror", "-Wshadow", "-Wno-vla-cxx-extension", - "-Wno-unknown-warning-option", # for compatibility across clang versions + "-Wno-unknown-warning-option", # for compatibility across compiler versions ] + ccflags_asan, LDFLAGS=ldflags_asan, LINKFLAGS=ldflags_asan, From 6635684f743f0222c081eadca310eed5b89ef3ee Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 17 Aug 2024 11:18:30 -0700 Subject: [PATCH 07/13] skip docker --- .github/workflows/tests.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 93fbbb4c68..26c42440c9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,8 +7,6 @@ on: pull_request: env: - RUN: docker run -e PYTHONWARNINGS="error,default::DeprecationWarning" --shm-size 1G --rm opendbc /bin/bash -c - BUILD: docker buildx build --pull --load --cache-to type=inline --cache-from type=registry,ref=ghcr.io/commaai/opendbc:latest -t opendbc -f Dockerfile . PYTHONWARNINGS: error jobs: @@ -22,10 +20,12 @@ jobs: # run: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] steps: - uses: actions/checkout@v4 - - name: Build Docker image - run: eval "$BUILD" - - name: Unit tests - run: ${{ env.RUN }} "pytest -n logical --durations=0" + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - run: pip install -e . + - run: scons -j$(nproc) + - run: pytest -n logical --durations=0 static-analysis: name: static analysis From 71669a4040963e5b36d4ee9c2fa76931a53d0fd6 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 17 Aug 2024 11:20:40 -0700 Subject: [PATCH 08/13] silence that one --- .github/workflows/tests.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 26c42440c9..2be4ad66b4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,7 +7,7 @@ on: pull_request: env: - PYTHONWARNINGS: error + PYTHONWARNINGS: error,default::DeprecationWarning jobs: unit-tests: @@ -31,9 +31,6 @@ jobs: name: static analysis runs-on: ubuntu-latest timeout-minutes: 1 - env: - # package install has DeprecationWarnings - PYTHONWARNINGS: default steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 From 15851574fbf3a47eaea52ffce6d0e5c55333b1d8 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 17 Aug 2024 11:22:37 -0700 Subject: [PATCH 09/13] disable for now --- .github/workflows/tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2be4ad66b4..5ca737b6c8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,8 +6,8 @@ on: - master pull_request: -env: - PYTHONWARNINGS: error,default::DeprecationWarning +#env: +# PYTHONWARNINGS: error,default::DeprecationWarning jobs: unit-tests: @@ -23,7 +23,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.11' - - run: pip install -e . + - run: pip install .[testing] - run: scons -j$(nproc) - run: pytest -n logical --durations=0 @@ -37,5 +37,5 @@ jobs: with: python-version: '3.11' - run: sudo apt install --no-install-recommends -y cppcheck - - run: pip install -e . + - run: pip install . - uses: pre-commit/action@v3.0.1 From d039a04c52892c13f95b24575813e90d46d7d982 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 17 Aug 2024 11:25:20 -0700 Subject: [PATCH 10/13] e --- .github/workflows/tests.yml | 4 ++-- Dockerfile | 9 --------- 2 files changed, 2 insertions(+), 11 deletions(-) delete mode 100644 Dockerfile diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5ca737b6c8..d6c31c0015 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.11' - - run: pip install .[testing] + - run: pip install -e .[testing] - run: scons -j$(nproc) - run: pytest -n logical --durations=0 @@ -37,5 +37,5 @@ jobs: with: python-version: '3.11' - run: sudo apt install --no-install-recommends -y cppcheck - - run: pip install . + - run: pip install -e . - uses: pre-commit/action@v3.0.1 diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 8ca88b63d4..0000000000 --- a/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM python:3.12 - -WORKDIR /project/opendbc -ENV PYTHONPATH=/project/opendbc - -COPY . . -RUN pip3 install --break-system-packages --no-cache-dir .[testing,docs] -RUN ls && rm -rf .git && \ - scons -c && scons -j$(nproc) \ From 146c03d51ea8430dd71921f35431397f8fc6455e Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 17 Aug 2024 12:00:17 -0700 Subject: [PATCH 11/13] fix build --- .github/workflows/tests.yml | 4 +++- .gitignore | 1 + SConstruct | 2 +- opendbc/can/SConscript | 16 +++++----------- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d6c31c0015..efea216422 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -22,8 +22,9 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: + cache: 'pip' python-version: '3.11' - - run: pip install -e .[testing] + - run: pip install -e .[testing,docs] - run: scons -j$(nproc) - run: pytest -n logical --durations=0 @@ -35,6 +36,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: + cache: 'pip' python-version: '3.11' - run: sudo apt install --no-install-recommends -y cppcheck - run: pip install -e . diff --git a/.gitignore b/.gitignore index 2e54c43c6f..bfe2238555 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +/build/ .mypy_cache/ *.pyc *.os diff --git a/SConstruct b/SConstruct index 932ff26e8b..4727dd4488 100644 --- a/SConstruct +++ b/SConstruct @@ -67,7 +67,7 @@ elif arch == "aarch64": python_libs.append(os.path.basename(python_path)) else: - envCython["LINKFLAGS"] = ["-pthread", "-shared"] + envCython["LINKFLAGS"] = ["-pthread", "-shared", "-lz"] envCython["LIBS"] = python_libs diff --git a/opendbc/can/SConscript b/opendbc/can/SConscript index 9f9aa291c5..ab2fbd1d3a 100644 --- a/opendbc/can/SConscript +++ b/opendbc/can/SConscript @@ -1,30 +1,24 @@ Import('env', 'envCython', 'common', 'arch') -import os - envDBC = env.Clone() dbc_file_path = '-DDBC_FILE_PATH=\'"%s"\'' % (envDBC.Dir("../dbc").abspath) envDBC['CXXFLAGS'] += [dbc_file_path] src = ["dbc.cc", "parser.cc", "packer.cc", "common.cc"] -libs = [common, ] # shared library for openpilot LINKFLAGS = envDBC["LINKFLAGS"] if arch == "Darwin": LINKFLAGS += ["-Wl,-install_name,@loader_path/libdbc.dylib"] -libdbc = envDBC.SharedLibrary('libdbc', src, LIBS=libs, LINKFLAGS=LINKFLAGS) +libdbc = envDBC.SharedLibrary('libdbc', src, LIBS=[common, ], LINKFLAGS=LINKFLAGS) # static library for tools like cabana -envDBC.Library('libdbc_static', src, LIBS=libs) +envDBC.Library('libdbc_static', src, LIBS=[common, ]) # Build packer and parser lenv = envCython.Clone() -lenv["LINKFLAGS"] += [libdbc[0].get_labspath()] -parser = lenv.Program('parser_pyx.so', 'parser_pyx.pyx') -packer = lenv.Program('packer_pyx.so', 'packer_pyx.pyx') - -lenv.Depends(parser, libdbc) -lenv.Depends(packer, libdbc) +lenv["RPATH"] = [libdbc[0].dir.abspath, ] +parser = lenv.Program('parser_pyx.so', 'parser_pyx.pyx', LIBS=[common, libdbc]) +packer = lenv.Program('packer_pyx.so', 'packer_pyx.pyx', LIBS=[common, libdbc]) opendbc_python = Alias("opendbc_python", [parser, packer]) From aa747b54b2e8ce955fd33a6eaeb392cc61c1ab20 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 17 Aug 2024 12:01:47 -0700 Subject: [PATCH 12/13] bring that back --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index efea216422..751ceff448 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,8 +6,8 @@ on: - master pull_request: -#env: -# PYTHONWARNINGS: error,default::DeprecationWarning +env: + PYTHONWARNINGS: error,default::DeprecationWarning jobs: unit-tests: From 01aef2195e038f63929a4e60838a43adc0165a02 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 17 Aug 2024 12:03:01 -0700 Subject: [PATCH 13/13] not ready yet --- .github/workflows/tests.yml | 5 +++-- SConstruct | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 751ceff448..c1f2b1e6a6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,8 +6,9 @@ on: - master pull_request: -env: - PYTHONWARNINGS: error,default::DeprecationWarning +# TODO: enable this +#env: +# PYTHONWARNINGS: error,default::DeprecationWarning jobs: unit-tests: diff --git a/SConstruct b/SConstruct index 4727dd4488..932ff26e8b 100644 --- a/SConstruct +++ b/SConstruct @@ -67,7 +67,7 @@ elif arch == "aarch64": python_libs.append(os.path.basename(python_path)) else: - envCython["LINKFLAGS"] = ["-pthread", "-shared", "-lz"] + envCython["LINKFLAGS"] = ["-pthread", "-shared"] envCython["LIBS"] = python_libs