From c62fe447d0baaa182d81bcf0d033b363d0134be1 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sun, 18 Aug 2024 10:42:55 -0700 Subject: [PATCH 01/10] remove cereal --- Dockerfile | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1a6122ee05..0535315861 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,19 +47,15 @@ COPY tests/misra/install.sh /tmp/ RUN /tmp/install.sh && rm -rf $CPPCHECK_DIR/.git/ ENV SKIP_CPPCHECK_INSTALL=1 -ENV CEREAL_REF="861144c136c91f70dcbc652c2ffe99f57440ad47" -ENV OPENDBC_REF="8e9d3688412405154a8189c421cfdc9d5feea715" - +# TODO: this should be a "pip install" or not even in this repo at all +ENV OPENDBC_REF="74e042d4e76651d21b48db2c87c092d8855e9bdc" RUN git config --global --add safe.directory /tmp/openpilot/panda RUN mkdir -p /tmp/openpilot/ && \ cd /tmp/openpilot/ && \ - git clone --depth 1 https://github.com/commaai/cereal && \ git clone --depth 1 https://github.com/commaai/opendbc && \ - cd cereal && git fetch origin $CEREAL_REF && git checkout FETCH_HEAD && rm -rf .git/ && cd .. && \ - cd opendbc && git fetch origin $OPENDBC_REF && git checkout FETCH_HEAD && rm -rf .git/ && cd .. && \ - cp -pR opendbc/SConstruct opendbc/site_scons/ . && \ - pip3 install --break-system-packages --no-cache-dir -r opendbc/requirements.txt && \ - scons -j8 --minimal opendbc/ cereal/ + cd opendbc && git fetch origin $OPENDBC_REF && git checkout FETCH_HEAD && rm -rf .git/ && \ + pip3 install --break-system-packages --no-cache-dir . && \ + scons -j8 --minimal opendbc/ # for Jenkins COPY README.md panda.tar.* /tmp/ From 9c1be15a115585769d3c9633b868eda50e682a2b Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sun, 18 Aug 2024 10:47:02 -0700 Subject: [PATCH 02/10] rm more shit --- Dockerfile | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0535315861..0f6bc51ec8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,29 +5,18 @@ ENV PYTHONPATH /tmp/openpilot:$PYTHONPATH ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ - make \ - bzip2 \ ca-certificates \ - capnproto \ clang \ g++ \ gcc-arm-none-eabi libnewlib-arm-none-eabi \ git \ - libarchive-dev \ - libbz2-dev \ - libcapnp-dev \ libffi-dev \ libtool \ libusb-1.0-0 \ - libzmq3-dev \ locales \ - opencl-headers \ - ocl-icd-opencl-dev \ python3 \ python3-dev \ python3-pip \ - python-is-python3 \ - zlib1g-dev \ && rm -rf /var/lib/apt/lists/* && \ apt clean && \ cd /usr/lib/gcc/arm-none-eabi/* && \ @@ -54,7 +43,7 @@ RUN mkdir -p /tmp/openpilot/ && \ cd /tmp/openpilot/ && \ git clone --depth 1 https://github.com/commaai/opendbc && \ cd opendbc && git fetch origin $OPENDBC_REF && git checkout FETCH_HEAD && rm -rf .git/ && \ - pip3 install --break-system-packages --no-cache-dir . && \ + pip3 install --break-system-packages --no-cache-dir -e . && \ scons -j8 --minimal opendbc/ # for Jenkins From 7e0d1299518dc0ab6756e81dd21102c0f41f9eae Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sun, 18 Aug 2024 10:50:40 -0700 Subject: [PATCH 03/10] add back make --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0f6bc51ec8..fdd0f7138f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,9 @@ ENV PYTHONUNBUFFERED 1 ENV PYTHONPATH /tmp/openpilot:$PYTHONPATH ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends \ +RUN apt-get update +RUN apt-get install -y --no-install-recommends \ + make \ ca-certificates \ clang \ g++ \ From 046c3967d6ada81b5a9578e2d5f6b13b2bfc88da Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sun, 18 Aug 2024 10:55:44 -0700 Subject: [PATCH 04/10] python3 --- .github/workflows/test.yaml | 2 +- Dockerfile | 4 ---- Jenkinsfile | 2 +- tests/misra/test_misra.sh | 2 +- tests/usbprotocol/test.sh | 2 +- 5 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 97c9e07f0e..2b5736f6f3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -45,7 +45,7 @@ jobs: - name: Build Docker image run: eval "$BUILD" - name: Test python package installer - run: ${{ env.RUN }} "python setup.py install" + run: ${{ env.RUN }} "python3 setup.py install" - name: Build panda images and bootstub run: ${{ env.RUN }} "scons -j4" - name: Build panda with SPI support diff --git a/Dockerfile b/Dockerfile index fdd0f7138f..b74882f3cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,15 +7,11 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update RUN apt-get install -y --no-install-recommends \ make \ - ca-certificates \ - clang \ g++ \ gcc-arm-none-eabi libnewlib-arm-none-eabi \ git \ libffi-dev \ - libtool \ libusb-1.0-0 \ - locales \ python3 \ python3-dev \ python3-pip \ diff --git a/Jenkinsfile b/Jenkinsfile index b3b2690861..59490ee28b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -92,7 +92,7 @@ pipeline { steps { script { retry (3) { - docker_run("reset hardware", 3, "python ./tests/hitl/reset_jungles.py") + docker_run("reset hardware", 3, "python3 ./tests/hitl/reset_jungles.py") } } } diff --git a/tests/misra/test_misra.sh b/tests/misra/test_misra.sh index e65f6a5e5a..7fd110467c 100755 --- a/tests/misra/test_misra.sh +++ b/tests/misra/test_misra.sh @@ -19,7 +19,7 @@ fi # ensure checked in coverage table is up to date cd $DIR if [ -z "$SKIP_TABLES_DIFF" ]; then - python $CPPCHECK_DIR/addons/misra.py -generate-table > coverage_table + python3 $CPPCHECK_DIR/addons/misra.py -generate-table > coverage_table if ! git diff --quiet coverage_table; then echo -e "${YELLOW}MISRA coverage table doesn't match. Update and commit:${NC}" exit 3 diff --git a/tests/usbprotocol/test.sh b/tests/usbprotocol/test.sh index 8e3886da7d..b4c32166b9 100755 --- a/tests/usbprotocol/test.sh +++ b/tests/usbprotocol/test.sh @@ -4,5 +4,5 @@ set -e # Loops over all HW_TYPEs, see board/boards/board_declarations.h for hw_type in {0..7}; do echo "Testing HW_TYPE: $hw_type" - HW_TYPE=$hw_type python -m unittest discover . + HW_TYPE=$hw_type python3 -m unittest discover . done From c90ef9734af0e4e99e3eca32cfcb4da7b42f61a8 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sun, 18 Aug 2024 11:01:38 -0700 Subject: [PATCH 05/10] python3 --- .github/workflows/test.yaml | 2 +- Dockerfile | 19 +++++++------------ Jenkinsfile | 2 +- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2b5736f6f3..3fcb65e9a8 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -11,7 +11,7 @@ concurrency: cancel-in-progress: true env: - RUN: docker run -v ${{ github.workspace }}:/tmp/openpilot/panda -w /tmp/openpilot/panda --rm panda /bin/bash -c + RUN: docker run -v ${{ github.workspace }}:/tmp/pythonpath/panda -w /tmp/pythonpath/panda --rm panda /bin/bash -c BUILD: | export DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from ghcr.io/commaai/panda:latest -t panda -f Dockerfile . diff --git a/Dockerfile b/Dockerfile index b74882f3cf..93775b42d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:24.04 ENV PYTHONUNBUFFERED 1 -ENV PYTHONPATH /tmp/openpilot:$PYTHONPATH +ENV PYTHONPATH /tmp/pythonpath:$PYTHONPATH ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update @@ -21,11 +21,6 @@ RUN apt-get install -y --no-install-recommends \ rm -rf arm/ && \ rm -rf thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp -RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 - COPY requirements.txt /tmp/ RUN pip3 install --break-system-packages --no-cache-dir -r /tmp/requirements.txt @@ -36,15 +31,15 @@ ENV SKIP_CPPCHECK_INSTALL=1 # TODO: this should be a "pip install" or not even in this repo at all ENV OPENDBC_REF="74e042d4e76651d21b48db2c87c092d8855e9bdc" -RUN git config --global --add safe.directory /tmp/openpilot/panda -RUN mkdir -p /tmp/openpilot/ && \ - cd /tmp/openpilot/ && \ +RUN git config --global --add safe.directory /tmp/pythonpath/panda +RUN mkdir -p /tmp/pythonpath/ && \ + cd /tmp/pythonpath/ && \ git clone --depth 1 https://github.com/commaai/opendbc && \ cd opendbc && git fetch origin $OPENDBC_REF && git checkout FETCH_HEAD && rm -rf .git/ && \ - pip3 install --break-system-packages --no-cache-dir -e . && \ + pip3 install --break-system-packages --no-cache-dir Cython numpy && \ scons -j8 --minimal opendbc/ # for Jenkins COPY README.md panda.tar.* /tmp/ -RUN mkdir /tmp/openpilot/panda && \ - tar -xvf /tmp/panda.tar.gz -C /tmp/openpilot/panda/ || true +RUN mkdir /tmp/pythonpath/panda && \ + tar -xvf /tmp/panda.tar.gz -C /tmp/pythonpath/panda/ || true diff --git a/Jenkinsfile b/Jenkinsfile index 59490ee28b..2ccf6be116 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,7 +4,7 @@ def docker_run(String step_label, int timeout_mins, String cmd) { --env PYTHONWARNINGS=error \ --volume /dev/bus/usb:/dev/bus/usb \ --volume /var/run/dbus:/var/run/dbus \ - --workdir /tmp/openpilot/panda \ + --workdir /tmp/pythonpath/panda \ --net host \ ${env.DOCKER_IMAGE_TAG} \ bash -c 'scons -j8 && ${cmd}'", \ From 23d068ecd98ba922342677ac0585808750009a6f Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sun, 18 Aug 2024 11:07:25 -0700 Subject: [PATCH 06/10] fix that --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 93775b42d9..f47fa4a3f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,14 +30,15 @@ RUN /tmp/install.sh && rm -rf $CPPCHECK_DIR/.git/ ENV SKIP_CPPCHECK_INSTALL=1 # TODO: this should be a "pip install" or not even in this repo at all -ENV OPENDBC_REF="74e042d4e76651d21b48db2c87c092d8855e9bdc" +ENV OPENDBC_REF="d377af6c2d01b30d3de892cee91f1ed8fb50d6e8" RUN git config --global --add safe.directory /tmp/pythonpath/panda RUN mkdir -p /tmp/pythonpath/ && \ - cd /tmp/pythonpath/ && \ + cd /tmp/ && \ git clone --depth 1 https://github.com/commaai/opendbc && \ cd opendbc && git fetch origin $OPENDBC_REF && git checkout FETCH_HEAD && rm -rf .git/ && \ pip3 install --break-system-packages --no-cache-dir Cython numpy && \ - scons -j8 --minimal opendbc/ + scons -j8 --minimal opendbc/ && \ + mv opendbc/ $PYTHONPATH && rm -rf /tmp/opendbc/ # for Jenkins COPY README.md panda.tar.* /tmp/ From 96a4a0b6471558ab1725750be3f4493d7ca89c20 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sun, 18 Aug 2024 11:08:01 -0700 Subject: [PATCH 07/10] put that back --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f47fa4a3f9..c3d773b49c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,7 @@ ENV PYTHONUNBUFFERED 1 ENV PYTHONPATH /tmp/pythonpath:$PYTHONPATH ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update -RUN apt-get install -y --no-install-recommends \ +RUN apt-get update && apt-get install -y --no-install-recommends \ make \ g++ \ gcc-arm-none-eabi libnewlib-arm-none-eabi \ From efa3af80313299b2486fb758eba673a01caa5462 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sun, 18 Aug 2024 11:10:02 -0700 Subject: [PATCH 08/10] rename --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3fcb65e9a8..5b62bf18fd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -103,8 +103,8 @@ jobs: timeout-minutes: 5 run: ${{ env.RUN }} "cd tests/misra && pytest -n8 test_mutation.py" - python_linter: - name: python linter + static_analysis: + name: static analysis runs-on: ubuntu-latest timeout-minutes: 20 steps: From 149c331c884dc5911afacbaaee91114b1d67902a Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sun, 18 Aug 2024 11:17:28 -0700 Subject: [PATCH 09/10] fix that --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c3d773b49c..669317a9ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:24.04 ENV PYTHONUNBUFFERED 1 -ENV PYTHONPATH /tmp/pythonpath:$PYTHONPATH +ENV PYTHONPATH /tmp/pythonpath ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -37,7 +37,7 @@ RUN mkdir -p /tmp/pythonpath/ && \ cd opendbc && git fetch origin $OPENDBC_REF && git checkout FETCH_HEAD && rm -rf .git/ && \ pip3 install --break-system-packages --no-cache-dir Cython numpy && \ scons -j8 --minimal opendbc/ && \ - mv opendbc/ $PYTHONPATH && rm -rf /tmp/opendbc/ + mv opendbc $PYTHONPATH && rm -rf /tmp/opendbc/ # for Jenkins COPY README.md panda.tar.* /tmp/ From cc34040c892407d7839abfa73b67de6a3dd6b3c1 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sun, 18 Aug 2024 11:36:54 -0700 Subject: [PATCH 10/10] can't move it --- Dockerfile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 669317a9ed..51a58ca803 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:24.04 -ENV PYTHONUNBUFFERED 1 -ENV PYTHONPATH /tmp/pythonpath +ENV PYTHONUNBUFFERED=1 +ENV PYTHONPATH=/tmp/pythonpath ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -29,15 +29,15 @@ RUN /tmp/install.sh && rm -rf $CPPCHECK_DIR/.git/ ENV SKIP_CPPCHECK_INSTALL=1 # TODO: this should be a "pip install" or not even in this repo at all -ENV OPENDBC_REF="d377af6c2d01b30d3de892cee91f1ed8fb50d6e8" -RUN git config --global --add safe.directory /tmp/pythonpath/panda -RUN mkdir -p /tmp/pythonpath/ && \ +RUN git config --global --add safe.directory $PYTHONPATH/panda +ENV OPENDBC_REF="5ed7a834a4e0e24c3968dd1e98ceb4b9d5f9791a" +RUN mkdir -p $PYTHONPATH && \ cd /tmp/ && \ - git clone --depth 1 https://github.com/commaai/opendbc && \ - cd opendbc && git fetch origin $OPENDBC_REF && git checkout FETCH_HEAD && rm -rf .git/ && \ + git clone --depth 1 https://github.com/commaai/opendbc opendbc_repo && \ + cd opendbc_repo && git fetch origin $OPENDBC_REF && git checkout FETCH_HEAD && rm -rf .git/ && \ pip3 install --break-system-packages --no-cache-dir Cython numpy && \ scons -j8 --minimal opendbc/ && \ - mv opendbc $PYTHONPATH && rm -rf /tmp/opendbc/ + ln -s $PWD/opendbc $PYTHONPATH/opendbc # for Jenkins COPY README.md panda.tar.* /tmp/