From 555dc2a69fe3382aa95c04289daa878d1d27251e Mon Sep 17 00:00:00 2001 From: Michael Carlstrom Date: Wed, 14 Feb 2024 19:17:06 -0500 Subject: [PATCH 01/10] start improvements --- .devcontainer/linux/devcontainer.json | 6 +- .../linux_test_environment/devcontainer.json | 24 ++++++ .devcontainer/macos/devcontainer.json | 4 +- .devcontainer/windows/devcontainer.json | 2 +- Dockerfile | 81 +++++++++++++------ 5 files changed, 85 insertions(+), 32 deletions(-) create mode 100644 .devcontainer/linux_test_environment/devcontainer.json diff --git a/.devcontainer/linux/devcontainer.json b/.devcontainer/linux/devcontainer.json index 06451cd1..52f25cf0 100644 --- a/.devcontainer/linux/devcontainer.json +++ b/.devcontainer/linux/devcontainer.json @@ -6,13 +6,13 @@ }, "mounts": [ // SSH Keys - "type=bind,source=/home/${localEnv:USER}/.ssh,target=/root/.ssh,readonly", + "type=bind,source=/home/${localEnv:USER}/.ssh,target=/home/rov/.ssh,readonly", // X11 for guis "source=/tmp/.X11-unix,target=/tmp/.X11-unix,type=bind,consistency=cached", // gitconfig - "type=bind,source=/home/${localEnv:USER}/.gitconfig,target=/root/.gitconfig,readonly" + "type=bind,source=/home/${localEnv:USER}/.gitconfig,target=/home/rov/.gitconfig,readonly" ], - "workspaceFolder": "/root/rov-24", + "workspaceFolder": "/home/rov/rov-24", "customizations": { "vscode" : { "extensions": ["ms-python.flake8", "ms-azuretools.vscode-docker", diff --git a/.devcontainer/linux_test_environment/devcontainer.json b/.devcontainer/linux_test_environment/devcontainer.json new file mode 100644 index 00000000..6ccadb93 --- /dev/null +++ b/.devcontainer/linux_test_environment/devcontainer.json @@ -0,0 +1,24 @@ +{ + "name": "ROV Linux Test", + "build": { + "dockerfile": "../../Dockerfile" + }, + "mounts": [ + "type=bind,source=/home/${localEnv:USER}/.ssh,target=/home/rov/.ssh,readonly", + "source=/tmp/.X11-unix,target=/tmp/.X11-unix,type=bind,consistency=cached", + "type=bind,source=/home/${localEnv:USER}/.gitconfig,target=/home/rov/.gitconfig,readonly" + ], + "workspaceFolder": "/home/rov/rov-24", + "customizations": { + "vscode": { + "extensions": [ + "ms-python.flake8", + "ms-azuretools.vscode-docker", + "streetsidesoftware.code-spell-checker", + "ms-python.python", + "ms-python.mypy-type-checker" + ] + } + }, + "privileged": true +} \ No newline at end of file diff --git a/.devcontainer/macos/devcontainer.json b/.devcontainer/macos/devcontainer.json index 105bf055..d6303381 100644 --- a/.devcontainer/macos/devcontainer.json +++ b/.devcontainer/macos/devcontainer.json @@ -5,9 +5,9 @@ "DISPLAY": "unix:0" }, "mounts": [ - "type=bind,source=/Users/${localEnv:USER}/.ssh,target=/root/.ssh,readonly", + "type=bind,source=/Users/${localEnv:USER}/.ssh,target=/home/rov/.ssh,readonly", ], - "workspaceFolder": "/root/rov-24", + "workspaceFolder": "/home/rov/rov-24", "customizations": { "vscode" : { "extensions": ["ms-python.flake8", "ms-azuretools.vscode-docker", diff --git a/.devcontainer/windows/devcontainer.json b/.devcontainer/windows/devcontainer.json index 5d22c581..b60e5a76 100644 --- a/.devcontainer/windows/devcontainer.json +++ b/.devcontainer/windows/devcontainer.json @@ -4,7 +4,7 @@ "containerEnv": { "DISPLAY": "${localEnv:DISPLAY}" }, - "workspaceFolder": "/root/rov-24", + "workspaceFolder": "/home/rov/rov-24", "customizations": { "vscode" : { "extensions": ["ms-python.flake8", "ms-azuretools.vscode-docker", diff --git a/Dockerfile b/Dockerfile index 07274d84..1bd6ed32 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,44 +1,73 @@ FROM osrf/ros:iron-desktop-full -RUN sudo apt-get update -y +RUN apt-get update -y \ + && apt-get install --no-install-recommends -y \ + # Install Video for Linux + v4l-utils=1.22.1-2build1 -y \ + # Install lsusb + usbutils=1:014-1build1 \ + # Install nano + nano=6.2-1 \ + # Install pip + python3-pip=22.0.2+dfsg-1ubuntu0.4\ + && apt-get upgrade -y \ + # Clean for better performance + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* -# Install pip -RUN sudo apt-get install python3-pip -y +#Set up rov user +ARG USER_NAME=rov +ARG USER_UID=1000 +ARG USER_GID=1000 -# Install Video for Linux -RUN sudo apt-get install v4l-utils -y +RUN groupadd ${USER_NAME} --gid ${USER_GID}\ + && useradd -l -m ${USER_NAME} -u ${USER_UID} -g ${USER_GID} -s /bin/bash -# Install lsusb -RUN sudo apt-get install usbutils -y +# Gives Sudo perms to USER +RUN echo "${USER_NAME} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/${USER_NAME} +RUN chmod 0440 /etc/sudoers.d/${USER_NAME} -# Install nano -RUN sudo apt-get install nano -y +# Adds $USER to environment +ENV USER ${USER_NAME} +# Switches to rov user +USER ${USER_NAME} -RUN sudo apt-get update -y -RUN sudo apt-get upgrade -y +# Update Pip +RUN pip install --no-cache-dir --upgrade pip==24.0 -WORKDIR /root/rov-24 +WORKDIR /home/${USER_NAME}/rov-24 +# Copies in Python deps COPY pyproject.toml . - -RUN pip install --no-cache-dir . \ - && rm pyproject.toml - -COPY . . - -# TODO for future nerd to do this via ENTRYPOINT which be better but, I could not get ENTRYPOINT to play with VsCODE. -RUN . /root/rov-24/.vscode/rov_setup.sh -RUN echo "$export PYTHONWARNINGS=ignore:::setuptools.command.install,ignore:::setuptools.command.easy_install,ignore:::pkg_resources" >> ~/.bashrc ; +# Copies in ROS deps +RUN mkdir src +COPY package.xml* ./src +# Copies in Install Script +COPY .vscode/install_dependencies.sh . # Installs ROS and python dependencies -RUN . /root/rov-24/.vscode/install_dependencies.sh +RUN ./install_dependencies.sh \ + # Clean up + && rm -r src \ + && rm install_dependencies.sh -RUN . /opt/ros/iron/setup.sh \ - && PYTHONWARNINGS=ignore:::setuptools.command.install,ignore:::setuptools.command.easy_install,ignore:::pkg_resources; export PYTHONWARNINGS\ - && colcon build --symlink-install + +# Remove build warnings +ENV PYTHONWARNINGS ignore:::setuptools.command.install,ignore:::setuptools.command.easy_install,ignore:::pkg_resources # Remove identity file generated by action/checkout # Start by finding all files ending in config # Then removes file paths without "git" # Then removes the 'sshCommand' line from each file -RUN find . -name "*config" | grep git | while read -r line; do sed -i '/sshCommand/d' $line; done +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN find . -name "*config" | grep git | while read -r line; do sed -i "/sshCommand/d" $line; done + +# TODO for future nerd to do this via ENTRYPOINT which be better but, I could not get ENTRYPOINT to play with VsCODE. +COPY .vscode/rov_setup.sh . +RUN ./rov_setup.sh + +# Do full copy and build as last step +COPY . . +RUN . /opt/ros/iron/setup.sh \ + && PYTHONWARNINGS=ignore:::setuptools.command.install,ignore:::setuptools.command.easy_install,ignore:::pkg_resources; export PYTHONWARNINGS\ + && colcon build --symlink-install \ No newline at end of file From 26999d6633d9b4864f50853a4159cd6aaa2fd1a2 Mon Sep 17 00:00:00 2001 From: Michael Carlstrom Date: Wed, 14 Feb 2024 21:04:51 -0500 Subject: [PATCH 02/10] make build smol --- Dockerfile | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1bd6ed32..473d7c96 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,12 +20,12 @@ ARG USER_NAME=rov ARG USER_UID=1000 ARG USER_GID=1000 -RUN groupadd ${USER_NAME} --gid ${USER_GID}\ +RUN groupadd ${USER_NAME} --gid ${USER_GID} \ && useradd -l -m ${USER_NAME} -u ${USER_UID} -g ${USER_GID} -s /bin/bash # Gives Sudo perms to USER -RUN echo "${USER_NAME} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/${USER_NAME} -RUN chmod 0440 /etc/sudoers.d/${USER_NAME} +RUN echo "${USER_NAME} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/${USER_NAME} \ + && chmod 0440 /etc/sudoers.d/${USER_NAME} # Adds $USER to environment ENV USER ${USER_NAME} @@ -55,19 +55,23 @@ RUN ./install_dependencies.sh \ # Remove build warnings ENV PYTHONWARNINGS ignore:::setuptools.command.install,ignore:::setuptools.command.easy_install,ignore:::pkg_resources +# TODO for future nerd to do this via ENTRYPOINT which be better but, I could not get ENTRYPOINT to play with VsCODE. +COPY .vscode/rov_setup.sh . +RUN ./rov_setup.sh \ + && rm rov_setup.sh + +COPY .git .git # Remove identity file generated by action/checkout # Start by finding all files ending in config # Then removes file paths without "git" # Then removes the 'sshCommand' line from each file SHELL ["/bin/bash", "-o", "pipefail", "-c"] -RUN find . -name "*config" | grep git | while read -r line; do sed -i "/sshCommand/d" $line; done - -# TODO for future nerd to do this via ENTRYPOINT which be better but, I could not get ENTRYPOINT to play with VsCODE. -COPY .vscode/rov_setup.sh . -RUN ./rov_setup.sh +USER root +RUN find . -name "*config" | grep git | while read -r line; do sed -i "/sshCommand/d" "$line"; done +USER ${USER_NAME} # Do full copy and build as last step COPY . . RUN . /opt/ros/iron/setup.sh \ && PYTHONWARNINGS=ignore:::setuptools.command.install,ignore:::setuptools.command.easy_install,ignore:::pkg_resources; export PYTHONWARNINGS\ - && colcon build --symlink-install \ No newline at end of file + && colcon build --symlink-install From 66d27e7a89b57a6e37e2d5e7c686f24e2e2d81b6 Mon Sep 17 00:00:00 2001 From: Michael Carlstrom Date: Wed, 14 Feb 2024 21:06:54 -0500 Subject: [PATCH 03/10] comment --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 473d7c96..a33dcf42 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,6 +37,10 @@ RUN pip install --no-cache-dir --upgrade pip==24.0 WORKDIR /home/${USER_NAME}/rov-24 +# One might wonder why the following is done. +# It is done so that the dependencies can be cached without needing to be reinstalled +# each and everytime. Since these do not change too often this improves performance in most cases + # Copies in Python deps COPY pyproject.toml . # Copies in ROS deps From eb017668da610d3974642465dbf2e704d87ef886 Mon Sep 17 00:00:00 2001 From: Michael Carlstrom Date: Thu, 15 Feb 2024 05:56:36 +0000 Subject: [PATCH 04/10] test --- .dockerignore | 3 ++- Dockerfile | 21 +++++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.dockerignore b/.dockerignore index e8b072e4..39eae289 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ build install -log \ No newline at end of file +log +*.egg-info \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index a33dcf42..440daad4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,6 @@ +# syntax=docker/dockerfile-upstream:master-labs +# Done so --parents flag works + FROM osrf/ros:iron-desktop-full RUN apt-get update -y \ @@ -9,12 +12,20 @@ RUN apt-get update -y \ # Install nano nano=6.2-1 \ # Install pip - python3-pip=22.0.2+dfsg-1ubuntu0.4\ + python3-pip=22.0.2+dfsg-1ubuntu0.4 \ + # Install geographiclib dependencies for mavros. + geographiclib-tools=1.52-1 \ && apt-get upgrade -y \ # Clean for better performance && apt-get clean \ && rm -rf /var/lib/apt/lists/* + +# # Done here because file needs sudo perms +# Switch to bash so the process subsition works. aka <() +SHELL ["/bin/bash", "-c"] +RUN . <(wget -qO- https://raw.githubusercontent.com/mavlink/mavros/ros2/mavros/scripts/install_geographiclib_datasets.sh) + #Set up rov user ARG USER_NAME=rov ARG USER_UID=1000 @@ -44,18 +55,16 @@ WORKDIR /home/${USER_NAME}/rov-24 # Copies in Python deps COPY pyproject.toml . # Copies in ROS deps -RUN mkdir src -COPY package.xml* ./src +# https://docs.docker.com/engine/reference/builder/#copy---parents +COPY --parents --chown=${USER_NAME}:${USER_NAME} ./*/*/package.xml ./*/*/*/package.xml ./ # Copies in Install Script COPY .vscode/install_dependencies.sh . # Installs ROS and python dependencies RUN ./install_dependencies.sh \ # Clean up - && rm -r src \ && rm install_dependencies.sh - # Remove build warnings ENV PYTHONWARNINGS ignore:::setuptools.command.install,ignore:::setuptools.command.easy_install,ignore:::pkg_resources @@ -75,7 +84,7 @@ RUN find . -name "*config" | grep git | while read -r line; do sed -i "/sshComm USER ${USER_NAME} # Do full copy and build as last step -COPY . . +COPY --chown=${USER_NAME}:${USER_NAME} . . RUN . /opt/ros/iron/setup.sh \ && PYTHONWARNINGS=ignore:::setuptools.command.install,ignore:::setuptools.command.easy_install,ignore:::pkg_resources; export PYTHONWARNINGS\ && colcon build --symlink-install From 9cd3da18e941106303d8bca668f7ed0a256b41a3 Mon Sep 17 00:00:00 2001 From: Michael Carlstrom Date: Thu, 15 Feb 2024 06:07:44 +0000 Subject: [PATCH 05/10] comments --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 440daad4..105e91ff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,9 +20,9 @@ RUN apt-get update -y \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* - -# # Done here because file needs sudo perms +# Done here because file needs sudo perms # Switch to bash so the process subsition works. aka <() +# Runs MAVROS helper install script SHELL ["/bin/bash", "-c"] RUN . <(wget -qO- https://raw.githubusercontent.com/mavlink/mavros/ros2/mavros/scripts/install_geographiclib_datasets.sh) @@ -50,8 +50,7 @@ WORKDIR /home/${USER_NAME}/rov-24 # One might wonder why the following is done. # It is done so that the dependencies can be cached without needing to be reinstalled -# each and everytime. Since these do not change too often this improves performance in most cases - +# each and everytime. Since these do not change too often this improves performance in most cases. # Copies in Python deps COPY pyproject.toml . # Copies in ROS deps @@ -73,6 +72,7 @@ COPY .vscode/rov_setup.sh . RUN ./rov_setup.sh \ && rm rov_setup.sh +# TODO probs do more testing here COPY .git .git # Remove identity file generated by action/checkout # Start by finding all files ending in config From eb8cccaf8ff7e0331e6d3570e891906e5902424a Mon Sep 17 00:00:00 2001 From: Michael Carlstrom Date: Thu, 15 Feb 2024 22:23:12 +0000 Subject: [PATCH 06/10] gtg --- Dockerfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 105e91ff..b45a6954 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,16 +72,13 @@ COPY .vscode/rov_setup.sh . RUN ./rov_setup.sh \ && rm rov_setup.sh -# TODO probs do more testing here -COPY .git .git +COPY --chown=${USER_NAME}:${USER_NAME} .git .git # Remove identity file generated by action/checkout # Start by finding all files ending in config # Then removes file paths without "git" # Then removes the 'sshCommand' line from each file SHELL ["/bin/bash", "-o", "pipefail", "-c"] -USER root RUN find . -name "*config" | grep git | while read -r line; do sed -i "/sshCommand/d" "$line"; done -USER ${USER_NAME} # Do full copy and build as last step COPY --chown=${USER_NAME}:${USER_NAME} . . From aa47f782e15b8d0de5a32d8d1953796485c4128b Mon Sep 17 00:00:00 2001 From: Michael Carlstrom Date: Thu, 15 Feb 2024 23:57:53 +0000 Subject: [PATCH 07/10] lock version --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7243516c..c15e94a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ RUN apt-get update -y \ geographiclib-tools=1.52-1 \ # Install missing libxcb-cursor0 xvfb for PyQt unit testing # https://pytest-qt.readthedocs.io/en/latest/troubleshooting.html - xvfb \ + xvfb=2:21.1.4-2ubuntu1.7~22.04.8 \ && apt-get upgrade -y \ # Clean for better performance && apt-get clean \ From 7465893b638b8da779ef5e4b642fa01271960f77 Mon Sep 17 00:00:00 2001 From: Michael Carlstrom Date: Fri, 16 Feb 2024 01:57:07 +0000 Subject: [PATCH 08/10] so clean --- Dockerfile | 9 +++++++-- src/surface/flight_control/package.xml | 1 - src/surface/gui/package.xml | 1 - 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index c15e94a4..1e01b102 100644 --- a/Dockerfile +++ b/Dockerfile @@ -85,6 +85,11 @@ RUN find . -name "*config" | grep git | while read -r line; do sed -i "/sshComm # Do full copy and build as last step COPY --chown=${USER_NAME}:${USER_NAME} . . + +ARG EXPECTED_OUTPUT="All system dependencies have been satisfied" + +# Checks that all rosdeps are installed. +RUN if [[ $(rosdep check -r --from-paths src) != "${EXPECTED_OUTPUT}" ]]; then echo "ROS deps not all installed. Trying adding another layer to COPY instruction for ROS deps."; exit 1; fi + RUN . /opt/ros/iron/setup.sh \ - && PYTHONWARNINGS=ignore:::setuptools.command.install,ignore:::setuptools.command.easy_install,ignore:::pkg_resources; export PYTHONWARNINGS\ - && colcon build --symlink-install + && colcon build --symlink-install diff --git a/src/surface/flight_control/package.xml b/src/surface/flight_control/package.xml index beedf691..38e5dda4 100644 --- a/src/surface/flight_control/package.xml +++ b/src/surface/flight_control/package.xml @@ -10,7 +10,6 @@ rclpy sensor_msgs - rov_msgs ros2launch ament_flake8 diff --git a/src/surface/gui/package.xml b/src/surface/gui/package.xml index 1925b0e9..7c151312 100644 --- a/src/surface/gui/package.xml +++ b/src/surface/gui/package.xml @@ -13,7 +13,6 @@ ros2launch ros2run - rov_msgs mavros From b88d37ac976afcf47be52dc308c17f9bc995fc88 Mon Sep 17 00:00:00 2001 From: Michael Carlstrom Date: Thu, 15 Feb 2024 21:35:52 -0500 Subject: [PATCH 09/10] cool boi fix --- Dockerfile | 8 ++++++-- src/surface/flight_control/package.xml | 1 + src/surface/gui/package.xml | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1e01b102..8ffbaf91 100644 --- a/Dockerfile +++ b/Dockerfile @@ -58,7 +58,7 @@ WORKDIR /home/${USER_NAME}/rov-24 COPY pyproject.toml . # Copies in ROS deps # https://docs.docker.com/engine/reference/builder/#copy---parents -COPY --parents --chown=${USER_NAME}:${USER_NAME} ./*/*/package.xml ./*/*/*/package.xml ./ +COPY --parents --chown=${USER_NAME}:${USER_NAME} ./*/*/package.xml ./*/*/*/package.xml ./*/*/*/*/package.xml ./ # Copies in Install Script COPY .vscode/install_dependencies.sh . @@ -89,7 +89,11 @@ COPY --chown=${USER_NAME}:${USER_NAME} . . ARG EXPECTED_OUTPUT="All system dependencies have been satisfied" # Checks that all rosdeps are installed. -RUN if [[ $(rosdep check -r --from-paths src) != "${EXPECTED_OUTPUT}" ]]; then echo "ROS deps not all installed. Trying adding another layer to COPY instruction for ROS deps."; exit 1; fi +RUN if [[ $(rosdep check -r --from-paths src --ignore-src) != "${EXPECTED_OUTPUT}" ]]; \ + then echo "ROS deps not all installed. Trying adding another layer to COPY instruction for ROS deps."; \ + rosdep check -r --from-paths src --ignore-src; \ + exit 1; \ + fi RUN . /opt/ros/iron/setup.sh \ && colcon build --symlink-install diff --git a/src/surface/flight_control/package.xml b/src/surface/flight_control/package.xml index 38e5dda4..beedf691 100644 --- a/src/surface/flight_control/package.xml +++ b/src/surface/flight_control/package.xml @@ -10,6 +10,7 @@ rclpy sensor_msgs + rov_msgs ros2launch ament_flake8 diff --git a/src/surface/gui/package.xml b/src/surface/gui/package.xml index 7c151312..1925b0e9 100644 --- a/src/surface/gui/package.xml +++ b/src/surface/gui/package.xml @@ -13,6 +13,7 @@ ros2launch ros2run + rov_msgs mavros From 8b05ca55c02736a9350a2c85b2947de82c4da69f Mon Sep 17 00:00:00 2001 From: benjaminwp18 <90342856+benjaminwp18@users.noreply.github.com> Date: Sat, 17 Feb 2024 18:55:34 +0000 Subject: [PATCH 10/10] Edit caching error message --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8ffbaf91..e0e0461b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -90,7 +90,7 @@ ARG EXPECTED_OUTPUT="All system dependencies have been satisfied" # Checks that all rosdeps are installed. RUN if [[ $(rosdep check -r --from-paths src --ignore-src) != "${EXPECTED_OUTPUT}" ]]; \ - then echo "ROS deps not all installed. Trying adding another layer to COPY instruction for ROS deps."; \ + then echo "ROS deps not all installed. Try adding another /* directory layer to the COPY instruction labeled 'Copies in ROS deps'."; \ rosdep check -r --from-paths src --ignore-src; \ exit 1; \ fi