From 9b339a79515ff8bb732a57319daae832b7e9960b Mon Sep 17 00:00:00 2001 From: Christoph Lauer Date: Tue, 26 Sep 2023 00:27:48 +0200 Subject: [PATCH] docker: optimizations - linux host: use pulse unix socket - mpd: run as user (pi / root) - mpd: remove port exposure to host, connections to mpd only come from other docker containers - less config adjustments for docker environment --- .dockerignore | 3 +- docker/armv7/jukebox.Dockerfile | 41 +++++++-------- docker/config/docker.pulse.mpd.conf | 4 +- docker/config/jukebox.overrides.yaml | 5 -- docker/docker-compose.linux.yml | 39 ++++++++++---- docker/docker-compose.mac.yml | 5 -- docker/docker-compose.windows.yml | 12 ----- docker/docker-compose.yml | 32 +++++++----- docker/jukebox.Dockerfile | 22 ++++---- docker/mpd.Dockerfile | 21 ++++---- docs/sphinx/developer/docker.rst | 66 ++++++++---------------- src/jukebox/jukebox/playlistgenerator.py | 2 +- 12 files changed, 115 insertions(+), 137 deletions(-) delete mode 100755 docker/docker-compose.windows.yml diff --git a/.dockerignore b/.dockerignore index fed16790c..55fe62a70 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,6 @@ .git +.githooks +.github .dockerignore .DS_Store @@ -6,7 +8,6 @@ docker docs installation -src shared # webapp diff --git a/docker/armv7/jukebox.Dockerfile b/docker/armv7/jukebox.Dockerfile index b2ca11944..b4f9168e2 100644 --- a/docker/armv7/jukebox.Dockerfile +++ b/docker/armv7/jukebox.Dockerfile @@ -5,22 +5,19 @@ FROM arm32v7/debian:buster-slim # These are only dependencies that are required to get as close to the # Raspberry Pi environment as possible. RUN apt-get update && apt-get install -y \ - alsa-utils \ libasound2-dev \ - libasound2-plugins \ pulseaudio \ pulseaudio-utils \ --no-install-recommends \ && rm -rf /var/lib/apt/lists/* -RUN usermod -aG audio,pulse,pulse-access root +ARG UID +ARG USER +ARG HOME +ENV INSTALLATION_PATH ${HOME}/RPi-Jukebox-RFID -ENV HOME /root -ENV MPD_HOST mpd -ENV INSTALLATION_DIR /home/pi/RPi-Jukebox-RFID -ENV DOCKER_DIR ${INSTALLATION_DIR}/docker - -WORKDIR $INSTALLATION_DIR +RUN test ${UID} -gt 0 && useradd -m -u ${UID} ${USER} || continue +RUN usermod -aG pulse ${USER} # Jukebox # Install all Jukebox dependencies @@ -34,13 +31,12 @@ RUN apt-get update && apt-get install -qq -y \ #resolvconf #python3-spidev -COPY . ${INSTALLATION_DIR} - # Install Jukebox # Install libzmq with Websocket support from pre-compiled source -ENV ZMQ_TMP_DIR libzmq -ENV ZMQ_PREFIX /usr/local -RUN cd ${HOME} && mkdir ${ZMQ_TMP_DIR} && cd ${ZMQ_TMP_DIR}; \ +ENV ZMQ_TMP_DIR "/root/libzmq" +ENV ZMQ_PREFIX "/usr/local" +ENV ZMQ_DRAFT_API 1 +RUN mkdir -p ${ZMQ_TMP_DIR} && cd ${ZMQ_TMP_DIR}; \ wget --quiet --load-cookies /tmp/cookies.txt "https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1KP6BqLF-i2dCUsHhOUpOwwuOmKsB5GKY' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=1KP6BqLF-i2dCUsHhOUpOwwuOmKsB5GKY" -O libzmq.tar.gz && rm -rf /tmp/cookies.txt; \ tar -xzf libzmq.tar.gz; \ rm -f libzmq.tar.gz; \ @@ -49,26 +45,29 @@ RUN cd ${HOME} && mkdir ${ZMQ_TMP_DIR} && cd ${ZMQ_TMP_DIR}; \ # Install libzmq with Websocket and compile # ENV LIBSODIUM_VERSION 1.0.18 # ENV ZMQ_VERSION 4.3.4 -# RUN cd ${HOME} && mkdir ${ZMQ_TMP_DIR} && cd ${ZMQ_TMP_DIR}; \ +# RUN mkdir -p ${ZMQ_TMP_DIR} && cd ${ZMQ_TMP_DIR}; \ # wget --quiet https://github.com/jedisct1/libsodium/releases/download/${LIBSODIUM_VERSION}-RELEASE/libsodium-${LIBSODIUM_VERSION}.tar.gz; \ # tar -zxvf libsodium-${LIBSODIUM_VERSION}.tar.gz; \ # cd libsodium-${LIBSODIUM_VERSION}/; \ # ./configure; \ # make && make install -# RUN cd ${HOME}/${ZMQ_TMP_DIR}; \ +# RUN cd ${ZMQ_TMP_DIR}; \ # wget https://github.com/zeromq/libzmq/releases/download/v${ZMQ_VERSION}/zeromq-${ZMQ_VERSION}.tar.gz -O libzmq.tar.gz; \ # tar -xzf libzmq.tar.gz; \ # zeromq-${ZMQ_VERSION}/configure --prefix=${ZMQ_PREFIX} --enable-drafts; \ # make && make install; -RUN pip3 install --pre pyzmq \ - --install-option=--enable-drafts \ - --install-option=--zmq=${ZMQ_PREFIX} +USER ${USER} +WORKDIR ${HOME} +COPY --chown=${USER}:${USER} . ${INSTALLATION_PATH}/ -RUN pip3 install --no-cache-dir -r ${INSTALLATION_DIR}/requirements.txt +RUN pip3 install --no-cache-dir -r ${INSTALLATION_PATH}/requirements.txt +RUN pip3 install --no-cache-dir --pre --no-binary pyzmq pyzmq EXPOSE 5555 5556 +WORKDIR ${INSTALLATION_PATH}/src/jukebox + # Run Jukebox # CMD bash -CMD python3 ${INSTALLATION_DIR}/src/jukebox/run_jukebox.py +CMD python3 ${INSTALLATION_PATH}/src/jukebox/run_jukebox.py diff --git a/docker/config/docker.pulse.mpd.conf b/docker/config/docker.pulse.mpd.conf index 96f27347e..ad7713d0d 100644 --- a/docker/config/docker.pulse.mpd.conf +++ b/docker/config/docker.pulse.mpd.conf @@ -11,7 +11,7 @@ # be disabled and audio files will only be accepted over ipc socket (using # file:// protocol) or streaming files over an accepted protocol. # -music_directory "/home/pi/RPi-Jukebox-RFID/shared/audiofolders" +music_directory "~/RPi-Jukebox-RFID/shared/audiofolders" # # This setting sets the MPD internal playlist directory. The purpose of this # directory is storage for playlists created by MPD. The server will use @@ -67,7 +67,7 @@ sticker_file "~/.config/mpd/sticker.sql" # initialization. This setting is disabled by default and MPD is run as the # current user. # -user "root" +# user "root" # # This setting specifies the group that MPD will run as. If not specified # primary group of user specified with "user" setting will be used (if set). diff --git a/docker/config/jukebox.overrides.yaml b/docker/config/jukebox.overrides.yaml index 0bd6fad07..ca79486d4 100644 --- a/docker/config/jukebox.overrides.yaml +++ b/docker/config/jukebox.overrides.yaml @@ -1,7 +1,2 @@ playermpd: host: mpd - mpd_conf: /etc/mpd.conf -pulse: - outputs: - primary: - pulse_sink_name: Channel_1__Channel_2.2 \ No newline at end of file diff --git a/docker/docker-compose.linux.yml b/docker/docker-compose.linux.yml index 2bc9ab812..9609dc18d 100755 --- a/docker/docker-compose.linux.yml +++ b/docker/docker-compose.linux.yml @@ -1,10 +1,29 @@ -version: "3.9" - -services: - mpd: - devices: - - /dev/snd - - jukebox: - devices: - - /dev/snd +version: "3.9" + +services: + mpd: + build: + args: + - UID=${UID:-1000} + - USER=pi + - HOME=/home/pi + environment: + - PULSE_SERVER=unix:/tmp/pulse-sock + volumes: + - ../shared/audiofolders:/home/pi/RPi-Jukebox-RFID/shared/audiofolders + - ../shared/playlists:/home/pi/.config/mpd/playlists + - ./config/docker.pulse.mpd.conf:/home/pi/.config/mpd/mpd.conf + - $XDG_RUNTIME_DIR/pulse/native:/tmp/pulse-sock + + jukebox: + build: + args: + - UID=${UID:-1000} + - USER=pi + - HOME=/home/pi + environment: + - PULSE_SERVER=unix:/tmp/pulse-sock + volumes: + - ../shared:/home/pi/RPi-Jukebox-RFID/shared + - ./config/docker.pulse.mpd.conf:/home/pi/.config/mpd/mpd.conf + - $XDG_RUNTIME_DIR/pulse/native:/tmp/pulse-sock diff --git a/docker/docker-compose.mac.yml b/docker/docker-compose.mac.yml index 27163cb93..d20d7ed6e 100644 --- a/docker/docker-compose.mac.yml +++ b/docker/docker-compose.mac.yml @@ -2,16 +2,11 @@ version: "3.9" services: mpd: - environment: - - PULSE_SERVER=tcp:host.docker.internal:4713 volumes: - - ./config/docker.pulse.mpd.conf:/root/.config/mpd/mpd.conf:rw - ~/.config/pulse:/root/.config/pulse - /usr/local/Cellar/pulseaudio/14.2/etc/pulse:/etc/pulse jukebox: - environment: - - PULSE_SERVER=tcp:host.docker.internal:4713 volumes: - ~/.config/pulse:/root/.config/pulse - /usr/local/Cellar/pulseaudio/14.2/etc/pulse:/etc/pulse diff --git a/docker/docker-compose.windows.yml b/docker/docker-compose.windows.yml deleted file mode 100755 index 51134451d..000000000 --- a/docker/docker-compose.windows.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: "3.9" - -services: - mpd: - environment: - - PULSE_SERVER=tcp:host.docker.internal:4713 - volumes: - - ./config/docker.pulse.mpd.conf:/root/.config/mpd/mpd.conf:rw - - jukebox: - environment: - - PULSE_SERVER=tcp:host.docker.internal:4713 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 73c462c31..68bb5c351 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -3,27 +3,34 @@ version: "3.9" services: mpd: build: + args: + - UID=0 + - USER=root + - HOME=/root context: ../ dockerfile: ./docker/mpd.Dockerfile container_name: mpd - ports: - - 6600:6600 - - 8800:8800 + environment: + - PULSE_SERVER=tcp:host.docker.internal:4713 restart: unless-stopped volumes: - - ../shared/audiofolders:/home/pi/RPi-Jukebox-RFID/shared/audiofolders:rw - - ../shared/playlists:/root/.config/mpd/playlists:rw - - ./config/docker.mpd.conf:/root/.config/mpd/mpd.conf:rw + - ../shared/audiofolders:/root/RPi-Jukebox-RFID/shared/audiofolders + - ../shared/playlists:/root/.config/mpd/playlists + - ./config/docker.pulse.mpd.conf:/root/.config/mpd/mpd.conf jukebox: build: + args: + - UID=0 + - USER=root + - HOME=/root context: ../ dockerfile: ./docker/jukebox.Dockerfile container_name: jukebox depends_on: - mpd - links: - - mpd + environment: + - PULSE_SERVER=tcp:host.docker.internal:4713 ports: - 5555:5555 - 5556:5556 @@ -31,9 +38,8 @@ services: restart: unless-stopped tty: true volumes: - - ../src/jukebox:/home/pi/RPi-Jukebox-RFID/src/jukebox - - ../shared:/home/pi/RPi-Jukebox-RFID/shared - - ./config/docker.mpd.conf:/etc/mpd.conf + - ../shared:/root/RPi-Jukebox-RFID/shared + - ./config/docker.pulse.mpd.conf:/root/.config/mpd/mpd.conf webapp: build: @@ -44,10 +50,8 @@ services: - jukebox environment: - CHOKIDAR_USEPOLLING=true - links: - - jukebox ports: - - 3001:3000 + - 3000:3000 restart: unless-stopped volumes: - ../src/webapp:/home/node/webapp diff --git a/docker/jukebox.Dockerfile b/docker/jukebox.Dockerfile index f1ac07141..e8ce1cb64 100644 --- a/docker/jukebox.Dockerfile +++ b/docker/jukebox.Dockerfile @@ -5,36 +5,38 @@ FROM debian:bullseye-slim # These are only dependencies that are required to get as close to the # Raspberry Pi environment as possible. RUN apt-get update && apt-get install -y \ - alsa-utils \ libasound2-dev \ - libasound2-plugins \ pulseaudio \ pulseaudio-utils \ --no-install-recommends \ && rm -rf /var/lib/apt/lists/* -RUN usermod -aG audio,pulse,pulse-access root +ARG UID +ARG USER +ARG HOME +ENV INSTALLATION_PATH ${HOME}/RPi-Jukebox-RFID -ENV HOME /root -ENV INSTALLATION_PATH /home/pi/RPi-Jukebox-RFID - -WORKDIR $INSTALLATION_PATH +RUN test ${UID} -gt 0 && useradd -m -u ${UID} ${USER} || continue +RUN usermod -aG pulse ${USER} # Jukebox # Install all Jukebox dependencies RUN apt-get update && apt-get install -qq -y \ --allow-downgrades --allow-remove-essential --allow-change-held-packages \ gcc at wget \ - espeak mpc mpg123 git ffmpeg spi-tools netcat alsa-tools \ + espeak mpc mpg123 git ffmpeg spi-tools netcat \ python3 python3-dev python3-pip python3-mutagen python3-gpiozero -COPY . ${INSTALLATION_PATH} +USER ${USER} +WORKDIR ${HOME} +COPY --chown=${USER}:${USER} . ${INSTALLATION_PATH}/ RUN pip3 install --no-cache-dir -r ${INSTALLATION_PATH}/requirements.txt RUN pip3 install pyzmq EXPOSE 5555 5556 +WORKDIR ${INSTALLATION_PATH}/src/jukebox + # Run Jukebox -# CMD bash CMD python3 ${INSTALLATION_PATH}/src/jukebox/run_jukebox.py diff --git a/docker/mpd.Dockerfile b/docker/mpd.Dockerfile index f77b65dc3..e13d7f472 100644 --- a/docker/mpd.Dockerfile +++ b/docker/mpd.Dockerfile @@ -2,9 +2,6 @@ FROM debian:bullseye-slim RUN set -eux ; \ apt-get update && apt-get install -y \ - alsa-utils \ - libasound2-dev \ - libasound2-plugins \ pulseaudio \ pulseaudio-utils \ mpd mpc \ @@ -12,16 +9,16 @@ RUN set -eux ; \ ; \ rm -rf /var/lib/apt/lists/* -ENV HOME /root -RUN mkdir ${HOME}/.config ${HOME}/.config/mpd ; \ - touch ${HOME}/.config/mpd/state -RUN mkdir -p /home/pi/RPi-Jukebox-RFID/shared/audiofolders - -RUN usermod -aG audio,pulse,pulse-access root +ARG UID +ARG USER +ARG HOME -VOLUME ${HOME}/.config/mpd +RUN useradd -m -u ${UID} ${USER} || continue +RUN usermod -aG pulse ${USER} -EXPOSE 6600 +USER ${USER} +RUN mkdir -p ${HOME}/.config/mpd ; \ + touch ${HOME}/.config/mpd/state -CMD [ ! -s ~/.config/mpd/pid ] && mpd --stdout --no-daemon ${HOME}/.config/mpd/mpd.conf +CMD mpd --stdout --no-daemon ${HOME}/.config/mpd/mpd.conf diff --git a/docs/sphinx/developer/docker.rst b/docs/sphinx/developer/docker.rst index 3a2150a21..f3de61902 100644 --- a/docs/sphinx/developer/docker.rst +++ b/docs/sphinx/developer/docker.rst @@ -47,7 +47,6 @@ Prerequisites `Override file `_ in your ``jukebox.yaml``. - * **[Currently required]** Update all relative paths (``../..``) in to ``/home/pi/RPi-Jukebox-RFID``. 4. Change directory into the ``./RPi-Jukebox-RFID/shared/audiofolders`` and copy a set of MP3 files into this folder (for more fun when testing). @@ -56,47 +55,29 @@ Run development environment In contrary to how everything is set up on the Raspberry Pi, it's good practice to isolate different components in different Docker images. They can be run individually or in combination. -To do that, we use ``docker-compose``. +To do that, we use ``docker compose``. Linux ^^^^^^^ -Make sure you don't use ``sudo`` to run your ``docker-compose``. Check out Docker's `post-installation guide ` +Make sure you don't use ``sudo`` to run your ``docker compose``. Check out Docker's `post-installation guide ` for more information. .. code-block:: bash // Build Images - $ docker-compose -f docker/docker-compose.yml -f docker/docker-compose.linux.yml build + $ docker compose -f docker/docker-compose.yml -f docker/docker-compose.linux.yml build // Run Docker Environment - $ docker-compose -f docker/docker-compose.yml -f docker/docker-compose.linux.yml up + $ docker compose -f docker/docker-compose.yml -f docker/docker-compose.linux.yml up // Shuts down Docker containers and Docker network - $ docker-compose -f docker/docker-compose.yml -f docker/docker-compose.linux.yml down + $ docker compose -f docker/docker-compose.yml -f docker/docker-compose.linux.yml down -Note: if you have ``mpd`` running on your system, you need to stop it using: - -.. code-block:: bash - - $ sudo systemctl stop mpd.socket - $ sudo mpd --kill - - -Otherwise you might get the error message: - -.. code-block:: bash - - $ docker-compose -f docker-compose.yml -f docker-compose.linux.yml up - Starting mpd ... - Starting mpd ... error - (...) - Error starting userland proxy: listen tcp4 0.0.0.0:6600: bind: address already in use - -Read these threads for details: `thread 1 `_ -and `thread 2 `_ - +The docker linux setup tries to mirror the current users pulseaudio socket from ``$XDG_RUNTIME_DIR/pulse/native`` to the containers ``mpd`` and ``jukebox``. +If your pulseaudio socket is in another place, adjust path in docker/docker-compose.linux.yml. You can find out the socket path with the command ``pactl info``. +To access the socket, the user inside the docker containers also must have the same UID as the user running docker. You can find out your UID by running ``id -u``. If your UID deviates from the default UID 1000, please run ``export UID=$(id -u)`` prior to any docker commands. Mac ^^^^^ @@ -107,13 +88,13 @@ for Mac hosts. .. code-block:: bash // Build Images - $ docker-compose -f docker/docker-compose.yml -f docker/docker-compose.mac.yml build + $ docker compose -f docker/docker-compose.yml -f docker/docker-compose.mac.yml build // Run Docker Environment - $ docker-compose -f docker/docker-compose.yml -f docker/docker-compose.mac.yml up + $ docker compose -f docker/docker-compose.yml -f docker/docker-compose.mac.yml up // Shuts down Docker containers and Docker network - $ docker-compose -f docker/docker-compose.yml -f docker/docker-compose.mac.yml down + $ docker compose -f docker/docker-compose.yml -f docker/docker-compose.mac.yml down Windows ^^^^^^^^^^^ @@ -139,30 +120,31 @@ Windows .. code-block:: bash // Build Images - $ docker-compose -f docker/docker-compose.yml -f docker/docker-compose.windows.yml build + $ docker compose -f docker/docker-compose.yml build // Run Docker Environment - $ docker-compose -f docker/docker-compose.yml -f docker/docker-compose.windows.yml up + $ docker compose -f docker/docker-compose.yml up // Shuts down Docker containers and Docker network - $ docker-compose -f docker/docker-compose.yml -f docker/docker-compose.windows.yml down + $ docker compose -f docker/docker-compose.yml down Test & Develop --------------------- The Dockerfile is defined to start all Phoniebox related services. -Open `http://localhost:3001 `_ in your browser to see the web application. +Windows: Open `host.docker.internal:3000 `_ in your browser to see the web application. +Linux / Mac: Open `http://localhost:3000 `_ in your browser to see the web application. While the ``webapp`` container does not require a reload while working on it (hot-reload is enabled), you will have to restart your ``jukebox`` container whenever you make a change (in the Python code). -Instead of stopping and starting the ``docker-compose`` command, you can individually restart your +Instead of stopping and starting the ``docker compose`` command, you can individually restart your ``jukebox`` container. Update the below path with your specific host environment. .. code-block:: bash - $ docker-compose -f docker/docker-compose.yml -f docker/docker-compose.[ENVIRONMENT].yml restart jukebox + $ docker compose -f docker/docker-compose.yml -f docker/docker-compose.[ENVIRONMENT].yml restart jukebox Known issues ---------------- @@ -209,6 +191,8 @@ Typical errors and following exceptions to be ignored in the Docker ``jukebox`` .. code-block:: bash + jukebox | 277:utils.py - jb.utils - MainThread - ERROR - CalledProcessError: Command 'git log --pretty='%h [%cs] %s %d' -n 1 --no-color' returned non-zero exit status 128. + jukebox | 287:utils.py - jb.utils - MainThread - ERROR - CalledProcessError: Command 'git describe --tag --dirty='-dirty'' returned non-zero exit status 128. jukebox | 634:plugs.py - jb.plugin - MainThread - ERROR - Ignoring failed package load finalizer: 'rfid.finalize()' jukebox | 635:plugs.py - jb.plugin - MainThread - ERROR - Reason: FileNotFoundError: [Errno 2] No such file or directory: '/home/pi/RPi-Jukebox-RFID/shared/settings/rfid.yaml' ... @@ -230,14 +214,8 @@ The following command can be run on a Mac. .. code-block:: bash - $ docker build -f docker/jukebox.Dockerfile -t jukebox . - $ docker run -it --rm \ - -v $(PWD)/src/jukebox:/home/pi/RPi-Jukebox-RFID/src/jukebox \ - -v $(PWD)/shared/audiofolders:/home/pi/RPi-Jukebox-RFID/shared/audiofolders \ - -v ~/.config/pulse:/root/.config/pulse \ - -v /usr/local/Cellar/pulseaudio/14.2/etc/pulse/:/etc/pulse \ - -e PULSE_SERVER=tcp:host.docker.internal:4713 \ - --name jukebox jukebox + $ docker compose -f docker/docker-compose.yml -f docker/docker-compose.mac.yml build jukebox + $ docker compose -f docker/docker-compose.yml -f docker/docker-compose.mac.yml run --rm --name jukebox jukebox Resources ^^^^^^^^^^^ diff --git a/src/jukebox/jukebox/playlistgenerator.py b/src/jukebox/jukebox/playlistgenerator.py index b1e12b0de..13dcc0f7d 100755 --- a/src/jukebox/jukebox/playlistgenerator.py +++ b/src/jukebox/jukebox/playlistgenerator.py @@ -191,7 +191,7 @@ def __init__(self, music_library_base_path='/'): but is omitted when generating the playlist entries. I.e. all files in the playlist are relative to this base dir """ self.playlist = [] - self._music_library_base_path = os.path.abspath(music_library_base_path) + self._music_library_base_path = os.path.abspath(os.path.expanduser(music_library_base_path)) # These two variables only store reference content to generate __str__ self._folder = '' self._recursive = False