From 670b6b69668e1640c3188c49cf753edc2b526e97 Mon Sep 17 00:00:00 2001 From: Pablo Romeo Date: Sat, 3 Sep 2022 18:28:03 -0300 Subject: [PATCH] Adding support for EasyAudioEncoder (EAE) audio transcoding (#165) Adding support for EAE. The new default is for it to be enabled but if issues occur it can be turned off by setting the environment variable EAE_SUPPORT in the Workers to "false" Solves #125 and #155 --- .github/workflows/main.yml | 6 +-- README.md | 1 + worker/app/start.sh | 37 ++++++++++++++- worker/app/worker.js | 46 +++++++++++++++++-- .../etc/cont-init.d/92-install-dependencies | 4 +- .../root/etc/cont-init.d/95-setup-codecs-dir | 11 +++++ worker/extended-image/Dockerfile | 4 +- 7 files changed, 96 insertions(+), 13 deletions(-) create mode 100644 worker/docker-mod/root/etc/cont-init.d/95-setup-codecs-dir diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7753ef4..aad5fd1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -78,7 +78,7 @@ jobs: with: context: ./pms file: ./pms/extended-image/Dockerfile - platforms: linux/amd64,linux/arm/v7 # linux/arm64 + platforms: linux/amd64,linux/arm64,linux/arm/v7 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta_pms.outputs.tags }} labels: ${{ steps.meta_pms.outputs.labels }} @@ -147,7 +147,7 @@ jobs: with: context: ./worker file: ./worker/extended-image/Dockerfile - platforms: linux/amd64,linux/arm/v7 # linux/arm64 + platforms: linux/amd64,linux/arm64,linux/arm/v7 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta_worker.outputs.tags }} labels: ${{ steps.meta_worker.outputs.labels }} @@ -215,7 +215,7 @@ jobs: uses: docker/build-push-action@v3 with: context: ./orchestrator - platforms: linux/amd64,linux/arm/v7 # linux/arm64 + platforms: linux/amd64,linux/arm64,linux/arm/v7 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta_orchestrator.outputs.tags }} labels: ${{ steps.meta_orchestrator.outputs.labels }} diff --git a/README.md b/README.md index 79d7cd4..873ca09 100644 --- a/README.md +++ b/README.md @@ -356,6 +356,7 @@ The image extends the [LinuxServer Plex](https://hub.docker.com/r/linuxserver/pl | Parameter | Function | | :----: | --- | | `FFMPEG_HWACCEL` | Allows a [hwaccel decoder](https://trac.ffmpeg.org/wiki/HWAccelIntro) to be passed to ffmpeg such as `nvdec` or `dvxa2` | +| `EAE_SUPPORT` | "true" or "false", controls usage of EasyAudioEncoder (Default = "true") | | `LISTENING_PORT` | Port where workers expose the internal healthcheck | | `STAT_CPU_INTERVAL` | Frequency at which the worker sends stats to the orchestrator (in ms). Default 2000 | | `ORCHESTRATOR_URL` | The url where the orchestrator service can be reached (ex: http://plex-orchestrator:3500) | diff --git a/worker/app/start.sh b/worker/app/start.sh index a579557..9f9ddb6 100644 --- a/worker/app/start.sh +++ b/worker/app/start.sh @@ -5,11 +5,13 @@ cd /usr/lib/plexmediaserver CLUSTERPLEX_PLEX_VERSION=$(strings "pms_original" | grep -P '^([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)-[0-9a-f]{9}') CLUSTERPLEX_PLEX_CODECS_VERSION=$(strings "Plex Transcoder" | grep -Po '[0-9a-f]{7}-[0-9]{4}$') CLUSTERPLEX_PLEX_EAE_VERSION=$(printf "eae-`strings "pms_original" | grep -P '^EasyAudioEncoder-eae-[0-9a-f]{7}-$' | cut -d- -f3`-42") +EAE_VERSION=1785 # fixed for now echo "CLUSTERPLEX_PLEX_VERSION => '${CLUSTERPLEX_PLEX_VERSION}'" echo "CLUSTERPLEX_PLEX_CODECS_VERSION => '${CLUSTERPLEX_PLEX_CODECS_VERSION}'" -echo "CLUSTERPLEX_PLEX_EAE_VERSION => '${CLUSTERPLEX_PLEX_EAE_VERSION}'" +echo "CLUSTERPLEX_PLEX_EAE_VERSION (extracted) => '${CLUSTERPLEX_PLEX_EAE_VERSION}'" echo "PLEX_ARCH => '${PLEX_ARCH}'" +echo "EAE_VERSION => '${EAE_VERSION}'" CLUSTERPLEX_PLEX_CODEC_ARCH="${PLEX_ARCH}" INTERNAL_PLEX_MEDIA_SERVER_INFO_MODEL="" @@ -37,6 +39,36 @@ echo "Codec location => ${CODEC_PATH}" mkdir -p ${CODEC_PATH} cd ${CODEC_PATH} +if [ "$EAE_SUPPORT" == "false" ] +then + echo "EAE_SUPPORT is turned off => ${EAE_SUPPORT}, skipping EasyAudioEncoder download" +else + if [ -d "${CODEC_PATH}/EasyAudioEncoder" ] + then + echo "EasyAudioEncoder already present" + else + echo "Downloading EasyAudioEncoder version => ${EAE_VERSION}" + UUID=$(cat /proc/sys/kernel/random/uuid) + # download eae definition to eae.xml + curl -s -o eae.xml "https://plex.tv/api/codecs/easyaudioencoder?build=${CLUSTERPLEX_PLEX_CODEC_ARCH}&deviceId=${UUID}&oldestPreviousVersion=${CLUSTERPLEX_PLEX_VERSION}&version=${EAE_VERSION}" + + # extract codec url + EAE_CODEC_URL=$(grep -Pio 'Codec url="\K[^"]*' eae.xml) + echo "EAE_CODEC_URL => ${EAE_CODEC_URL}" + echo "Downloading EasyAudioEncoder" + curl -s -o "EasyAudioEncoder-${EAE_VERSION}-${CLUSTERPLEX_PLEX_CODEC_ARCH}.zip" "${EAE_CODEC_URL}" + echo "Decompressing EasyAudioEncoder" + unzip -o "EasyAudioEncoder-${EAE_VERSION}-${CLUSTERPLEX_PLEX_CODEC_ARCH}.zip" -d "EasyAudioEncoder" + # extract license key + echo "Extracting License Key" + EAE_LICENSE_KEY=$(grep -Po 'license="\K([A-Za-z0-9]{10}\s[A-Za-z0-9]{60}\s[A-Za-z0-9]{64})' eae.xml) + EAE_LICENSE_CONTENT="${EAE_LICENSE_KEY}" + EAE_LICENSE_PATH="${CODEC_PATH}/EasyAudioEncoder/EasyAudioEncoder/eae-license.txt" + echo "License Path output => ${EAE_LICENSE_PATH}" + echo $EAE_LICENSE_CONTENT >> $EAE_LICENSE_PATH + fi +fi + #original list: libhevc_decoder libh264_decoder libdca_decoder libac3_decoder libmp3_decoder libaac_decoder libaac_encoder libmpeg4_decoder libmpeg2video_decoder liblibmp3lame_encoder liblibx264_encoder; do cat /app/codecs.txt | while read line do @@ -49,8 +81,9 @@ do fi done -export FFMPEG_EXTERNAL_LIBS="/codecs/${CLUSTERPLEX_PLEX_CODECS_VERSION}-${CLUSTERPLEX_PLEX_CODEC_ARCH}/" +export FFMPEG_EXTERNAL_LIBS="${CODEC_PATH}/" export PLEX_MEDIA_SERVER_INFO_MODEL="${INTERNAL_PLEX_MEDIA_SERVER_INFO_MODEL}" +export EAE_EXECUTABLE="${CODEC_PATH}/EasyAudioEncoder/EasyAudioEncoder/EasyAudioEncoder" cd /app diff --git a/worker/app/worker.js b/worker/app/worker.js index 6618117..1f41ce3 100644 --- a/worker/app/worker.js +++ b/worker/app/worker.js @@ -4,13 +4,21 @@ const STAT_CPU_OPS_DURATION = process.env.STAT_CPU_OPS_DURATION || 1000 const ORCHESTRATOR_URL = process.env.ORCHESTRATOR_URL || 'http://localhost:3500' const TRANSCODER_PATH = process.env.TRANSCODER_PATH || '/usr/lib/plexmediaserver/' const TRANSCODER_NAME = process.env.TRANSCODER_NAME || 'Plex Transcoder' +const EAE_SUPPORT = process.env.EAE_SUPPORT || "true" +const EAE_EXECUTABLE = process.env.EAE_EXECUTABLE || "" // hwaccel decoder: https://trac.ffmpeg.org/wiki/HWAccelIntro const FFMPEG_HWACCEL = process.env.FFMPEG_HWACCEL || false +// Settings debug info +console.log(`EAE_SUPPORT => ${EAE_SUPPORT}`) +console.log(`EAE_EXECUTABLE => ${EAE_EXECUTABLE}`) +console.log(`FFMPEG_HWACCEL => ${FFMPEG_HWACCEL}`) + var app = require('express')(); var server = require('http').createServer(app); var socket = require('socket.io-client')(ORCHESTRATOR_URL); var cpuStat = require('cpu-stat'); +var fs = require('fs'); const { spawn, exec } = require('child_process'); const { v4: uuid } = require('uuid'); const { fib, dist } = require('cpu-benchmark'); @@ -78,7 +86,7 @@ socket.on('worker.task.request', taskRequest => { var processedEnvironmentVariables = processEnv(taskRequest.payload.env) - var child + var child, childEAE if (taskRequest.payload.args[0] === 'testpayload') { console.log(`args => ${JSON.stringify(taskRequest.payload.args)}`) console.log(`env => ${JSON.stringify(processedEnvironmentVariables)}`) @@ -95,13 +103,40 @@ socket.on('worker.task.request', taskRequest => { } } + if (EAE_SUPPORT == "true" && EAE_EXECUTABLE != "") { + if (!fs.existsSync(processedEnvironmentVariables.EAE_ROOT)){ + console.log(`EAE Support - Creating EAE_ROOT destination`) + fs.mkdirSync(processedEnvironmentVariables.EAE_ROOT, { recursive: true }); + } + + console.log(`EAE Support - Spawning EasyAudioEncoder from "${EAE_EXECUTABLE}", cwd => ${processedEnvironmentVariables.EAE_ROOT}`) + childEAE = spawn(EAE_EXECUTABLE, [], { + cwd: processedEnvironmentVariables.EAE_ROOT, + env: processedEnvironmentVariables + }); + childEAE.stdout.pipe(process.stdout); + childEAE.stderr.pipe(process.stderr); + childEAE.on('error', (err) => { + console.error('EAE Support - EAE failed:') + console.error(err) + }) + childEAE.on('close', () => { + console.log('EAE Support - Closing') + }) + childEAE.on('exit', () => { + console.log('EAE Support - Exiting') + }) + } else { + childEAE = null + } + child = spawn(TRANSCODER_PATH + TRANSCODER_NAME, taskRequest.payload.args, { cwd: taskRequest.payload.cwd, env: processedEnvironmentVariables }); } - taskMap.set(taskRequest.taskId, child) + taskMap.set(taskRequest.taskId, { transcodeProcess: child, eaeProcess: childEAE }) child.stdout.pipe(process.stdout); child.stderr.pipe(process.stderr); @@ -140,8 +175,11 @@ socket.on('worker.task.request', taskRequest => { socket.on('worker.task.kill', data => { let taskEntry = taskMap.get(data.taskId) if (taskEntry) { - console.log(`Killing child process for task ${data.taskId}`) - taskEntry.kill() + console.log(`Killing child processes for task ${data.taskId}`) + taskEntry.transcodeProcess.kill() + if (taskEntry.eaeProcess != null) { + taskEntry.eaeProcess.kill() + } console.log('Removing process from taskMap') taskMap.delete(data.taskId) } diff --git a/worker/docker-mod/root/etc/cont-init.d/92-install-dependencies b/worker/docker-mod/root/etc/cont-init.d/92-install-dependencies index c076703..ac0ea50 100644 --- a/worker/docker-mod/root/etc/cont-init.d/92-install-dependencies +++ b/worker/docker-mod/root/etc/cont-init.d/92-install-dependencies @@ -7,8 +7,8 @@ else echo "**** install runtime packages ****" echo "**** apt-get update ****" apt-get update - echo "**** install binutils, libatomic1, file ****" - apt-get install -y binutils libatomic1 file + echo "**** install binutils, libatomic1, file, unzip ****" + apt-get install -y binutils libatomic1 file unzip echo "**** install 'n' ****" curl -L https://raw.githubusercontent.com/tj/n/master/bin/n -o n echo "**** install nodejs ****" diff --git a/worker/docker-mod/root/etc/cont-init.d/95-setup-codecs-dir b/worker/docker-mod/root/etc/cont-init.d/95-setup-codecs-dir new file mode 100644 index 0000000..1372156 --- /dev/null +++ b/worker/docker-mod/root/etc/cont-init.d/95-setup-codecs-dir @@ -0,0 +1,11 @@ +#!/usr/bin/with-contenv bash + +echo "**** Setting up codecs directory ****" +if [ -d "/codecs" ] +then + echo "Directory already present" +else + echo "Creating /codecs and changing ownership" + mkdir -p /codecs + chown abc:abc /codecs +fi diff --git a/worker/extended-image/Dockerfile b/worker/extended-image/Dockerfile index 4a1d54c..eb61261 100644 --- a/worker/extended-image/Dockerfile +++ b/worker/extended-image/Dockerfile @@ -5,8 +5,8 @@ LABEL maintainer="pabloromeo" RUN echo "**** install runtime packages ****" && \ echo "**** apt-get update ****" && \ apt-get update && \ - echo "**** install binutils ****" && \ - apt-get install -y binutils libatomic1 && \ + echo "**** install binutils libatomic1 unzip ****" && \ + apt-get install -y binutils libatomic1 unzip && \ echo "**** install 'n' ****" && \ curl -L https://raw.githubusercontent.com/tj/n/master/bin/n -o n && \ echo "**** install nodejs ****" && \