Merge remote-tracking branch 'upstream/master'

.github/workflows/build.yml

@@ -30,7 +30,13 @@ jobs:
           password: ${{ github.token }}
       - name: Image Name
         id: imagename
-        run: echo "name=ghcr.io/${GITHUB_REPOSITORY,,}/base" >> $GITHUB_OUTPUT
+        run: |
+          IMG="${GITHUB_REPOSITORY,,}/base"
+          DLIMG="${GITHUB_REPOSITORY,,}/dl_cache"
+          echo "name=ghcr.io/${IMG/ /-}" >> $GITHUB_OUTPUT
+          echo "rawname=${IMG/ /-}" >> $GITHUB_OUTPUT
+          echo "dlname=ghcr.io/${DLIMG/ /-}" >> $GITHUB_OUTPUT
+          echo "dlrawname=${DLIMG/ /-}" >> $GITHUB_OUTPUT
       - name: Stop Commands
         run: T="$(echo -n ${{ github.token }} | sha256sum | head -c 64)" && echo -e "::add-mask::${T}\n::stop-commands::${T}"
       - name: Build target base image
@@ -42,6 +48,22 @@ jobs:
           tags: ${{ steps.imagename.outputs.name }}:latest
           cache-to: type=registry,mode=max,ref=${{ steps.imagename.outputs.name }}:cache
           cache-from: type=registry,ref=${{ steps.imagename.outputs.name }}:cache
+      - name: Generate download cache Dockerfile
+        id: dl_cache
+        run: |
+          ./generate.sh dl only
+          echo "dltagname=$(./util/get_dl_cache_tag.sh)" >> $GITHUB_OUTPUT
+          cat Dockerfile.dl
+      - name: Build download cache image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: Dockerfile.dl
+          pull: true
+          push: true
+          tags: ${{ steps.imagename.outputs.dlname }}:${{ steps.dl_cache.outputs.dltagname }}
+          cache-to: type=registry,mode=max,ref=${{ steps.imagename.outputs.dlname }}:cache
+          cache-from: type=registry,ref=${{ steps.imagename.outputs.dlname }}:cache
   build_target_bases:
     name: Build target base image
     needs: build_base
@@ -69,7 +91,11 @@ jobs:
           password: ${{ github.token }}
       - name: Image Name
         id: imagename
-        run: echo "name=ghcr.io/${GITHUB_REPOSITORY,,}/base-${{ matrix.target }}" >> $GITHUB_OUTPUT
+        run: |
+          IMG="${GITHUB_REPOSITORY,,}/base-${{ matrix.target }}"
+          echo "name=ghcr.io/${IMG/ /-}" >> $GITHUB_OUTPUT
+          echo "rawname=${IMG/ /-}" >> $GITHUB_OUTPUT
+          echo "gh_repo=${GITHUB_REPOSITORY,,}" >> $GITHUB_OUTPUT
       - name: Stop Commands
         run: T="$(echo -n ${{ github.token }} | sha256sum | head -c 64)" && echo -e "::add-mask::${T}\n::stop-commands::${T}"
       - name: Build target base image
@@ -81,6 +107,8 @@ jobs:
           tags: ${{ steps.imagename.outputs.name }}:latest
           cache-to: type=registry,mode=max,ref=${{ steps.imagename.outputs.name }}:cache
           cache-from: type=registry,ref=${{ steps.imagename.outputs.name }}:cache
+          build-args: |
+            GH_REPO=ghcr.io/${{ steps.imagename.outputs.gh_repo }}
   build_targets:
     name: Build target-variant image
     needs: build_target_bases
@@ -113,9 +141,9 @@ jobs:
       - name: Image Name
         id: imagename
         run: |
-          IMG="ghcr.io/${GITHUB_REPOSITORY,,}/${{ matrix.target }}-${{ matrix.variant }}"
-          echo "name=${IMG/ /-}" >> $GITHUB_OUTPUT
-          echo "gh_repo=${GITHUB_REPOSITORY,,}" >> $GITHUB_OUTPUT
+          IMG="${GITHUB_REPOSITORY,,}/${{ matrix.target }}-${{ matrix.variant }}"
+          echo "name=ghcr.io/${IMG/ /-}" >> $GITHUB_OUTPUT
+          echo "rawname=${IMG/ /-}" >> $GITHUB_OUTPUT
       - name: Stop Commands
         run: T="$(echo -n ${{ github.token }} | sha256sum | head -c 64)" && echo -e "::add-mask::${T}\n::stop-commands::${T}"
       - name: Build target base image
@@ -127,8 +155,6 @@ jobs:
           tags: ${{ steps.imagename.outputs.name }}:latest
           cache-to: type=registry,mode=max,ref=${{ steps.imagename.outputs.name }}:cache
           cache-from: type=registry,ref=${{ steps.imagename.outputs.name }}:cache
-          build-args: |
-            GH_REPO=ghcr.io/${{ steps.imagename.outputs.gh_repo }}
   build_ffmpeg:
     name: Build ffmpeg
     needs: build_targets

.gitignore

@@ -1,4 +1,4 @@
-/Dockerfile
+/Dockerfile*
 /ffbuild/
 /artifacts/
 /.cache/

build.sh

@@ -100,10 +100,10 @@ EOF
 cd ffbuild/pkgroot
 if [[ "${TARGET}" == win* ]]; then
     OUTPUT_FNAME="${BUILD_NAME}.zip"
-    zip -9 -r "${ARTIFACTS_PATH}/${OUTPUT_FNAME}" "$BUILD_NAME"
+    docker run --rm -i $TTY_ARG "${UIDARGS[@]}" -v "${ARTIFACTS_PATH}":/out -v "${PWD}/${BUILD_NAME}":"/${BUILD_NAME}" -w / "$IMAGE" zip -9 -r "/out/${OUTPUT_FNAME}" "$BUILD_NAME"
 else
     OUTPUT_FNAME="${BUILD_NAME}.tar.xz"
-    tar cJf "${ARTIFACTS_PATH}/${OUTPUT_FNAME}" "$BUILD_NAME"
+    docker run --rm -i $TTY_ARG "${UIDARGS[@]}" -v "${ARTIFACTS_PATH}":/out -v "${PWD}/${BUILD_NAME}":"/${BUILD_NAME}" -w / "$IMAGE" tar cJf "/out/${OUTPUT_FNAME}" "$BUILD_NAME"
 fi
 cd -
 

generate.sh

@@ -3,28 +3,82 @@ set -e
 cd "$(dirname "$0")"
 source util/vars.sh
 
-rm -f Dockerfile
+export LC_ALL=C.UTF-8
+
+rm -f Dockerfile Dockerfile.{dl,final,dl.final}
 
 layername() {
     printf "layer-"
     basename "$1" | sed 's/.sh$//'
 }
 
+to_df() {
+    _of="${TODF:-Dockerfile}"
+    printf "$@" >> "$_of"
+    echo >> "$_of"
+}
+
+default_dl() {
+    to_df "RUN git-mini-clone \"$SCRIPT_REPO\" \"$SCRIPT_COMMIT\" \"$1\""
+}
+
+###
+### Generate download Dockerfile
+###
+
+exec_dockerstage_dl() {
+    SCRIPT="$1"
+    (
+        SELF="$SCRIPT"
+        SELFLAYER="$(layername "$STAGE")"
+        source "$SCRIPT"
+        ffbuild_dockerdl || exit $?
+        TODF="Dockerfile.dl.final" ffbuild_dockerlayer_dl || exit $?
+    )
+}
+
+export TODF="Dockerfile.dl"
+
+to_df "FROM ${REGISTRY}/${REPO}/base:latest AS base"
+to_df "ENV TARGET=$TARGET VARIANT=$VARIANT REPO=$REPO ADDINS_STR=$ADDINS_STR"
+to_df "WORKDIR \$FFBUILD_DLDIR"
+
+for STAGE in scripts.d/*.sh scripts.d/*/*.sh; do
+    to_df "FROM base AS $(layername "$STAGE")"
+    exec_dockerstage_dl "$STAGE"
+done
+
+to_df "FROM base AS intermediate"
+cat Dockerfile.dl.final >> "$TODF"
+rm Dockerfile.dl.final
+
+to_df "FROM scratch"
+to_df "COPY --from=intermediate /opt/ffdl/. /"
+
+if [[ "$TARGET" == "dl" && "$VARIANT" == "only" ]]; then
+    exit 0
+fi
+
+DL_IMAGE="${DL_IMAGE_RAW}:$(./util/get_dl_cache_tag.sh)"
+
+###
+### Generate main Dockerfile
+###
+
 exec_dockerstage() {
     SCRIPT="$1"
     (
         SELF="$SCRIPT"
         source "$SCRIPT"
+
         ffbuild_enabled || exit 0
+
+        to_df "ENV SELF=\"$SELF\""
         ffbuild_dockerstage || exit $?
     )
 }
 
-to_df() {
-    _of="${TODF:-Dockerfile}"
-    printf "$@" >> "$_of"
-    echo >> "$_of"
-}
+export TODF="Dockerfile"
 
 to_df "FROM ${REGISTRY}/${REPO}/base-${TARGET}:latest AS base"
 to_df "ENV TARGET=$TARGET VARIANT=$VARIANT REPO=$REPO ADDINS_STR=$ADDINS_STR"