Add PRIVILEGED_NESTED to the buildah task

Signed-off-by: Wei Shi <wshi@redhat.com>

pipelines/docker-build-multi-platform-oci-ta/README.md

@@ -61,6 +61,7 @@ This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/reposito
 |LABELS| Additional key=value labels that should be applied to the image| []| |
 |PLATFORM| The platform to build on| None| |
 |PREFETCH_INPUT| In case it is not empty, the prefetched content should be made available to the build.| | '$(params.prefetch-input)'|
+|PRIVILEGED_NESTED| Whether to enable privileged mode| false| |
 |SKIP_UNUSED_STAGES| Whether to skip stages in Containerfile that seem unused by subsequent stages| true| |
 |SOURCE_ARTIFACT| The Trusted Artifact URI pointing to the artifact with the application source code.| None| '$(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT)'|
 |SQUASH| Squash all new and previous layers added as a part of this build, as per --squash| false| |

pipelines/docker-build-oci-ta/README.md

@@ -58,6 +58,7 @@ This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/reposito
 |IMAGE_EXPIRES_AFTER| Delete image tag after specified time. Empty means to keep the image tag. Time values could be something like 1h, 2d, 3w for hours, days, and weeks, respectively.| | '$(params.image-expires-after)'|
 |LABELS| Additional key=value labels that should be applied to the image| []| |
 |PREFETCH_INPUT| In case it is not empty, the prefetched content should be made available to the build.| | '$(params.prefetch-input)'|
+|PRIVILEGED_NESTED| Whether to enable privileged mode| false| |
 |SKIP_UNUSED_STAGES| Whether to skip stages in Containerfile that seem unused by subsequent stages| true| |
 |SOURCE_ARTIFACT| The Trusted Artifact URI pointing to the artifact with the application source code.| None| '$(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT)'|
 |SQUASH| Squash all new and previous layers added as a part of this build, as per --squash| false| |

pipelines/docker-build/README.md

@@ -57,6 +57,7 @@ This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/reposito
 |IMAGE_EXPIRES_AFTER| Delete image tag after specified time. Empty means to keep the image tag. Time values could be something like 1h, 2d, 3w for hours, days, and weeks, respectively.| | '$(params.image-expires-after)'|
 |LABELS| Additional key=value labels that should be applied to the image| []| |
 |PREFETCH_INPUT| In case it is not empty, the prefetched content should be made available to the build.| | '$(params.prefetch-input)'|
+|PRIVILEGED_NESTED| Whether to enable privileged mode| false| |
 |SKIP_UNUSED_STAGES| Whether to skip stages in Containerfile that seem unused by subsequent stages| true| |
 |SQUASH| Squash all new and previous layers added as a part of this build, as per --squash| false| |
 |STORAGE_DRIVER| Storage driver to configure for buildah| vfs| |

pipelines/fbc-builder/README.md

@@ -55,6 +55,7 @@ This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/reposito
 |IMAGE_EXPIRES_AFTER| Delete image tag after specified time. Empty means to keep the image tag. Time values could be something like 1h, 2d, 3w for hours, days, and weeks, respectively.| | '$(params.image-expires-after)'|
 |LABELS| Additional key=value labels that should be applied to the image| []| |
 |PREFETCH_INPUT| In case it is not empty, the prefetched content should be made available to the build.| | |
+|PRIVILEGED_NESTED| Whether to enable privileged mode| false| |
 |SKIP_UNUSED_STAGES| Whether to skip stages in Containerfile that seem unused by subsequent stages| true| |
 |SQUASH| Squash all new and previous layers added as a part of this build, as per --squash| false| |
 |STORAGE_DRIVER| Storage driver to configure for buildah| vfs| |

task-generator/remote/main.go

@@ -164,6 +164,7 @@ if ! [[ $IS_LOCALHOST ]]; then
   export BUILD_DIR=$(cat /ssh/user-dir)
   export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
   echo "$BUILD_DIR"
+  # shellcheck disable=SC2086
   ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
 
   PORT_FORWARD=""
@@ -234,13 +235,23 @@ if ! [[ $IS_LOCALHOST ]]; then
 			}
 		}
 		ret += "\nif ! [[ $IS_LOCALHOST ]]; then"
+		ret += "\n"
+		ret += `  PRIVILEGED_NESTED_FLAGS=()
+  if [[ "${PRIVILEGED_NESTED}" == "true" ]]; then
+    # This is a workaround for building bootc images because the cache filesystem (/var/tmp/ on the host) must be a real filesystem that supports setting SELinux security attributes.
+    # https://github.com/coreos/rpm-ostree/discussions/4648
+    # shellcheck disable=SC2086
+    ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/var/tmp"
+    PRIVILEGED_NESTED_FLAGS=(--privileged --mount "type=bind,source=$BUILD_DIR/var/tmp,target=/var/tmp,relabel=shared")
+  fi`
 		ret += "\n  rsync -ra scripts \"$SSH_HOST:$BUILD_DIR\""
 		containerScript := "scripts/script-" + step.Name + ".sh"
 		for _, e := range step.Env {
 			env += "    -e " + e.Name + "=\"$" + e.Name + "\" \\\n"
 		}
 		podmanArgs += "    -v \"$BUILD_DIR/scripts:/scripts:Z\" \\\n"
-		ret += "\n  ssh $SSH_ARGS \"$SSH_HOST\" $PORT_FORWARD podman  run " + env + "" + podmanArgs + "    --user=0  --rm  \"$BUILDER_IMAGE\" /" + containerScript + ` "$@"`
+		podmanArgs += "    \"${PRIVILEGED_NESTED_FLAGS[@]}\" \\\n"
+		ret += "\n  # shellcheck disable=SC2086\n  ssh $SSH_ARGS \"$SSH_HOST\" $PORT_FORWARD podman  run " + env + "" + podmanArgs + "    --user=0  --rm  \"$BUILDER_IMAGE\" /" + containerScript + ` "$@"`
 
 		// Sync the contents of the workspaces back so subsequent tasks can use them
 		for _, workspace := range task.Spec.Workspaces {

task/buildah-oci-ta/0.2/README.md

@@ -23,6 +23,7 @@ When prefetch-dependencies task was activated it is using its artifacts to run b
 |IMAGE_EXPIRES_AFTER|Delete image tag after specified time. Empty means to keep the image tag. Time values could be something like 1h, 2d, 3w for hours, days, and weeks, respectively.|""|false|
 |LABELS|Additional key=value labels that should be applied to the image|[]|false|
 |PREFETCH_INPUT|In case it is not empty, the prefetched content should be made available to the build.|""|false|
+|PRIVILEGED_NESTED|Whether to enable privileged mode|false|false|
 |SKIP_UNUSED_STAGES|Whether to skip stages in Containerfile that seem unused by subsequent stages|true|false|
 |SOURCE_ARTIFACT|The Trusted Artifact URI pointing to the artifact with the application source code.||true|
 |SQUASH|Squash all new and previous layers added as a part of this build, as per --squash|false|false|

task/buildah-oci-ta/0.2/buildah-oci-ta.yaml

@@ -82,6 +82,10 @@ spec:
         be made available to the build.
       type: string
       default: ""
+    - name: PRIVILEGED_NESTED
+      description: Whether to enable privileged mode
+      type: string
+      default: "false"
     - name: SKIP_UNUSED_STAGES
       description: Whether to skip stages in Containerfile that seem unused
         by subsequent stages
@@ -196,6 +200,8 @@ spec:
         value: $(params.IMAGE)
       - name: IMAGE_EXPIRES_AFTER
         value: $(params.IMAGE_EXPIRES_AFTER)
+      - name: PRIVILEGED_NESTED
+        value: $(params.PRIVILEGED_NESTED)
       - name: SKIP_UNUSED_STAGES
         value: $(params.SKIP_UNUSED_STAGES)
       - name: SOURCE_CODE_DIR
@@ -372,6 +378,12 @@ spec:
 
         BUILDAH_ARGS+=("${BUILD_ARG_FLAGS[@]}")
 
+        if [ "${PRIVILEGED_NESTED}" == "true" ]; then
+          BUILDAH_ARGS+=("--security-opt=label=disable")
+          BUILDAH_ARGS+=("--cap-add=all")
+          BUILDAH_ARGS+=("--device=/dev/fuse")
+        fi
+
         if [ -n "${ADD_CAPABILITIES}" ]; then
           BUILDAH_ARGS+=("--cap-add=${ADD_CAPABILITIES}")
         fi

task/buildah-remote-oci-ta/0.1/buildah-remote-oci-ta.yaml

@@ -241,6 +241,7 @@ spec:
         export BUILD_DIR=$(cat /ssh/user-dir)
         export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
         echo "$BUILD_DIR"
+        # shellcheck disable=SC2086
         ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
 
         PORT_FORWARD=""
@@ -446,7 +447,16 @@ spec:
       chmod +x scripts/script-build.sh
 
       if ! [[ $IS_LOCALHOST ]]; then
+        PRIVILEGED_NESTED_FLAGS=()
+        if [[ "${PRIVILEGED_NESTED}" == "true" ]]; then
+          # This is a workaround for building bootc images because the cache filesystem (/var/tmp/ on the host) must be a real filesystem that supports setting SELinux security attributes.
+          # https://github.com/coreos/rpm-ostree/discussions/4648
+          # shellcheck disable=SC2086
+          ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/var/tmp"
+          PRIVILEGED_NESTED_FLAGS=(--privileged --mount "type=bind,source=$BUILD_DIR/var/tmp,target=/var/tmp,relabel=shared")
+        fi
         rsync -ra scripts "$SSH_HOST:$BUILD_DIR"
+        # shellcheck disable=SC2086
         ssh $SSH_ARGS "$SSH_HOST" $PORT_FORWARD podman  run $PODMAN_PORT_FORWARD \
           --tmpfs /run/secrets \
           -e ADDITIONAL_SECRET="$ADDITIONAL_SECRET" \
@@ -477,6 +487,7 @@ spec:
           -v "$BUILD_DIR/.docker/:/root/.docker:Z" \
           -v "$BUILD_DIR/results/:/tekton/results:Z" \
           -v "$BUILD_DIR/scripts:/scripts:Z" \
+          "${PRIVILEGED_NESTED_FLAGS[@]}" \
           --user=0  --rm  "$BUILDER_IMAGE" /scripts/script-build.sh "$@"
         rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/shared/" /shared/
         rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/workdir/" /var/workdir/

task/buildah-remote-oci-ta/0.2/buildah-remote-oci-ta.yaml

@@ -81,6 +81,10 @@ spec:
       to the build.
     name: PREFETCH_INPUT
     type: string
+  - default: "false"
+    description: Whether to enable privileged mode
+    name: PRIVILEGED_NESTED
+    type: string
   - default: "true"
     description: Whether to skip stages in Containerfile that seem unused by subsequent
       stages
@@ -176,6 +180,8 @@ spec:
       value: $(params.IMAGE)
     - name: IMAGE_EXPIRES_AFTER
       value: $(params.IMAGE_EXPIRES_AFTER)
+    - name: PRIVILEGED_NESTED
+      value: $(params.PRIVILEGED_NESTED)
     - name: SKIP_UNUSED_STAGES
       value: $(params.SKIP_UNUSED_STAGES)
     - name: SOURCE_CODE_DIR
@@ -261,6 +267,7 @@ spec:
         export BUILD_DIR=$(cat /ssh/user-dir)
         export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
         echo "$BUILD_DIR"
+        # shellcheck disable=SC2086
         ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
 
         PORT_FORWARD=""
@@ -406,6 +413,12 @@ spec:
 
       BUILDAH_ARGS+=("${BUILD_ARG_FLAGS[@]}")
 
+      if [ "${PRIVILEGED_NESTED}" == "true" ]; then
+        BUILDAH_ARGS+=("--security-opt=label=disable")
+        BUILDAH_ARGS+=("--cap-add=all")
+        BUILDAH_ARGS+=("--device=/dev/fuse")
+      fi
+
       if [ -n "${ADD_CAPABILITIES}" ]; then
         BUILDAH_ARGS+=("--cap-add=${ADD_CAPABILITIES}")
       fi
@@ -575,7 +588,16 @@ spec:
       chmod +x scripts/script-build.sh
 
       if ! [[ $IS_LOCALHOST ]]; then
+        PRIVILEGED_NESTED_FLAGS=()
+        if [[ "${PRIVILEGED_NESTED}" == "true" ]]; then
+          # This is a workaround for building bootc images because the cache filesystem (/var/tmp/ on the host) must be a real filesystem that supports setting SELinux security attributes.
+          # https://github.com/coreos/rpm-ostree/discussions/4648
+          # shellcheck disable=SC2086
+          ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/var/tmp"
+          PRIVILEGED_NESTED_FLAGS=(--privileged --mount "type=bind,source=$BUILD_DIR/var/tmp,target=/var/tmp,relabel=shared")
+        fi
         rsync -ra scripts "$SSH_HOST:$BUILD_DIR"
+        # shellcheck disable=SC2086
         ssh $SSH_ARGS "$SSH_HOST" $PORT_FORWARD podman  run $PODMAN_PORT_FORWARD \
           --tmpfs /run/secrets \
           -e ACTIVATION_KEY="$ACTIVATION_KEY" \
@@ -588,6 +610,7 @@ spec:
           -e HERMETIC="$HERMETIC" \
           -e IMAGE="$IMAGE" \
           -e IMAGE_EXPIRES_AFTER="$IMAGE_EXPIRES_AFTER" \
+          -e PRIVILEGED_NESTED="$PRIVILEGED_NESTED" \
           -e SKIP_UNUSED_STAGES="$SKIP_UNUSED_STAGES" \
           -e SOURCE_CODE_DIR="$SOURCE_CODE_DIR" \
           -e SQUASH="$SQUASH" \
@@ -608,6 +631,7 @@ spec:
           -v "$BUILD_DIR/.docker/:/root/.docker:Z" \
           -v "$BUILD_DIR/results/:/tekton/results:Z" \
           -v "$BUILD_DIR/scripts:/scripts:Z" \
+          "${PRIVILEGED_NESTED_FLAGS[@]}" \
           --user=0  --rm  "$BUILDER_IMAGE" /scripts/script-build.sh "$@"
         rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/shared/" /shared/
         rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/workdir/" /var/workdir/

task/buildah-remote/0.1/buildah-remote.yaml

@@ -229,6 +229,7 @@ spec:
         export BUILD_DIR=$(cat /ssh/user-dir)
         export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
         echo "$BUILD_DIR"
+        # shellcheck disable=SC2086
         ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
 
         PORT_FORWARD=""
@@ -438,7 +439,16 @@ spec:
       chmod +x scripts/script-build.sh
 
       if ! [[ $IS_LOCALHOST ]]; then
+        PRIVILEGED_NESTED_FLAGS=()
+        if [[ "${PRIVILEGED_NESTED}" == "true" ]]; then
+          # This is a workaround for building bootc images because the cache filesystem (/var/tmp/ on the host) must be a real filesystem that supports setting SELinux security attributes.
+          # https://github.com/coreos/rpm-ostree/discussions/4648
+          # shellcheck disable=SC2086
+          ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/var/tmp"
+          PRIVILEGED_NESTED_FLAGS=(--privileged --mount "type=bind,source=$BUILD_DIR/var/tmp,target=/var/tmp,relabel=shared")
+        fi
         rsync -ra scripts "$SSH_HOST:$BUILD_DIR"
+        # shellcheck disable=SC2086
         ssh $SSH_ARGS "$SSH_HOST" $PORT_FORWARD podman  run $PODMAN_PORT_FORWARD \
           --tmpfs /run/secrets \
           -e BUILDAH_FORMAT="$BUILDAH_FORMAT" \
@@ -470,6 +480,7 @@ spec:
           -v "$BUILD_DIR/.docker/:/root/.docker:Z" \
           -v "$BUILD_DIR/results/:/tekton/results:Z" \
           -v "$BUILD_DIR/scripts:/scripts:Z" \
+          "${PRIVILEGED_NESTED_FLAGS[@]}" \
           --user=0  --rm  "$BUILDER_IMAGE" /scripts/script-build.sh "$@"
         rsync -ra "$SSH_HOST:$BUILD_DIR/workspaces/source/" "$(workspaces.source.path)/"
         rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/shared/" /shared/

task/buildah-remote/0.2/buildah-remote.yaml

@@ -119,6 +119,10 @@ spec:
     description: Additional key=value labels that should be applied to the image
     name: LABELS
     type: array
+  - default: "false"
+    description: Whether to enable privileged mode
+    name: PRIVILEGED_NESTED
+    type: string
   - description: The platform to build on
     name: PLATFORM
     type: string
@@ -185,6 +189,8 @@ spec:
       value: $(params.SQUASH)
     - name: SKIP_UNUSED_STAGES
       value: $(params.SKIP_UNUSED_STAGES)
+    - name: PRIVILEGED_NESTED
+      value: $(params.PRIVILEGED_NESTED)
     - name: BUILDER_IMAGE
       value: quay.io/konflux-ci/buildah-task:latest@sha256:b2d6c32d1e05e91920cd4475b2761d58bb7ee11ad5dff3ecb59831c7572b4d0c
     - name: PLATFORM
@@ -243,6 +249,7 @@ spec:
         export BUILD_DIR=$(cat /ssh/user-dir)
         export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
         echo "$BUILD_DIR"
+        # shellcheck disable=SC2086
         ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
 
         PORT_FORWARD=""
@@ -382,6 +389,12 @@ spec:
 
       BUILDAH_ARGS+=("${BUILD_ARG_FLAGS[@]}")
 
+      if [ "${PRIVILEGED_NESTED}" == "true" ]; then
+        BUILDAH_ARGS+=("--security-opt=label=disable")
+        BUILDAH_ARGS+=("--cap-add=all")
+        BUILDAH_ARGS+=("--device=/dev/fuse")
+      fi
+
       if [ -n "${ADD_CAPABILITIES}" ]; then
         BUILDAH_ARGS+=("--cap-add=${ADD_CAPABILITIES}")
       fi
@@ -552,7 +565,16 @@ spec:
       chmod +x scripts/script-build.sh
 
       if ! [[ $IS_LOCALHOST ]]; then
+        PRIVILEGED_NESTED_FLAGS=()
+        if [[ "${PRIVILEGED_NESTED}" == "true" ]]; then
+          # This is a workaround for building bootc images because the cache filesystem (/var/tmp/ on the host) must be a real filesystem that supports setting SELinux security attributes.
+          # https://github.com/coreos/rpm-ostree/discussions/4648
+          # shellcheck disable=SC2086
+          ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/var/tmp"
+          PRIVILEGED_NESTED_FLAGS=(--privileged --mount "type=bind,source=$BUILD_DIR/var/tmp,target=/var/tmp,relabel=shared")
+        fi
         rsync -ra scripts "$SSH_HOST:$BUILD_DIR"
+        # shellcheck disable=SC2086
         ssh $SSH_ARGS "$SSH_HOST" $PORT_FORWARD podman  run $PODMAN_PORT_FORWARD \
           --tmpfs /run/secrets \
           -e BUILDAH_FORMAT="$BUILDAH_FORMAT" \
@@ -574,6 +596,7 @@ spec:
           -e ADD_CAPABILITIES="$ADD_CAPABILITIES" \
           -e SQUASH="$SQUASH" \
           -e SKIP_UNUSED_STAGES="$SKIP_UNUSED_STAGES" \
+          -e PRIVILEGED_NESTED="$PRIVILEGED_NESTED" \
           -e COMMIT_SHA="$COMMIT_SHA" \
           -e DOCKERFILE="$DOCKERFILE" \
           -v "$BUILD_DIR/workspaces/source:$(workspaces.source.path):Z" \
@@ -585,6 +608,7 @@ spec:
           -v "$BUILD_DIR/.docker/:/root/.docker:Z" \
           -v "$BUILD_DIR/results/:/tekton/results:Z" \
           -v "$BUILD_DIR/scripts:/scripts:Z" \
+          "${PRIVILEGED_NESTED_FLAGS[@]}" \
           --user=0  --rm  "$BUILDER_IMAGE" /scripts/script-build.sh "$@"
         rsync -ra "$SSH_HOST:$BUILD_DIR/workspaces/source/" "$(workspaces.source.path)/"
         rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/shared/" /shared/

task/buildah/0.2/buildah.yaml

@@ -107,6 +107,10 @@ spec:
     description: Additional key=value labels that should be applied to the image
     type: array
     default: []
+  - name: PRIVILEGED_NESTED
+    description: Whether to enable privileged mode
+    type: string
+    default: "false"
 
   results:
   - description: Digest of the image just built
@@ -166,6 +170,8 @@ spec:
       value: $(params.SQUASH)
     - name: SKIP_UNUSED_STAGES
       value: $(params.SKIP_UNUSED_STAGES)
+    - name: PRIVILEGED_NESTED
+      value: $(params.PRIVILEGED_NESTED)
 
   steps:
   - image: quay.io/konflux-ci/buildah-task:latest@sha256:b2d6c32d1e05e91920cd4475b2761d58bb7ee11ad5dff3ecb59831c7572b4d0c
@@ -303,6 +309,12 @@ spec:
 
       BUILDAH_ARGS+=("${BUILD_ARG_FLAGS[@]}")
 
+      if [ "${PRIVILEGED_NESTED}" == "true" ]; then
+        BUILDAH_ARGS+=("--security-opt=label=disable")
+        BUILDAH_ARGS+=("--cap-add=all")
+        BUILDAH_ARGS+=("--device=/dev/fuse")
+      fi
+
       if [ -n "${ADD_CAPABILITIES}" ]; then
         BUILDAH_ARGS+=("--cap-add=${ADD_CAPABILITIES}")
       fi