Skip to content

Commit

Permalink
02.Version cache - docker cache build framework
Browse files Browse the repository at this point in the history
During docker build, host files can be passed to the docker build through
docker context files. But there is no straightforward way to transfer
the files from docker build to host.

This feature provides a tricky way to pass the cache contents from docker
build to host. It uses the multi-stage docker file to copy the cache
content, cleanup the temporary files and creates the final sonic docker image.
  • Loading branch information
Kalimuthu-Velappan committed Dec 1, 2022
1 parent 11d579c commit d5edc7b
Showing 11 changed files with 82 additions and 16 deletions.
13 changes: 9 additions & 4 deletions Makefile.work
Original file line number Diff line number Diff line change
@@ -228,8 +228,12 @@ SLAVE_TAG = $(shell \

COLLECT_DOCKER=DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
scripts/collect_docker_version_files.sh \
$(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) \
target
$(SLAVE_BASE_IMAGE) \
target \
$(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) \
$(SLAVE_DIR) \
$(SLAVE_DIR)/Dockerfile

OVERLAY_MODULE_CHECK := \
lsmod | grep -q "^overlay " &>/dev/null || \
zgrep -q 'CONFIG_OVERLAY_FS=y' /proc/config.gz &>/dev/null || \
@@ -376,7 +380,8 @@ DOCKER_SLAVE_BASE_BUILD = docker build --no-cache \
--build-arg http_proxy=$(http_proxy) \
--build-arg https_proxy=$(https_proxy) \
--build-arg no_proxy=$(no_proxy) \
$(SLAVE_DIR) $(SPLIT_LOG) $(DOCKER_BASE_LOG)
$(SLAVE_DIR) \
$(SPLIT_LOG) $(DOCKER_BASE_LOG)

DOCKER_BASE_PULL = docker pull \
$(REGISTRY_SERVER):$(REGISTRY_PORT)/$(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG)
@@ -507,7 +512,7 @@ SONIC_BUILD_INSTRUCTION := $(MAKE) \


ifeq ($(filter clean,$(MAKECMDGOALS)),)
COLLECT_BUILD_VERSION = { DBGOPT='$(DBGOPT)' scripts/collect_build_version_files.sh \$$?; }
COLLECT_BUILD_VERSION = { scripts/collect_build_version_files.sh \$$?; }
endif

ifdef SOURCE_FOLDER
11 changes: 11 additions & 0 deletions files/build_templates/build_docker_cache.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# Base docker build
FROM {{IMAGENAME}}

# Copy the cache data to host
From scratch as output
COPY --from={{IMAGENAME}} /cache.tgz cache.tgz

# Clean up the cache data
FROM {{IMAGENAME}} as final
RUN rm /cache.tgz

2 changes: 1 addition & 1 deletion scripts/collect_build_version_files.sh
Original file line number Diff line number Diff line change
@@ -23,6 +23,6 @@ mkdir -p $VERSION_SLAVE_PATH

scripts/versions_manager.py merge -t $VERSION_SLAVE_PATH -b $LOG_VERSION_PATH -e $POST_VERSION_PATH

rm -rf $BUILD_VERSION_PATH/*
[ -d $BUILD_VERSION_PATH ] && rm -rf $BUILD_VERSION_PATH/*

exit $RET
17 changes: 15 additions & 2 deletions scripts/collect_docker_version_files.sh
Original file line number Diff line number Diff line change
@@ -1,12 +1,17 @@
#!/bin/bash

set -x
DOCKER_IMAGE=$1
TARGET_PATH=$2
DOCKER_IMAGE_TAG=$3
DOCKER_PATH=$4
DOCKER_FILE=$5

[ -z "$TARGET_PATH" ] && TARGET_PATH=./target

DOCKER_IMAGE_NAME=$(echo $DOCKER_IMAGE | cut -d: -f1 | sed "s/-$DOCKER_USERNAME\$//")
DOCKER_CONTAINER=$DOCKER_IMAGE_NAME
#Create the container specific to the user tag and slave tag
DOCKER_CONTAINER=${DOCKER_IMAGE_TAG/:/-}
TARGET_VERSIONS_PATH=$TARGET_PATH/versions/dockers/$DOCKER_IMAGE_NAME

[ -d $TARGET_VERSIONS_PATH ] && rm -rf $TARGET_VERSIONS_PATH
@@ -18,8 +23,16 @@ export DOCKER_CLI_EXPERIMENTAL=enabled
if docker container inspect $DOCKER_IMAGE > /dev/null 2>&1; then
docker container rm $DOCKER_IMAGE > /dev/null
fi
docker create --name $DOCKER_CONTAINER --entrypoint /bin/bash $DOCKER_IMAGE
docker create --name $DOCKER_CONTAINER --entrypoint /bin/bash $DOCKER_IMAGE_TAG
docker cp -L $DOCKER_CONTAINER:/etc/os-release $TARGET_VERSIONS_PATH/
docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/pre-versions $TARGET_VERSIONS_PATH/
docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/post-versions $TARGET_VERSIONS_PATH/

# Save the cache contents from docker build
IMAGENAME=${DOCKER_IMAGE_TAG} j2 files/build_templates/build_docker_cache.j2 > ${DOCKER_FILE}.cleanup
docker tag ${DOCKER_IMAGE_TAG} tmp-${DOCKER_IMAGE_TAG}
DOCKER_BUILDKIT=1 docker build -f ${DOCKER_PATH}/Dockerfile.cleanup --target output -o target/vcache/${DOCKER_IMAGE_NAME} ${DOCKER_PATH}
DOCKER_BUILDKIT=1 docker build -f ${DOCKER_PATH}/Dockerfile.cleanup --no-cache --target final --tag ${DOCKER_IMAGE_TAG} ${DOCKER_PATH}
docker rmi tmp-${DOCKER_IMAGE_TAG}

docker container rm $DOCKER_CONTAINER
4 changes: 3 additions & 1 deletion scripts/collect_host_image_version_files.sh
Original file line number Diff line number Diff line change
@@ -3,11 +3,13 @@
TARGET=$1
FILESYSTEM_ROOT=$2
VERSIONS_PATH=$TARGET/versions/host-image
IMAGENAME="host-image"

[ -d $VERSIONS_PATH ] && sudo rm -rf $VERSIONS_PATH
mkdir -p $VERSIONS_PATH

sudo LANG=C chroot $FILESYSTEM_ROOT post_run_buildinfo
mkdir -p target/vcache/${IMAGENAME}
sudo LANG=C chroot $FILESYSTEM_ROOT post_run_buildinfo ${IMAGENAME}

cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/pre-versions $VERSIONS_PATH/
cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/post-versions $VERSIONS_PATH/
13 changes: 11 additions & 2 deletions scripts/prepare_docker_buildinfo.sh
Original file line number Diff line number Diff line change
@@ -11,6 +11,7 @@ DISTRO=$5
DOCKERFILE_PATH=$(dirname "$DOCKERFILE_TARGE")
BUILDINFO_PATH="${DOCKERFILE_PATH}/buildinfo"
BUILDINFO_VERSION_PATH="${BUILDINFO_PATH}/versions"
DOCKER_PATH=$(dirname $DOCKERFILE)

[ -d $BUILDINFO_PATH ] && rm -rf $BUILDINFO_PATH
mkdir -p $BUILDINFO_VERSION_PATH
@@ -31,8 +32,11 @@ scripts/docker_version_control.sh $@

DOCKERFILE_PRE_SCRIPT='# Auto-Generated for buildinfo
COPY ["buildinfo", "/usr/local/share/buildinfo"]
COPY vcache/ /sonic/target/vcache/'${IMAGENAME}'
RUN dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb
RUN pre_run_buildinfo'
ENV IMAGENAME='${IMAGENAME}'
RUN pre_run_buildinfo '${IMAGENAME}'
'

# Add the auto-generate code if it is not added in the target Dockerfile
if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKERFILE_TARGE; then
@@ -42,7 +46,7 @@ if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKE
awk -v text="${DOCKERFILE_PRE_SCRIPT}" -v linenumber=$LINE_NUMBER 'NR==linenumber{print text}1' $DOCKERFILE > $TEMP_FILE

# Append the docker build script at the end of the docker file
echo -e "\nRUN post_run_buildinfo" >> $TEMP_FILE
echo -e "\nRUN post_run_buildinfo ${IMAGENAME} " >> $TEMP_FILE

cat $TEMP_FILE > $DOCKERFILE_TARGE
rm -f $TEMP_FILE
@@ -55,3 +59,8 @@ cp -rf src/sonic-build-hooks/buildinfo/* $BUILDINFO_PATH
scripts/versions_manager.py generate -t "$BUILDINFO_VERSION_PATH" -n "$IMAGENAME" -d "$DISTRO" -a "$ARCH"

touch $BUILDINFO_VERSION_PATH/versions-deb

# Create the cache directories
LOCAL_CACHE_DIR=target/vcache/${IMAGENAME}
mkdir -p ${LOCAL_CACHE_DIR} ${DOCKER_PATH}/vcache/
chmod -f 777 ${LOCAL_CACHE_DIR} ${DOCKER_PATH}/vcache/
15 changes: 12 additions & 3 deletions slave.mk
Original file line number Diff line number Diff line change
@@ -111,6 +111,7 @@ configure :
$(Q)mkdir -p $(PYTHON_DEBS_PATH)
$(Q)mkdir -p $(PYTHON_WHEELS_PATH)
$(Q)mkdir -p $(DPKG_ADMINDIR_PATH)
$(Q)mkdir -p $(TARGET_PATH)/vcache
$(Q)echo $(PLATFORM) > .platform
$(Q)echo $(PLATFORM_ARCH) > .arch

@@ -914,8 +915,10 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_SIMPLE_DOCKER_IMAGES)) : $(TARGET_PATH)/%.g
--label Tag=$(SONIC_IMAGE_VERSION) \
-f $(TARGET_DOCKERFILE)/Dockerfile.buildinfo \
-t $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)

if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi
scripts/collect_docker_version_files.sh $(DOCKER_IMAGE_REF) $(TARGET_PATH)
scripts/collect_docker_version_files.sh $* $(TARGET_PATH) $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)

$(call docker-image-save,$*,$@)
# Clean up
if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi
@@ -993,6 +996,7 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform
mkdir -p $($*.gz_PATH)/files $(LOG)
mkdir -p $($*.gz_PATH)/python-debs $(LOG)
mkdir -p $($*.gz_PATH)/python-wheels $(LOG)
mkdir -p $(TARGET_PATH)/vcache/$* $($*.gz_PATH)/vcache $(LOG)
sudo mount --bind $($*.gz_DEBS_PATH) $($*.gz_PATH)/debs $(LOG)
sudo mount --bind $($*.gz_FILES_PATH) $($*.gz_PATH)/files $(LOG)
sudo mount --bind $(PYTHON_DEBS_PATH) $($*.gz_PATH)/python-debs $(LOG)
@@ -1034,9 +1038,11 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform
--label Tag=$(SONIC_IMAGE_VERSION) \
$($(subst -,_,$(notdir $($*.gz_PATH)))_labels) \
-t $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)

if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi
scripts/collect_docker_version_files.sh $(DOCKER_IMAGE_REF) $(TARGET_PATH)
scripts/collect_docker_version_files.sh $* $(TARGET_PATH) $(DOCKER_IMAGE_REF) $($*.gz_PATH) $($*.gz_PATH)/Dockerfile $(LOG)
if [ ! -z $(filter $*.gz,$(SONIC_PACKAGES_LOCAL)) ]; then docker tag $(DOCKER_IMAGE_REF) $*:$(SONIC_IMAGE_VERSION); fi

$(call docker-image-save,$*,$@)
# Clean up
if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi
@@ -1065,6 +1071,7 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG

mkdir -p $($*.gz_PATH)/debs $(LOG)
sudo mount --bind $($*.gz_DEBS_PATH) $($*.gz_PATH)/debs $(LOG)
mkdir -p $(TARGET_PATH)/vcache/$*-dbg $($*.gz_PATH)/vcache $(LOG)
# Export variables for j2. Use path for unique variable names, e.g. docker_orchagent_debs
$(eval export $(subst -,_,$(notdir $($*.gz_PATH)))_dbg_debs=$(shell printf "$(subst $(SPACE),\n,$(call expand,$($*.gz_DBG_DEPENDS),RDEPENDS))\n" | awk '!a[$$0]++'))
$(eval export $(subst -,_,$(notdir $($*.gz_PATH)))_image_dbgs=$(shell printf "$(subst $(SPACE),\n,$(call expand,$($*.gz_DBG_IMAGE_PACKAGES)))\n" | awk '!a[$$0]++'))
@@ -1088,9 +1095,11 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG
--label Tag=$(SONIC_IMAGE_VERSION) \
--file $($*.gz_PATH)/Dockerfile-dbg \
-t $(DOCKER_DBG_IMAGE_REF) $($*.gz_PATH) $(LOG)

if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi
scripts/collect_docker_version_files.sh $(DOCKER_DBG_IMAGE_REF) $(TARGET_PATH)
scripts/collect_docker_version_files.sh $*-dbg $(TARGET_PATH) $(DOCKER_DBG_IMAGE_REF) $($*.gz_PATH) $($*.gz_PATH)/Dockerfile-dbg $(LOG)
if [ ! -z $(filter $*.gz,$(SONIC_PACKAGES_LOCAL)) ]; then docker tag $(DOCKER_IMAGE_REF) $*:$(SONIC_IMAGE_VERSION); fi

$(call docker-image-save,$*-$(DBG_IMAGE_MARK),$@)
# Clean up
docker rmi -f $(DOCKER_IMAGE_REF) &> /dev/null || true
2 changes: 1 addition & 1 deletion sonic-slave-buster/Dockerfile.j2
Original file line number Diff line number Diff line change
@@ -599,7 +599,7 @@ RUN add-apt-repository \
$(lsb_release -cs) \
stable"
RUN apt-get update
RUN apt-get install -y docker-ce=5:18.09.5~3-0~debian-buster docker-ce-cli=5:18.09.5~3-0~debian-buster
RUN apt-get install -y docker-ce=5:20.10.21~3-0~debian-buster docker-ce-cli=5:20.10.21~3-0~debian-buster
RUN echo "DOCKER_OPTS=\"--experimental --storage-driver=vfs {{ DOCKER_EXTRA_OPTS }}\"" >> /etc/default/docker
RUN update-alternatives --set iptables /usr/sbin/iptables-legacy
9 changes: 9 additions & 0 deletions src/sonic-build-hooks/scripts/buildinfo_base.sh
Original file line number Diff line number Diff line change
@@ -15,6 +15,15 @@ REPR_MIRROR_URL_PATTERN='http:\/\/packages.trafficmanager.net\/debian'
DPKG_INSTALLTION_LOCK_FILE=/tmp/.dpkg_installation.lock

. $BUILDINFO_PATH/config/buildinfo.config
if [ -e /vcache ]; then
PKG_CACHE_PATH=/vcache/${IMAGENAME}
else
PKG_CACHE_PATH=/sonic/target/vcache/${IMAGENAME}
fi
PKG_CACHE_FILE_NAME=${PKG_CACHE_PATH}/cache.tgz
mkdir -p ${PKG_CACHE_PATH}



URL_PREFIX=$(echo "${PACKAGE_URL_PREFIX}" | sed -E "s#(//[^/]*/).*#\1#")

6 changes: 4 additions & 2 deletions src/sonic-build-hooks/scripts/post_run_buildinfo
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
#!/bin/bash

IMAGENAME=$1

. /usr/local/share/buildinfo/scripts/buildinfo_base.sh

# Collect the version files
collect_version_files $POST_VERSION_PATH

[ -d $BUILD_VERSION_PATH ] && [ ! -z "$(ls -A $BUILD_VERSION_PATH)" ] && cp -rf $BUILD_VERSION_PATH/* $POST_VERSION_PATH
rm -rf $BUILD_VERSION_PATH/*
#Save the cache file for exporting it to host.
tar -C ${PKG_CACHE_PATH} --exclude=cache.tgz -zcvf /cache.tgz .

# Disable the build hooks
symlink_build_hooks -d
6 changes: 6 additions & 0 deletions src/sonic-build-hooks/scripts/pre_run_buildinfo
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
#!/bin/bash

IMAGENAME=$1

. /usr/local/share/buildinfo/scripts/buildinfo_base.sh

[ -d $DIFF_VERSION_PATH ] && rm -rf $DIFF_VERSION_PATH
@@ -15,10 +17,14 @@ update_version_files
symlink_build_hooks
set_reproducible_mirrors

mkdir -p /var/cache/apt/archives/

chmod -R a+rw $BUILDINFO_PATH

if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ] && [ -f $VERSION_DEB_PREFERENCE ]; then
cp -f $VERSION_DEB_PREFERENCE /etc/apt/preferences.d/
fi

DISTRO=${DISTRO} apt-get update && apt-get install -y rsync

exit 0

0 comments on commit d5edc7b

Please sign in to comment.