05.Version cache - docker dpkg caching support

This feature caches all the deb files during docker build and stores them
into version cache.

It loads the cache file if already exists in the version cache and copies the extracted
deb file from cache file into Debian cache path( /var/cache/apt/archives).

The apt-install always installs the deb file from the cache if exists, this
avoid unnecessary package download from the repo and speeds up the overall build.

The cache file is selected based on the SHA value of version dependency
files.

Makefile.work

@@ -234,7 +234,6 @@ SLAVE_TAG = $(shell \
 	(cat $(SLAVE_DIR)/Dockerfile.user \
 		 $(SLAVE_DIR)/Dockerfile \
 		 $(SLAVE_DIR)/buildinfo/versions/versions-* \
-		.git/HEAD 2>/dev/null \
 	&& echo $(USER)/$(PWD)/$(CONFIGURED_PLATFORM)) \
 	| sha1sum \
 	| awk '{print substr($$1,0,11);}')

build_debian.sh

@@ -443,10 +443,10 @@ if [[ $TARGET_BOOTLOADER == grub ]]; then
         GRUB_PKG=grub-efi-arm64-bin
     fi
 
-    sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y download \
+    sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get install -d -o dir::cache=/var/cache/apt \
         $GRUB_PKG
 
-    sudo mv $FILESYSTEM_ROOT/grub*.deb $FILESYSTEM_ROOT/$PLATFORM_DIR/grub
+    sudo cp $FILESYSTEM_ROOT/var/cache/apt/archives/grub*.deb $FILESYSTEM_ROOT/$PLATFORM_DIR/grub
 fi
 
 ## Disable kexec supported reboot which was installed by default

platform/broadcom/sai.mk

@@ -17,4 +17,9 @@ $(BRCM_DNX_SAI)_URL = "$(LIBSAIBCM_DNX_URL_PREFIX)/$(BRCM_DNX_SAI)"
 SONIC_ONLINE_DEBS += $(BRCM_XGS_SAI)
 SONIC_ONLINE_DEBS += $(BRCM_DNX_SAI)
 $(BRCM_XGS_SAI_DEV)_DEPENDS += $(BRCM_XGS_SAI)
+
+$(BRCM_XGS_SAI)_SKIP_VERSION=y
+$(BRCM_XGS_SAI_DEV)_SKIP_VERSION=y
+$(BRCM_DNX_SAI)_SKIP_VERSION=y
+
 $(eval $(call add_conflict_package,$(BRCM_XGS_SAI_DEV),$(LIBSAIVS_DEV)))

scripts/collect_docker_version_files.sh

@@ -48,4 +48,55 @@ DOCKER_BUILDKIT=1 docker build -f ${DOCKER_PATH}/Dockerfile.cleanup  --no-cache
 docker rmi tmp-${DOCKER_IMAGE_TAG}
 docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/log ${BUILD_LOG_PATH}/
 
+
+# Save the cache contents from docker build
+LOCAL_CACHE_FILE=target/vcache/${DOCKER_IMAGE_NAME}/cache.tgz
+CACHE_ENCODE_FILE=${DOCKER_PATH}/vcache/cache.base64
+sleep 1; sync ${CACHE_ENCODE_FILE}
+
+# Decode the cache content into gz format
+SRC_VERSION_PATH=files/build/versions
+if [[ -e ${CACHE_ENCODE_FILE} ]]; then
+
+	cat ${CACHE_ENCODE_FILE} | base64 -d  >${LOCAL_CACHE_FILE}
+	rm -f ${CACHE_ENCODE_FILE}
+fi
+
+# Version package cache
+IMAGE_DBGS_NAME=${DOCKER_IMAGE_NAME//-/_}_image_dbgs
+if [[ ${DOCKER_IMAGE_NAME} == sonic-slave-* ]]; then
+	GLOBAL_CACHE_DIR=${SONIC_VERSION_CACHE_SOURCE}/${DOCKER_IMAGE_NAME}
+else
+	GLOBAL_CACHE_DIR=/vcache/${DOCKER_IMAGE_NAME}
+fi
+
+if [[ ! -z ${SONIC_VERSION_CACHE} && -e ${CACHE_ENCODE_FILE} ]]; then
+
+	# Select version files for SHA calculation
+	VERSION_FILES="${SRC_VERSION_PATH}/dockers/${DOCKER_IMAGE_NAME}/versions-*-${DISTRO}-${ARCH} ${SRC_VERSION_PATH}/default/versions-*"
+	DEP_FILES="${DOCKER_PATH}/Dockerfile.j2"
+	if [[ ${DOCKER_IMAGE_NAME} =~ '-dbg' ]]; then
+		DEP_FILES="${DEP_FILES} build_debug_docker_j2.sh"
+	fi
+
+	# Calculate the version SHA
+	VERSION_SHA="$( (echo -n "${!IMAGE_DBGS_NAME}"; cat ${DEP_FILES} ${VERSION_FILES}) | sha1sum | awk '{print substr($1,0,23);}')"
+    GLOBAL_CACHE_FILE=${GLOBAL_CACHE_DIR}/${DOCKER_IMAGE_NAME}-${VERSION_SHA}.tgz
+
+	GIT_FILE_STATUS=$(git status -s ${DEP_FILES})
+
+	# If the cache file is not exists in the global cache for the given SHA, 
+	# store the new cache file into version cache path.
+	if [ -f ${LOCAL_CACHE_FILE} ]; then
+		if [[ -z ${GIT_FILE_STATUS} && ! -e ${GLOBAL_CACHE_FILE} ]]; then
+			mkdir -p ${GLOBAL_CACHE_DIR}
+			chmod -f 777 ${GLOBAL_CACHE_DIR}
+			FLOCK ${GLOBAL_CACHE_FILE}
+			cp ${LOCAL_CACHE_FILE} ${GLOBAL_CACHE_FILE} 
+			chmod -f 777 ${LOCAL_CACHE_FILE} ${GLOBAL_CACHE_FILE} 
+			FUNLOCK ${GLOBAL_CACHE_FILE}
+		fi
+	fi
+fi
+
 docker container rm $DOCKER_CONTAINER

scripts/prepare_docker_buildinfo.sh

@@ -84,3 +84,75 @@ touch $BUILDINFO_VERSION_PATH/versions-deb
 LOCAL_CACHE_DIR=target/vcache/${IMAGENAME}
 mkdir -p ${LOCAL_CACHE_DIR} ${DOCKER_PATH}/vcache/
 chmod -f 777 ${LOCAL_CACHE_DIR} ${DOCKER_PATH}/vcache/
+
+if [[ "$SKIP_BUILD_HOOK" == y || ${ENABLE_VERSION_CONTROL_DOCKER} != y ]]; then
+	exit 0
+fi
+
+# Version cache 
+DOCKER_IMAGE_NAME=${IMAGENAME}
+IMAGE_DBGS_NAME=${DOCKER_IMAGE_NAME//-/_}_image_dbgs
+
+if [[ ${DOCKER_IMAGE_NAME} == sonic-slave-* ]]; then
+	GLOBAL_CACHE_DIR=${SONIC_VERSION_CACHE_SOURCE}/${DOCKER_IMAGE_NAME}
+else
+	GLOBAL_CACHE_DIR=/vcache/${DOCKER_IMAGE_NAME}
+fi
+
+SRC_VERSION_PATH=files/build/versions
+if [ ! -z ${SONIC_VERSION_CACHE} ]; then
+
+	# Version files for SHA calculation
+	VERSION_FILES="${SRC_VERSION_PATH}/dockers/${DOCKER_IMAGE_NAME}/versions-*-${DISTRO}-${ARCH} ${SRC_VERSION_PATH}/default/versions-*"
+	DEP_FILES="Dockerfile.j2"
+	if [[ ${DOCKER_IMAGE_NAME} =~ '-dbg' ]]; then
+		DEP_DBG_FILES="build_debug_docker_j2.sh"
+	fi
+
+	#Calculate the version SHA
+	VERSION_SHA="$( (echo -n "${!IMAGE_DBGS_NAME}"; \
+		(cd ${DOCKER_PATH}; cat ${DEP_FILES}); \
+		cat ${DEP_DBG_FILES} ${VERSION_FILES}) \
+		| sha1sum | awk '{print substr($1,0,23);}')"
+
+	GLOBAL_CACHE_FILE=${GLOBAL_CACHE_DIR}/${DOCKER_IMAGE_NAME}-${VERSION_SHA}.tgz
+	LOCAL_CACHE_FILE=${LOCAL_CACHE_DIR}/cache.tgz
+	GIT_FILE_STATUS=$(git status -s ${DEP_FILES})
+
+	# Create the empty cache tar file as local cache
+	if [[ ! -f ${LOCAL_CACHE_FILE} ]]; then
+		tar -zcf ${LOCAL_CACHE_FILE} -T /dev/null
+		chmod -f 777 ${LOCAL_CACHE_FILE}
+	fi
+
+	# Global cache file exists, load from global cache.
+	if [[  -e ${GLOBAL_CACHE_FILE} ]]; then
+		cp ${GLOBAL_CACHE_FILE} ${LOCAL_CACHE_FILE}
+		touch ${GLOBAL_CACHE_FILE}
+	else
+		# When file is modified, Global SHA is calculated with the local change.
+		# Load from the previous version of build cache if exists
+		VERSIONS=( "HEAD" "HEAD~1" "HEAD~2" )
+		for VERSION in ${VERSIONS[@]}; do
+			VERSION_PREV_SHA="$( (echo -n "${!IMAGE_DBGS_NAME}"; \
+				(cd ${DOCKER_PATH}; git --no-pager show $(ls -f ${DEP_FILES}|sed 's|.*|'${VERSION}':./&|g')); \
+				(git --no-pager show $(ls -f ${DEP_DBG_FILES} ${VERSION_FILES}|sed 's|.*|'${VERSION}':&|g'))) \
+				| sha1sum | awk '{print substr($1,0,23);}')"
+			GLOBAL_PREV_CACHE_FILE=${GLOBAL_CACHE_DIR}/${DOCKER_IMAGE_NAME}-${VERSION_PREV_SHA}.tgz
+			if [[  -e ${GLOBAL_PREV_CACHE_FILE} ]]; then
+				cp ${GLOBAL_PREV_CACHE_FILE} ${LOCAL_CACHE_FILE}
+				touch ${GLOBAL_PREV_CACHE_FILE}
+				break
+			fi
+		done
+	fi
+
+	rm -f ${DOCKER_PATH}/vcache/cache.tgz
+	ln -f ${LOCAL_CACHE_FILE} ${DOCKER_PATH}/vcache/cache.tgz
+
+
+else
+	# Delete the cache file if version cache is disabled.
+	rm -f ${DOCKER_PATH}/vcache/cache.tgz
+fi
+

src/sonic-build-hooks/scripts/collect_version_files

@@ -12,9 +12,12 @@ DIST=$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)
 mkdir -p $TARGET_PATH
 chmod a+rw $TARGET_PATH
 
-dpkg-query -W -f '${Package}==${Version}\n' >> "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}"
-([ -x "/usr/local/bin/pip2" ] || [ -x "/usr/bin/pip2" ]) && pip2 freeze >> "${TARGET_PATH}/versions-py2-${DIST}-${ARCH}"
-([ -x "/usr/local/bin/pip3" ] || [ -x "/usr/bin/pip3" ]) && pip3 freeze >> "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}"
+# Skip the package that does have a static build version. 
+# SAI package versions are changed too frequently.
+SKIP_VERSION_PACKAGE="libsaibcm|libpaibcm|linuxptp|@ file://"
+dpkg-query -W -f '${Package}==${Version}\n' | grep -Ev "${SKIP_VERSION_PACKAGE}" > "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}"
+([ -x "/usr/local/bin/pip2" ] || [ -x "/usr/bin/pip2" ]) && pip2 freeze --all| grep -Ev "${SKIP_VERSION_PACKAGE}" > "${TARGET_PATH}/versions-py2-${DIST}-${ARCH}"
+([ -x "/usr/local/bin/pip3" ] || [ -x "/usr/bin/pip3" ]) && pip3 freeze --all| grep -Ev "${SKIP_VERSION_PACKAGE}" > "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}"
 
 [ -f "${BUILD_WEB_VERSION_FILE}" ] && cp ${BUILD_WEB_VERSION_FILE} ${TARGET_PATH}
 [ -f "${BUILD_GIT_VERSION_FILE}" ] && cp ${BUILD_GIT_VERSION_FILE} ${TARGET_PATH}

src/sonic-build-hooks/scripts/post_run_buildinfo

@@ -13,3 +13,14 @@ collect_version_files $POST_VERSION_PATH
 #Save the cache file for exporting it to host.
 tar -C ${PKG_CACHE_PATH} --exclude=cache.tgz -zcvf /cache.tgz .
 
+[ -d $BUILD_VERSION_PATH ] && [ ! -z "$(ls -A $BUILD_VERSION_PATH)" ] && cp -rf $BUILD_VERSION_PATH/* $POST_VERSION_PATH
+rm -rf $BUILD_VERSION_PATH/*
+if [ ! -z "$(get_version_cache_option)" ]; then
+	# Restore he deletion of cache files
+	cat <<-EOF >/etc/apt/apt.conf.d/docker-clean
+	DPkg::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };
+	APT::Update::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };
+	EOF
+
+fi
+

src/sonic-build-hooks/scripts/pre_run_buildinfo

@@ -18,6 +18,22 @@ symlink_build_hooks
 set_reproducible_mirrors
 
 mkdir -p /var/cache/apt/archives/
+mkdir -p ${PKG_CACHE_PATH}/deb/
+
+if [ ! -z "$(get_version_cache_option)" ]; then
+	# Skip the deletion of cache files
+	cat <<-EOF >/etc/apt/apt.conf.d/docker-clean
+	DPkg::Post-Invoke { "test -f /usr/bin/rsync && rsync -avzh --ignore-errors /var/cache/apt/archives/ ${PKG_CACHE_PATH}/deb/; rm -f /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };
+	APT::Update::Post-Invoke { "test -f /usr/bin/rsync && rsync -avzh --ignore-errors /var/cache/apt/archives/ ${PKG_CACHE_PATH}/deb/; rm -f /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };
+	APT::Keep-Downloaded-Packages "true";
+	EOF
+fi
+
+# Extract the cache inside the docker build.
+if [ -f ${PKG_CACHE_FILE_NAME} ]; then
+	tar -C ${PKG_CACHE_PATH} -xvf  ${PKG_CACHE_FILE_NAME}
+	test -e ${PKG_CACHE_PATH}/deb && cp ${PKG_CACHE_PATH}/deb/* /var/cache/apt/archives/
+fi
 
 chmod -R a+rw $BUILDINFO_PATH
 

src/sonic-build-hooks/scripts/utils.sh

@@ -17,7 +17,7 @@ function FLOCK()
     eval $(echo exec {${lock_fd}}\<\>"${filename}.flock")
     #echo ${!lock_fd}
     if ! flock -x -w ${timeout} "${!lock_fd}" ; then
-        echo "ERROR: Lock timeout trying to access ${filename}.flock";
+        echo "ERROR: Lock timeout trying to access ${filename}.flock" 1>&2;
         exit 1;
     fi
     #echo "Lock acquired .."