Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[build] add support for 2 stage rootfs build #15924

Merged
merged 11 commits into from
Oct 11, 2023
26 changes: 25 additions & 1 deletion Makefile.cache
Original file line number Diff line number Diff line change
Expand Up @@ -357,6 +357,21 @@ define SAVE_CACHE
$(if $(call CHECK_WCACHE_ENABLED,$(1)), $(call SAVE_INTO_CACHE,$(1),$(2)))
endef

RFS_DEP_FILES := $(wildcard \
$(addprefix scripts/, build_debian_base_system.sh prepare_debian_image_buildinfo.sh build_mirror_config.sh) \
$(shell git ls-files files/initramfs-tools) \
$(shell git ls-files files/image_config) \
$(shell git ls-files files/apparmor) \
$(shell git ls-files files/apt) \
$(shell git ls-files files/sshd) \
$(shell git ls-files files/dhcp) \
src/sonic-build-hooks/buildinfo/trusted.gpg.d \
platform/$(CONFIGURED_PLATFORM)/modules \
files/docker/docker.service.conf \
files/build_templates/default_users.json.j2 \
files/build_scripts/generate_asic_config_checksum.py \
files/scripts/core_cleanup.py \
build_debian.sh onie-image.conf)


# Set the target path for each target.
Expand Down Expand Up @@ -384,11 +399,17 @@ $(foreach pkg, $(SONIC_INSTALL_PKGS), \
$(eval $(pkg)_DST_PATH := $(if $($(pkg)_DST_PATH), $($(pkg)_DST_PATH), $(FSROOT_PATH))) \
$(eval $(FSROOT_PATH)/$(pkg)_TARGET := $(pkg)) )

$(foreach pkg, $(SONIC_RFS_TARGETS), \
$(eval $(pkg)_DST_PATH := $(if $($(pkg)_DST_PATH), $($(pkg)_DST_PATH), $(TARGET_PATH))) \
$(eval $(pkg)_CACHE_MODE := GIT_CONTENT_SHA) \
$(eval $(pkg)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)) \
$(eval $(pkg)_DEP_FILES := $(SONIC_COMMON_BASE_FILES_LIST) $(RFS_DEP_FILES)) \
$(eval $(TARGET_PATH)/$(pkg)_TARGET := $(pkg)) )

# define the DEP files(.dep and .smdep) and SHA files (.sha and smsha) for each target
$(foreach pkg, $(SONIC_MAKE_DEBS) $(SONIC_DPKG_DEBS) $(SONIC_ONLINE_DEBS) $(SONIC_COPY_DEBS) \
$(SONIC_MAKE_FILES) $(SONIC_PYTHON_STDEB_DEBS) $(SONIC_PYTHON_WHEELS) \
$(SONIC_DOCKER_IMAGES) $(SONIC_DOCKER_DBG_IMAGES) $(SONIC_INSTALL_PKGS), \
$(SONIC_DOCKER_IMAGES) $(SONIC_DOCKER_DBG_IMAGES) $(SONIC_INSTALL_PKGS) $(SONIC_RFS_TARGETS), \
$(eval $(pkg)_MOD_SRC_PATH:=$(if $($(pkg)_SRC_PATH),$($(pkg)_SRC_PATH),$($(pkg)_PATH))) \
$(eval $(pkg)_BASE_PATH:=$(if $($(pkg)_BASE_PATH),$($(pkg)_BASE_PATH),$(CURDIR))) \
$(eval $(pkg)_DEP_FLAGS_FILE:=$($(pkg)_DST_PATH)/$(pkg).flags) \
Expand Down Expand Up @@ -489,6 +510,7 @@ $(eval $(call FLAGS_DEP_RULES, $(SONIC_PYTHON_STDEB_DEBS), $(PYTHON_DEBS_PATH),f
$(eval $(call FLAGS_DEP_RULES, $(SONIC_PYTHON_WHEELS), $(PYTHON_WHEELS_PATH),flags))
$(eval $(call FLAGS_DEP_RULES, $(SONIC_DOCKER_IMAGES) $(SONIC_DOCKER_DBG_IMAGES), $(TARGET_PATH),flags))
$(eval $(call FLAGS_DEP_RULES, $(SONIC_INSTALL_PKGS), $(FSROOT_PATH),flags))
$(eval $(call FLAGS_DEP_RULES, $(SONIC_RFS_TARGETS), $(TARGET_PATH),flags))



Expand Down Expand Up @@ -585,6 +607,7 @@ $(eval $(call SHA_DEP_RULES, $(SONIC_PYTHON_STDEB_DEBS), $(PYTHON_DEBS_PATH),dep
$(eval $(call SHA_DEP_RULES, $(SONIC_PYTHON_WHEELS), $(PYTHON_WHEELS_PATH),dep))
$(eval $(call SHA_DEP_RULES, $(SONIC_DOCKER_IMAGES) $(SONIC_DOCKER_DBG_IMAGES), $(TARGET_PATH),dep))
$(eval $(call SHA_DEP_RULES, $(SONIC_INSTALL_PKGS), $(FSROOT_PATH),dep))
$(eval $(call SHA_DEP_RULES, $(SONIC_RFS_TARGETS), $(TARGET_PATH),dep))



Expand Down Expand Up @@ -618,6 +641,7 @@ SONIC_CACHE_CLEAN_TARGETS = $(addsuffix -clean,$(addprefix $(TARGET_PATH)/, \
$(SONIC_DOCKER_IMAGES) \
$(SONIC_DOCKER_DBG_IMAGES) \
$(SONIC_SIMPLE_DOCKER_IMAGES) \
$(SONIC_RFS_TARGETS) \
$(SONIC_INSTALLERS)))
$(SONIC_CACHE_CLEAN_TARGETS) :: $(TARGET_PATH)/%-clean : .platform
@rm -f $($*_DEP_FLAGS_FILE) $($*_MOD_HASH_FILE) $($*_SMOD_HASH_FILE) \
Expand Down
77 changes: 58 additions & 19 deletions build_debian.sh
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ TRUSTED_GPG_DIR=$BUILD_TOOL_PATH/trusted.gpg.d
exit 1
}

## Check if not a last stage of RFS build
if [[ $RFS_SPLIT_LAST_STAGE != y ]]; then

## Prepare the file system directory
if [[ -d $FILESYSTEM_ROOT ]]; then
sudo rm -rf $FILESYSTEM_ROOT || die "Failed to clean chroot directory"
Expand All @@ -71,11 +74,6 @@ touch $FILESYSTEM_ROOT/$PLATFORM_DIR/firsttime
## ensure proc is mounted
sudo mount proc /proc -t proc || true

## make / as a mountpoint in chroot env, needed by dockerd
pushd $FILESYSTEM_ROOT
sudo mount --bind . .
popd

## Build the host debian base system
echo '[INFO] Build host debian base system...'
TARGET_PATH=$TARGET_PATH scripts/build_debian_base_system.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT
Expand Down Expand Up @@ -576,24 +574,11 @@ if [ -f files/image_config/ntp/ntp-systemd-wrapper ]; then
sudo cp ./files/image_config/ntp/ntp-systemd-wrapper $FILESYSTEM_ROOT/usr/lib/ntp/
fi

## Version file
## Version file part 1
sudo mkdir -p $FILESYSTEM_ROOT/etc/sonic
if [ -f files/image_config/sonic_release ]; then
sudo cp files/image_config/sonic_release $FILESYSTEM_ROOT/etc/sonic/
fi
export build_version="${SONIC_IMAGE_VERSION}"
export debian_version="$(cat $FILESYSTEM_ROOT/etc/debian_version)"
export kernel_version="${kversion}"
export asic_type="${sonic_asic_platform}"
export asic_subtype="${TARGET_MACHINE}"
export commit_id="$(git rev-parse --short HEAD)"
export branch="$(git rev-parse --abbrev-ref HEAD)"
export release="$(if [ -f $FILESYSTEM_ROOT/etc/sonic/sonic_release ]; then cat $FILESYSTEM_ROOT/etc/sonic/sonic_release; fi)"
export build_date="$(date -u)"
export build_number="${BUILD_NUMBER:-0}"
export built_by="$USER@$BUILD_HOSTNAME"
export sonic_os_version="${SONIC_OS_VERSION}"
j2 files/build_templates/sonic_version.yml.j2 | sudo tee $FILESYSTEM_ROOT/etc/sonic/sonic_version.yml

# Default users info
export password_expire="$( [[ "$CHANGE_DEFAULT_PASSWORD" == "y" ]] && echo true || echo false )"
Expand All @@ -615,6 +600,60 @@ if [[ ! -f './asic_config_checksum' ]]; then
fi
sudo cp ./asic_config_checksum $FILESYSTEM_ROOT/etc/sonic/asic_config_checksum

## Check if not a last stage of RFS build
fi

if [[ $RFS_SPLIT_FIRST_STAGE == y ]]; then
echo '[INFO] Finished with RFS first stage'
echo '[INFO] Umount all'

## Display all process details access /proc
sudo LANG=C chroot $FILESYSTEM_ROOT fuser -vm /proc
## Kill the processes
sudo LANG=C chroot $FILESYSTEM_ROOT fuser -km /proc || true
## Wait fuser fully kill the processes
sudo timeout 15s bash -c 'until LANG=C chroot $0 umount /proc; do sleep 1; done' $FILESYSTEM_ROOT || true

sudo rm -f $TARGET_PATH/$RFS_SQUASHFS_NAME
sudo mksquashfs $FILESYSTEM_ROOT $TARGET_PATH/$RFS_SQUASHFS_NAME -Xcompression-level 1

exit 0
fi

if [[ $RFS_SPLIT_LAST_STAGE == y ]]; then
echo '[INFO] RFS build: second stage'

## ensure proc is mounted
sudo mount proc /proc -t proc || true

sudo fuser -vm $FILESYSTEM_ROOT || true
sudo rm -rf $FILESYSTEM_ROOT
sudo unsquashfs -d $FILESYSTEM_ROOT $TARGET_PATH/$RFS_SQUASHFS_NAME

## make / as a mountpoint in chroot env, needed by dockerd
pushd $FILESYSTEM_ROOT
sudo mount --bind . .
popd

trap_push 'sudo LANG=C chroot $FILESYSTEM_ROOT umount /proc || true'
sudo LANG=C chroot $FILESYSTEM_ROOT mount proc /proc -t proc
fi

## Version file part 2
export build_version="${SONIC_IMAGE_VERSION}"
export debian_version="$(cat $FILESYSTEM_ROOT/etc/debian_version)"
export kernel_version="${kversion}"
export asic_type="${sonic_asic_platform}"
export asic_subtype="${TARGET_MACHINE}"
export commit_id="$(git rev-parse --short HEAD)"
export branch="$(git rev-parse --abbrev-ref HEAD)"
export release="$(if [ -f $FILESYSTEM_ROOT/etc/sonic/sonic_release ]; then cat $FILESYSTEM_ROOT/etc/sonic/sonic_release; fi)"
export build_date="$(date -u)"
export build_number="${BUILD_NUMBER:-0}"
export built_by="$USER@$BUILD_HOSTNAME"
export sonic_os_version="${SONIC_OS_VERSION}"
j2 files/build_templates/sonic_version.yml.j2 | sudo tee $FILESYSTEM_ROOT/etc/sonic/sonic_version.yml

if [ -f sonic_debian_extension.sh ]; then
./sonic_debian_extension.sh $FILESYSTEM_ROOT $PLATFORM_DIR $IMAGE_DISTRO
fi
Expand Down
1 change: 0 additions & 1 deletion rules/functions
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,6 @@ SONIC_DBG_DOCKERS += $(2)
endef



###############################################################################
## Utility functions
###############################################################################
Expand Down
96 changes: 94 additions & 2 deletions slave.mk
Original file line number Diff line number Diff line change
Expand Up @@ -449,6 +449,34 @@ else
$(info SONiC Build System for $(CONFIGURED_PLATFORM):$(CONFIGURED_ARCH))
endif

# Definition of SONIC_RFS_TARGETS
define rfs_get_installer_dependencies
$(call rfs_build_target_name,$(1),$($(1)_MACHINE))
endef

define rfs_build_target_name
$(1)__$(2)__rfs.squashfs
endef

define rfs_define_target
$(eval rfs_target=$(call rfs_build_target_name,$(1),$($(1)_MACHINE)))
$(eval $(rfs_target)_INSTALLER=$(1))
$(eval $(rfs_target)_MACHINE=$($(1)_MACHINE))
$(eval SONIC_RFS_TARGETS+=$(rfs_target))

$(if $($(1)_DEPENDENT_MACHINE),\
$(eval dependent_rfs_target=$(call rfs_build_target_name,$(1),$($(1)_DEPENDENT_MACHINE)))
$(eval $(dependent_rfs_target)_INSTALLER=$(1))
$(eval $(dependent_rfs_target)_MACHINE=$($(1)_DEPENDENT_MACHINE))
$(eval SONIC_RFS_TARGETS+=$(dependent_rfs_target))
$(eval $(rfs_target)_DEPENDENT_RFS=$(dependent_rfs_target)))
endef

$(foreach installer,$(SONIC_INSTALLERS),$(eval $(call rfs_define_target,$(installer))))
$(foreach installer, $(SONIC_INSTALLERS), $(eval $(installer)_RFS_DEPENDS=$(call rfs_get_installer_dependencies,$(installer))))

SONIC_TARGET_LIST += $(addprefix $(TARGET_PATH)/, $(SONIC_RFS_TARGETS))

# Overwrite the buildinfo in slave container
ifeq ($(filter clean,$(MAKECMDGOALS)),)
$(shell DBGOPT='$(DBGOPT)' scripts/prepare_slave_container_buildinfo.sh $(SLAVE_DIR) $(CONFIGURED_ARCH) $(BLDENV))
Expand Down Expand Up @@ -1211,6 +1239,62 @@ $(DOCKER_LOAD_TARGETS) : $(TARGET_PATH)/%.gz-load : .platform docker-start $$(TA
## Installers
###############################################################################

$(addprefix $(TARGET_PATH)/, $(SONIC_RFS_TARGETS)) : $(TARGET_PATH)/% : \
Copy link
Collaborator

@qiluo-msft qiluo-msft Aug 20, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

SONIC_RFS_TARGETS

Could you make sure the new target SONIC_RFS_TARGETS is reused if we build multiple vendors image in series?

We also have reproducible mechanism in rules/*.dep files. Is it possible to define this target's dep file, so the target is reused as cached for other later build jobs?

@xumia to review. #Closed

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The target is reused. I've also added a cache: a880d53
Since the targets are defined dynamically, I could not add the '.dep' file. Instead, the relevant variables are also set dynamically.

.platform \
build_debian.sh \
$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(INITRAMFS_TOOLS) $(LINUX_KERNEL)) \
$(addsuffix -install,$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(DEBOOTSTRAP))) \
$$(addprefix $(TARGET_PATH)/,$$($$*_DEPENDENT_RFS)) \
$(call dpkg_depend,$(TARGET_PATH)/%.dep)
$(HEADER)

$(call LOAD_CACHE,$*,$@)

# Skip building the target if it is already loaded from cache
if [ -z '$($*_CACHE_LOADED)' ] ; then

$(eval installer=$($*_INSTALLER))
$(eval machine=$($*_MACHINE))

export debs_path="$(IMAGE_DISTRO_DEBS_PATH)"
export initramfs_tools="$(IMAGE_DISTRO_DEBS_PATH)/$(INITRAMFS_TOOLS)"
export linux_kernel="$(IMAGE_DISTRO_DEBS_PATH)/$(LINUX_KERNEL)"
export kversion="$(KVERSION)"
export image_type="$($(installer)_IMAGE_TYPE)"
export sonicadmin_user="$(USERNAME)"
export sonic_asic_platform="$(patsubst %-$(CONFIGURED_ARCH),%,$(CONFIGURED_PLATFORM))"
export RFS_SPLIT_FIRST_STAGE=y
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nitpick, but can you set RFS_SPLIT_LAST_STAGE to n here, so that in case in the future, some code gets added that references this variable before RFS_SPLIT_FIRST_STAGE, it will still be defined?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, added in b6f9006

export RFS_SPLIT_LAST_STAGE=n

j2 -f env files/initramfs-tools/union-mount.j2 onie-image.conf > files/initramfs-tools/union-mount
j2 -f env files/initramfs-tools/arista-convertfs.j2 onie-image.conf > files/initramfs-tools/arista-convertfs
Comment on lines +1269 to +1270
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Need to exclude these generated files (union-mount and arista-convertfs) from RFS_DEP_FILES.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These files are not in the .dep since the .dep gets created first. I've switched to git ls-files to be safe anyway.

Copy link
Contributor

@k-v1 k-v1 Oct 2, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These files exist after distclean.
If you do

  1. make configure PLATFORM=xxx && make target/sonic-xxx.bin with DPKG cache.
  2. make distclean
  3. make configure PLATFORM=xxx && make target/sonic-xxx.bin with DPKG cache
    then at step 3 you find these files in rfs .dep file.
    git ls-files should be enough to fix.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Make sense. Anyway, I've fixed it to use the git ls-files


RFS_SQUASHFS_NAME=$* \
USERNAME="$(USERNAME)" \
PASSWORD="$(PASSWORD)" \
CHANGE_DEFAULT_PASSWORD="$(CHANGE_DEFAULT_PASSWORD)" \
TARGET_MACHINE=$(machine) \
IMAGE_TYPE=$($(installer)_IMAGE_TYPE) \
TARGET_PATH=$(TARGET_PATH) \
TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \
SONIC_ENABLE_SECUREBOOT_SIGNATURE="$(SONIC_ENABLE_SECUREBOOT_SIGNATURE)" \
SIGNING_KEY="$(SIGNING_KEY)" \
SIGNING_CERT="$(SIGNING_CERT)" \
PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
DBGOPT='$(DBGOPT)' \
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) \
CROSS_BUILD_ENVIRON=$(CROSS_BUILD_ENVIRON) \
MASTER_KUBERNETES_VERSION=$(MASTER_KUBERNETES_VERSION) \
MASTER_CRI_DOCKERD=$(MASTER_CRI_DOCKERD) \
./build_debian.sh $(LOG)

$(call SAVE_CACHE,$*,$@)

fi

$(FOOTER)

# targets for building installers with base image
$(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
.platform \
Expand Down Expand Up @@ -1265,7 +1349,9 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
$(addprefix $(FILES_PATH)/,$($(SONIC_CTRMGRD)_FILES)) \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY3)) \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SYSTEM_HEALTH)) \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3))
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3)) \
$$(addprefix $(TARGET_PATH)/,$$($$*_RFS_DEPENDS))

$(HEADER)
# Pass initramfs and linux kernel explicitly. They are used for all platforms
export debs_path="$(IMAGE_DISTRO_DEBS_PATH)"
Expand Down Expand Up @@ -1421,13 +1507,17 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
chmod +x sonic_debian_extension.sh,
)

export RFS_SPLIT_FIRST_STAGE=n
export RFS_SPLIT_LAST_STAGE=y

# Build images for the MACHINE, DEPENDENT_MACHINE defined.
$(foreach dep_machine, $($*_MACHINE) $($*_DEPENDENT_MACHINE), \
DEBUG_IMG="$(INSTALL_DEBUG_TOOLS)" \
DEBUG_SRC_ARCHIVE_DIRS="$(DBG_SRC_ARCHIVE)" \
DEBUG_SRC_ARCHIVE_FILE="$(DBG_SRC_ARCHIVE_FILE)" \
scripts/dbg_files.sh

RFS_SQUASHFS_NAME=$*__$(dep_machine)__rfs.squashfs \
DEBUG_IMG="$(INSTALL_DEBUG_TOOLS)" \
DEBUG_SRC_ARCHIVE_FILE="$(DBG_SRC_ARCHIVE_FILE)" \
USERNAME="$(USERNAME)" \
Expand Down Expand Up @@ -1522,7 +1612,9 @@ SONIC_CLEAN_TARGETS += $(addsuffix -clean,$(addprefix $(TARGET_PATH)/, \
$(SONIC_DOCKER_IMAGES) \
$(SONIC_DOCKER_DBG_IMAGES) \
$(SONIC_SIMPLE_DOCKER_IMAGES) \
$(SONIC_INSTALLERS)))
$(SONIC_INSTALLERS) \
$(SONIC_RFS_TARGETS)))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add caching support for rootfs preparation in squashfs file system?
You can refer DPKG caching framework for deb packages.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added in a880d53


$(SONIC_CLEAN_TARGETS) :: $(TARGET_PATH)/%-clean : .platform
$(Q)rm -rf $(TARGET_PATH)/$* target/versions/dockers/$(subst .gz,,$*)

Expand Down