From d34339b9460e29ee8dff7d1eb1f5f9ff06174b1b Mon Sep 17 00:00:00 2001 From: Jia Yuan Lo Date: Sat, 6 Aug 2022 15:25:53 +0800 Subject: [PATCH] enhance(scripts/generate-bootstraps.sh): add caching support * format the script too --- scripts/generate-bootstraps.sh | 74 +++++++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 10 deletions(-) diff --git a/scripts/generate-bootstraps.sh b/scripts/generate-bootstraps.sh index 1d85d5243415bef..f2a18212b24d179 100755 --- a/scripts/generate-bootstraps.sh +++ b/scripts/generate-bootstraps.sh @@ -13,6 +13,11 @@ trap 'rm -rf $BOOTSTRAP_TMPDIR' EXIT # and <10. BOOTSTRAP_ANDROID10_COMPATIBLE=false +# Optional caching of downloaded packages in output directory +# Apt only for now +TERMUX_PACKAGE_DOWNLOAD_CACHE=false +TERMUX_PACKAGE_DOWNLOAD_CACHE_DIR="$PWD/output" + # By default, bootstrap archives will be built for all architectures # supported by Termux application. # Override with option '--architectures'. @@ -118,9 +123,10 @@ pull_package() { local package_tmpdir="${BOOTSTRAP_PKGDIR}/${package_name}" mkdir -p "$package_tmpdir" - if [ ${TERMUX_PACKAGE_MANAGER} = "apt" ]; then - local package_url + if [ "$TERMUX_PACKAGE_MANAGER" = apt ]; then + local package_url package_file package_url="$REPO_BASE_URL/$(echo "${PACKAGE_METADATA[${package_name}]}" | grep -i "^Filename:" | awk '{ print $2 }')" + package_file=$(echo "$package_url" | sed -e 's|\(.*\)\/\(.*\).deb$|\2.deb|') if [ "${package_url}" = "$REPO_BASE_URL" ] || [ "${package_url}" = "${REPO_BASE_URL}/" ]; then echo "[!] Failed to determine URL for package '$package_name'." exit 1 @@ -144,10 +150,31 @@ pull_package() { unset dep fi + if [ "$TERMUX_PACKAGE_DOWNLOAD_CACHE" = true ]; then + # since each generate-bootstraps.sh run will have different package_tmpdir + # we dont have to check existence of both files and even so prioritise the cached copy + # also use symbolic link to save space and I/O + echo "[*] CACHE: Checking for cached copy of '$package_file' for package '$package_name'..." + if [ -e "$TERMUX_PACKAGE_DOWNLOAD_CACHE_DIR/$package_file" ]; then + echo "[*] CACHE: Found cached copy '$package_file'! Symlinking..." + rm -f "$package_tmpdir/package.deb" + ln -fs "$TERMUX_PACKAGE_DOWNLOAD_CACHE_DIR/$package_file" "$package_tmpdir/package.deb" + else + echo "[*] CACHE: Cached copy of '$package_file' not found. Skipping..." + fi + fi + if [ ! -e "$package_tmpdir/package.deb" ]; then echo "[*] Downloading '$package_name'..." curl --fail --location --output "$package_tmpdir/package.deb" "$package_url" + if [ "$TERMUX_PACKAGE_DOWNLOAD_CACHE" = true ]; then + echo "[*] CACHE: Storing '$package_name' package.deb at '$TERMUX_PACKAGE_DOWNLOAD_CACHE_DIR/$package_file'..." + cp -f "$package_tmpdir/package.deb" "$TERMUX_PACKAGE_DOWNLOAD_CACHE_DIR/$package_file" + fi + fi + + if [ -e "$package_tmpdir/package.deb" ]; then echo "[*] Extracting '$package_name'..." (cd "$package_tmpdir" ar x package.deb @@ -295,6 +322,11 @@ show_usage() { echo " Multiple packages should be passed as" echo " comma-separated list." echo + echo " -c, --cache [DIR] Enable caching. Downloaded package files will be" + echo " stored at output directory unless specified another" + echo " path. The files can be reused at later runs." + echo " This option is disabled by default and for apt only." + echo echo " --pm MANAGER Set up a package manager in bootstrap." echo " It can only be pacman or apt (the default is apt)." echo @@ -311,7 +343,7 @@ show_usage() { echo "Architectures: ${TERMUX_ARCHITECTURES[*]}" echo "Repository Base Url: ${REPO_BASE_URL}" echo "Prefix: ${TERMUX_PREFIX}" - echo "Package manager: ${TERMUX_PACKAGE_MANAGER}" + echo "Package manager: ${TERMUX_PACKAGE_MANAGER}" echo } @@ -343,7 +375,7 @@ while (($# > 0)); do REPO_BASE_URL="${REPO_BASE_URLS[${TERMUX_PACKAGE_MANAGER}]}" shift 1 else - echo "[!] Option '--pm' requires an argument." 1>&2 + echo "[!] Option '--pm' requires an argument." >&2 show_usage exit 1 fi @@ -372,6 +404,13 @@ while (($# > 0)); do exit 1 fi ;; + -c|--cache) + if [ $# -gt 1 ] && [ -n "$2" ] && [[ $2 != -* ]]; then + TERMUX_PACKAGE_DOWNLOAD_CACHE_DIR="$2" + shift 1 + fi + TERMUX_PACKAGE_DOWNLOAD_CACHE=true + ;; *) echo "[!] Got unknown option '$1'" show_usage @@ -382,23 +421,38 @@ while (($# > 0)); do done if [[ "$TERMUX_PACKAGE_MANAGER" == *" "* ]] || [[ " ${TERMUX_PACKAGE_MANAGERS[*]} " != *" $TERMUX_PACKAGE_MANAGER "* ]]; then - echo "[!] Invalid package manager '$TERMUX_PACKAGE_MANAGER'" 1>&2 - echo "Supported package managers: '${TERMUX_PACKAGE_MANAGERS[*]}'" 1>&2 + echo "[!] Invalid package manager '$TERMUX_PACKAGE_MANAGER'" >&2 + echo "Supported package managers: '${TERMUX_PACKAGE_MANAGERS[*]}'" >&2 exit 1 fi if [ -z "$REPO_BASE_URL" ]; then - echo "[!] The repository base url is not set." 1>&2 + echo "[!] The repository base url is not set." >&2 exit 1 fi +if [ "$TERMUX_PACKAGE_DOWNLOAD_CACHE" = true ]; then + if [ "$TERMUX_PACKAGE_MANAGER" != apt ]; then + echo "[!] Option '--cache' does not support '$TERMUX_PACKAGE_MANAGER' yet." >&2 + exit 1 + fi + echo "[*] CACHE: Caching enabled. Cached packages will be stored and reused." + echo "[*] CACHE: Cache directory = $TERMUX_PACKAGE_DOWNLOAD_CACHE_DIR" + echo "[*] CACHE: Temporary directory = $BOOTSTRAP_PKGDIR" + mkdir -p "$TERMUX_PACKAGE_DOWNLOAD_CACHE_DIR" + if [ ! -d "$TERMUX_PACKAGE_DOWNLOAD_CACHE_DIR" ]; then + echo "[!] CACHE: Failed to create cache directory '$TERMUX_PACKAGE_DOWNLOAD_CACHE_DIR'." >&2 + exit 1 + fi +fi + for package_arch in "${TERMUX_ARCHITECTURES[@]}"; do BOOTSTRAP_ROOTFS="$BOOTSTRAP_TMPDIR/rootfs-${package_arch}" BOOTSTRAP_PKGDIR="$BOOTSTRAP_TMPDIR/packages-${package_arch}" # Create initial directories for $TERMUX_PREFIX if ! ${BOOTSTRAP_ANDROID10_COMPATIBLE}; then - if [ ${TERMUX_PACKAGE_MANAGER} = "apt" ]; then + if [ "$TERMUX_PACKAGE_MANAGER" = apt ]; then mkdir -p "${BOOTSTRAP_ROOTFS}/${TERMUX_PREFIX}/etc/apt/apt.conf.d" mkdir -p "${BOOTSTRAP_ROOTFS}/${TERMUX_PREFIX}/etc/apt/preferences.d" mkdir -p "${BOOTSTRAP_ROOTFS}/${TERMUX_PREFIX}/var/lib/dpkg/info" @@ -420,7 +474,7 @@ for package_arch in "${TERMUX_ARCHITECTURES[@]}"; do # Read package metadata. unset PACKAGE_METADATA declare -A PACKAGE_METADATA - if [ ${TERMUX_PACKAGE_MANAGER} = "apt" ]; then + if [ "$TERMUX_PACKAGE_MANAGER" = apt ]; then read_package_list_deb "$package_arch" else read_package_list_pac "$package_arch" @@ -460,7 +514,7 @@ for package_arch in "${TERMUX_ARCHITECTURES[@]}"; do # Additional. pull_package ed - if [ ${TERMUX_PACKAGE_MANAGER} = "apt" ]; then + if [ "$TERMUX_PACKAGE_MANAGER" = apt ]; then pull_package debianutils fi pull_package dos2unix