gen-pack

#
# Open-CMSIS-Pack gen-pack Bash library
#
# Copyright (c) 2022-2024 Arm Limited. All rights reserved.
#
# Provided as-is without warranty under Apache 2.0 License
# SPDX-License-Identifier: Apache-2.0
#

GEN_PACK_LIB_VERSION="0.11.0"
GEN_PACK_LIB_SOURCE="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
GEN_PACK_SCRIPT_SOURCE="$(dirname "$(readlink -f "$0")")"

SCRIPT="${SCRIPT:-$(basename "$0")}"
LOG_PREFIX="${LOG_PREFIX:-"$(tput setaf 4 2>/dev/null)${SCRIPT}>$(tput sgr0 2>/dev/null) "}"

shopt -s expand_aliases

if [ "$(cut -d. -f1 <<<"${BASH_VERSION}")" -lt 5 ]; then
  echo "${LOG_PREFIX}gen-pack requires Bash version 5 or later." >&2
  echo "${LOG_PREFIX}Running Bash version is ${BASH_VERSION}!" >&2
  case $(uname -s) in
    'WindowsNT'|MINGW*|CYGWIN*)
      echo "${LOG_PREFIX}Update MSYS2/MinGW, or install latest Git for Windows." >&2
      ;;
    'Darwin')
      echo "${LOG_PREFIX}Get Homebrew and run brew install bash." >&2
      ;;
    *)
      echo "${LOG_PREFIX}Check your OS package manager for Bach v5." >&2
      ;;
  esac
  echo "${LOG_PREFIX}"
  echo "${LOG_PREFIX}If Bash v5 is installed but not used as the interpreter,"
  echo "${LOG_PREFIX}check the shebang line of the enclosing gen_pack.sh."
  echo "${LOG_PREFIX}To be OS agnostic the shebang should be #!/usr/bin/env bash"
  echo "${LOG_PREFIX}and Bash v5 executable must be the default one according"
  echo "${LOG_PREFIX}to your systems PATH order."
  echo "${LOG_PREFIX}"
  exit 1
fi

echo "${LOG_PREFIX}Loading gen-pack library ${GEN_PACK_LIB_VERSION} from ${GEN_PACK_LIB_SOURCE}"

. "${GEN_PACK_LIB_SOURCE}/lib/patches"
. "${GEN_PACK_LIB_SOURCE}/lib/usage"
. "${GEN_PACK_LIB_SOURCE}/lib/getopts"
. "${GEN_PACK_LIB_SOURCE}/lib/utilities"
. "${GEN_PACK_LIB_SOURCE}/lib/pdsc"
. "${GEN_PACK_LIB_SOURCE}/lib/gittools"
. "${GEN_PACK_LIB_SOURCE}/lib/logging"
. "${GEN_PACK_LIB_SOURCE}/lib/helper"

check_locale

check_placeholder "PACK_DIRS"
check_placeholder "PACK_BASE_FILES"
check_placeholder "PACK_DELETE_FILES"
check_placeholder "PACK_PATCH_FILES"
check_placeholder "PACKCHK_DEPS"
PACK_CHANGELOG_MODE="${PACK_CHANGELOG_MODE:-full}"
check_placeholder "PACK_CHANGELOG_MODE"
check_placeholder "PACK_CHECKSUM_EXCLUDE"

#
# Add directories listed in PACK_DIRS to build folder.
# An empty PACK_DIRS setting picks up all folders.
#
# Usage: add_dirs <build>
#  <build>   The temporary build folder, typically PACK_BUILD
#
function add_dirs {
  # Add directories
  if [ -z "${PACK_DIRS}" ]; then
    echo_log "PACK_DIRS is empty, defaulting to all folders."
    PACK_DIRS=$(OIFS="$IFS"; IFS=$'\n'; find . -maxdepth 1 -type d -not -name ".*" -not -path "./$(realpath --relative-to="$(pwd)" "${PACK_BUILD}")" -not -path "./$(realpath --relative-to="$(pwd)" "${PACK_OUTPUT}")"; IFS="$OIFS")
  fi
  echo_log Adding directories to pack:
  echo_log "${PACK_DIRS}"
  echo_log " "
  for d in ${PACK_DIRS}; do
    d=$(realpath --relative-to="$(pwd)" "$d")
    local tar_opts=(--exclude-vcs --exclude-ignore=.gpignore --exclude=.gpignore)
    if [[ $d = ../* ]]; then
      tar "${tar_opts[@]}" -C "$(dirname "$d")" -cf - "$(basename "$d")" | tar -C "$1" -xf -
    else
      tar "${tar_opts[@]}" -cf - "$d" | tar -C "$1" -xf -
    fi
  done
}

#
# Add single files listed in PACK_BASE_FILES to build folder.
#
# Usage: add_files <build>
#  <build>   The temporary build folder, typically PACK_BUILD
#
#
function add_files {
  # Add files
  if [ -n "${PACK_BASE_FILES}" ]; then
    echo_log Adding files to pack:
    echo_log "${PACK_BASE_FILES}"
    echo_log " "
    for f in ${PACK_BASE_FILES}; do
      d=$(realpath --relative-to="$(pwd)" "$f")
      if [[ $f = ../* ]]; then
        cp -r "$f" "$1"
      else
        cp -f --parents "$f" "$1/"
      fi
    done
  else
    echo_log "PACK_BASE_FILES is empty, no additional files to be copied."
  fi
}

#
# Remove specific files listed in PACK_DELETE_FILES from build folder.
#
# Usage: delete_files <build>
#  <build>   The temporary build folder, typically PACK_BUILD
#
function delete_files {
  # Delete files
  if [ -n "${PACK_DELETE_FILES}" ]; then
    echo_log Deleting files from pack:
    echo_log "${PACK_DELETE_FILES}"
    echo_log " "
    for f in ${PACK_DELETE_FILES}; do
      find "$(dirname "$1/$f")" -name "$(basename "$1/$f")" -exec rm -rf "{}" +
    done
  else
    echo_log "PACK_DELETE_FILES is empty, no files to be removed."
  fi
}

#
# Apply patches listed in PACK_PATCH_FILES to files in build folder.
#
# Usage: apply_patches <build>
#  <build>   The temporary build folder, typically PACK_BUILD
#
function apply_patches {
  if [ -z "${PACK_PATCH_FILES}" ]; then
    echo_log "PACK_PATCH_FILES is empty, no patches to be applied."
    return
  fi

  echo_log Applying patches to pack:
  echo_log "${PACK_PATCH_FILES}"
  echo_log " "
  
  local src dest
  src="$(cwd)"
  dest="${1:-${PACK_BUILD}}"
  
  pushd "${dest}" > /dev/null || exit
  for patchfile in ${PACK_PATCH_FILES}; do
    local patchfile="${src}/${patchfile}"
    local targets
    declare -A style

    style["${patchfile}"]="$(detect_eol_style "${patchfile}")"
    convert_eol "LF" "${patchfile}"

    mapfile -t targets < <(                         \
        patch --dry-run -p0 -t -i "${patchfile}"  | \
          grep "ing file"                         | \
          sed -E "s/.*ing file '?([^']*)'?/\1/"     \
      )

    for target in "${targets[@]}"; do
      style["${target}"]="$(detect_eol_style "${target}")"
      convert_eol "LF" "${target}"
    done

    patch -p0 -t -i "${patchfile}"

    # Revert EOL style
    for target in "${targets[@]}"; do
      convert_eol "${style["${target}"]}" "${target}"
    done
    convert_eol "${style["${patchfile}"]}" "${patchfile}"
  done
  popd > /dev/null || exit
}

#
# Run pack schema check
#
# Usage: check_schema <pdsc>
#  <pdsc>   The pack description to check
#
function check_schema {
  if [ -z "${UTILITY_XMLLINT}" ]; then
    echo_err "Cannot run schema check when missing xmllint utility!"
    return 1
  fi

  echo_log "Running schema check for $1"
  local CMSISSCHEMA
  CMSISSCHEMA=$(realpath -m "${CMSIS_TOOLSDIR:-.}/../PACK.xsd")
  local SCHEMA="${CMSISSCHEMA}"
  if [ ! -f "$SCHEMA" ]; then
    echo_log "Fetching schema file..."

    local SCHEMAURL
    SCHEMAURL=$(grep -Pio "noNamespaceSchemaLocation=\"\K[^\"]+" "$1")
    if [ -z "${SCHEMAURL}" ]; then
      echo_err "PDSC file is missing schema url. Consider adding attribute 'noNamespaceSchemaLocation' to '<package>' tag."
    fi

    local SCHEMAVERSION
    SCHEMAVERSION=$(grep -Pio "schemaVersion=\"\K[^\"]+" "$1")
    if [ -z "${SCHEMAVERSION}" ]; then
      echo_err "PDSC file is missing schema version. Consider adding attribute 'schemaVersion' to '<package>' tag."
    fi

    local SCHEMA="${TEMP:-/tmp}/PACK.xsd"

    local errorlevel=1
    if [ -n "${SCHEMAURL}" ]; then
      curl_download "${SCHEMAURL}" "${SCHEMA}"
      errorlevel=$?  
    fi
    if [ -n "${SCHEMAVERSION}" ] && [ $errorlevel -ne 0 ]; then
      echo_err "Schema referenced in PDSC file not found. Looking for schema with version '${SCHEMAVERSION}' ..."
      curl_download "https://raw.githubusercontent.com/Open-CMSIS-Pack/Open-CMSIS-Pack-Spec/v${SCHEMAVERSION}/schema/PACK.xsd" "${SCHEMA}"
      errorlevel=$?
    fi
    if [ $errorlevel -ne 0 ]; then
      echo_err "Schema referenced by PDSC file's schema version not found. Looking for latest schema ..."
      curl_download "https://raw.githubusercontent.com/Open-CMSIS-Pack/Open-CMSIS-Pack-Spec/main/schema/PACK.xsd" "${SCHEMA}"
      errorlevel=$?
    fi
    if [ $errorlevel -ne 0 ]; then
      echo_err "build aborted: No schema file could be found!"
      echo_log " "
      echo_log "  Hint: Assure one of the following to get the schema check working."
      echo_log "        - Provide PACK.XSD into '${CMSISSCHEMA}'."
      echo_log "        - Provide schema URL in PDSC file's xs:noNamespaceSchemaLocation attribute."
      echo_log "        - Provide schema release tag version in PDSC file's schemaVersion attribute."
      echo_log "        Available schema releases are listed on https://github.com/Open-CMSIS-Pack/Open-CMSIS-Pack-Spec/tags."
      echo_log " "
      exit $errorlevel
    fi
  fi
  echo_v "\"${UTILITY_XMLLINT}\" --noout --schema \"${SCHEMA}\" \"$1\""
  "${UTILITY_XMLLINT}" --noout --schema "${SCHEMA}" "$1"
  local errorlevel=$?
  if [ $errorlevel -ne 0 ]; then
    echo_err "build aborted: Schema check of $1 against ${SCHEMA} failed!"
    echo_err " "
    exit 1
  fi
}

#
# Run packchk
#
# Usage: check_pack <pdsc>
#  <pdsc>   The pack description to check
#
function check_pack {
  PACKCHK_DEPS="${PACKCHK_DEPS:-ARM.CMSIS.pdsc}"
  local dependencies include=()

  mapfile -t dependencies < <( \
    grep -Pio "<package\s+\Kvendor=\"[^\"]+\"\s+name=\"[^\"]+\"" "$1" | \
    sed -E 's/vendor="([^"]*)"[ ]*name="([^"]*)"/\1.\2.pdsc/' \
  )

  # shellcheck disable=SC2206
  dependencies+=(${PACKCHK_DEPS})

  fetch_pdsc_files include "${dependencies[@]}"
  local webdir="${CMSIS_PACK_ROOT}/.Web"

  if [ "${UTILITY_PACKCHK_HAS_SCHEMACHECK:-1}" -eq 0 ]; then
    PACKCHK_ARGS+=(--disable-validation)
  fi

  # shellcheck disable=SC2086
  echo_v "\"${UTILITY_PACKCHK}\" ${include[*]/#/-i } -n \"${PACK_OUTPUT}/PackName.txt\"" "${PACKCHK_ARGS[@]}" \"$1\"
  # shellcheck disable=SC2048,SC2086
  "${UTILITY_PACKCHK}" ${include[*]/#/-i } -n "${PACK_OUTPUT}/PackName.txt" "${PACKCHK_ARGS[@]}" "$1"
  local errorlevel=$?
  if [ $errorlevel -ne 0 ]; then
    echo_err "build aborted: pack check failed"
    echo_err " "
    exit 1
  fi
}

#
# Create checksum file for the pack content
# The checksum file is placed to <build>/<vendor>.<pack>.sha1
#
# By default all packaged files are considered.
# Files matching a pattern specified in PACK_CHECKSUM_EXCLUDE are excluded.
#
# Usage: create_sha1 <build> <vendor> <pack>
#   <build>   Pack build folder
#   <vendor>  Pack vendor name
#   <pack>    Pack name
#
function create_sha1 {
  pushd "$1" > /dev/null || exit
  
  local ignore
  local find_args=()
  mapfile -t ignore <<< "${PACK_CHECKSUM_EXCLUDE:-}"
  for p in "${ignore[@]}"; do
    p="${p#"${p%%[![:space:]]*}"}"
    p="${p%"${p##*[![:space:]]}"}"
    if [ -n "$p" ]; then
      find_args+=("-not" "-path" "*$p")
    fi
  done

  find . -type f "${find_args[@]}" -exec "${UTILITY_SHA1SUM}" -b {} + > "${TEMP:-/tmp}/$2.$3.sha1"

  if [ -s "${TEMP:-/tmp}/$2.$3.sha1" ]; then
    mv "${TEMP:-/tmp}/$2.$3.sha1" "$2.$3.sha1"
  fi

  popd > /dev/null || exit
}

#
# Top level function to generate a pack.
#
function gen_pack {
  getopts "$@"
  set -- "${POSITIONAL[@]}" # restore positional parameters

  echo_log "Starting CMSIS-Pack Generation: $(date)"
  echo_log "  in $(pwd)"
  echo_log ""

  find_pack_root
  find_zip
  find_packchk
  find_xmllint
  find_sha1sum
  find_curl
  if [ -n "${CHANGELOG+x}" ]; then
    find_git
    case ${PACK_CHANGELOG_MODE} in
      'full'|'release')
        find_ghcli
      ;;
      'tag')
        echo_v "Skipping ghcli"
      ;;
      *)
        echo_err "PACK_CHANGELOG_MODE has invalid value '${PACK_CHANGELOG_MODE}'!"
        echo_log "  Valid options are: full, release, or tag."
        echo_log "  Defaulting to: full"
        PACK_CHANGELOG_MODE="full"
        find_ghcli
      ;;
    esac
  fi
  find_eol_converter

  echo_log " "

  PACK_DESCRIPTION_FILE=$(locate_pdsc "$1")
  fail_and_bail $?
  echo_log "PDSC       is located in ${PACK_DESCRIPTION_FILE}"
  echo_log " "

  PACK_BASEDIR=$(dirname "${PACK_DESCRIPTION_FILE}")
  PACK_VENDOR=$(pdsc_vendor "${PACK_DESCRIPTION_FILE}")
  PACK_NAME=$(pdsc_name "${PACK_DESCRIPTION_FILE}")
  echo_log "Generating Pack for ${PACK_VENDOR}.${PACK_NAME}"

  PACK_BUILD=$(realpath "${PACK_BASEDIR}/${PACK_BUILD:-build}")
  PACK_OUTPUT=$(realpath "${PACK_BASEDIR}/${PACK_OUTPUT:-output}")
  PACK_BUILD_DESCRIPTION_FILE="${PACK_BUILD}/$(basename "${PACK_DESCRIPTION_FILE}")"

  echo_log " via ${PACK_BUILD}"
  echo_log " to ${PACK_OUTPUT}"
  echo_log ""

  if [ -d "${PACK_BUILD}" ]; then
    rm -r "${PACK_BUILD}"
  fi
  mkdir -p "${PACK_BUILD}"
  mkdir -p "${PACK_OUTPUT}"

  pushd "${PACK_BASEDIR}" >/dev/null || exit

  [[ ${PREPROCESS} == 1 && "$(type -t preprocess)" == "function" ]] && preprocess "${PACK_BUILD}"

  if [ -n "${CHANGELOG+x}" ]; then
    echo_log "Generating changelog with mode '${PACK_CHANGELOG_MODE}' ..."
    pdsc_update_releases "${PACK_DESCRIPTION_FILE}" "${PACK_BUILD_DESCRIPTION_FILE}" "${CHANGELOG}"
  else
    cp "${PACK_DESCRIPTION_FILE}" "${PACK_BUILD_DESCRIPTION_FILE}"
  fi

  add_dirs "${PACK_BUILD}"
  add_files "${PACK_BUILD}"
  delete_files "${PACK_BUILD}"
  apply_patches "${PACK_BUILD}"

  [[ ${POSTPROCESS} == 1 && "$(type -t postprocess)" == "function" ]] && postprocess "${PACK_BUILD}"

  check_schema "${PACK_BUILD_DESCRIPTION_FILE}"
  check_pack "${PACK_BUILD_DESCRIPTION_FILE}"
  create_sha1 "${PACK_BUILD}" "${PACK_VENDOR}" "${PACK_NAME}"

  PACK_ARCHIVE="${PACK_OUTPUT}/$(cat "${PACK_OUTPUT}/PackName.txt")"
  archive "${PACK_BUILD}" "${PACK_ARCHIVE}"

  if [[ $KEEP == 0 ]]; then
    echo_log "Cleaning up ${PACK_BUILD}..."
    rm -rf "${PACK_BUILD}"
    rm -f "${PACK_OUTPUT}/PackName.txt"
  fi

  echo_log ""
  echo_log "${LOG_COLOR_GREEN}Pack $(basename "${PACK_ARCHIVE}") generated successfully!"
}