Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix source upstream URL #65

Merged
merged 2 commits into from
May 20, 2023
Merged

Fix source upstream URL #65

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b5d9c79
fix source upstream URL
kgizdov May 17, 2023
8f5c3e4
remove dangling verbose-echo call
kgizdov May 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 78 additions & 58 deletions package/parse-submodules
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,20 +22,20 @@ __SILENT_PIPE='/dev/null'
__PIPE="${__SILENT_PIPE}"
__DRY_RUN=0

function err_echo {
function err-echo {
# echo to default error descriptor
>&2 echo "${*}"
}

function verbose_echo {
function verbose-echo {
# only print to screen when second argument is 1
[[ ${2} == 1 ]] && err_echo "${1}"
[[ ${2} == 1 ]] && err-echo "${1}"
return 0
}

function fail {
# fail with error message
err_echo "${1}"
err-echo "${1}"
exit "${2:-1}" # return a code specified by $2 or $1
}

Expand Down Expand Up @@ -70,34 +70,34 @@ helper options:

function check-submodules {
local _ref="${1:-${__GIT_REF}}"
verbose_echo "Checking if submodules exist for Git reference: ${_ref}" ${__VERBOSE_MODE}
verbose-echo "Checking if submodules exist for Git reference: ${_ref}" ${__VERBOSE_MODE}
git ls-tree --full-name --name-only -r "${_ref}" | grep .gitmodules &>${__PIPE}
}

function check-existance {
function check-existence {
local _repo_url="${1}"
verbose_echo "Checking access to Git repository at: ${_repo_url}" ${__VERBOSE_MODE}
verbose-echo "Checking access to Git repository at: ${_repo_url}" ${__VERBOSE_MODE}
git ls-remote "${_repo_url}" CHECK_GIT_REMOTE_URL_REACHABILITY &>${__PIPE}
}

function check-git-archive {
local _repo_url="${1}"
verbose_echo "Checking archive support of Git repository: ${_repo_url}" ${__VERBOSE_MODE}
verbose-echo "Checking archive support of Git repository: ${_repo_url}" ${__VERBOSE_MODE}
git archive --remote="${_repo_url}" --list &>${__PIPE}
}

function check-branch-or-tag {
local _repo_url="${1}"
local _ref="${2}"
verbose_echo "Checking if given Git reference '${_ref}' is a branch or tag name of repository at: ${_repo_url}" ${__VERBOSE_MODE}
verbose-echo "Checking if given Git reference '${_ref}' is a branch or tag name of repository at: ${_repo_url}" ${__VERBOSE_MODE}
local _branch="$(git ls-remote --symref "${_repo_url}" "${_ref}" | grep -v 'ref:' | awk '{sub(/refs\/(heads|tags)\//, "", $2); print $2}' | head -n1)"
if [[ -z $_branch ]]; then
verbose_echo "Reference in not a tag, branch or other symbolic reference that can be used for remote clone reference." ${__VERBOSE_MODE}
verbose-echo "Reference in not a tag, branch or other symbolic reference that can be used for remote clone reference." ${__VERBOSE_MODE}
return 1
elif [[ "$_branch" == "HEAD" ]]; then
verbose_echo "Reference 'HEAD' of Git repository at '${_repo_url}' points to the default branch and can be converted to a symbolic link for shallow clone." ${__VERBOSE_MODE}
verbose-echo "Reference 'HEAD' of Git repository at '${_repo_url}' points to the default branch and can be converted to a symbolic link for shallow clone." ${__VERBOSE_MODE}
else
verbose_echo "Reference branch or tag of Git repository at '${_repo_url}' is '"${_branch}"'" ${__VERBOSE_MODE}
verbose-echo "Reference branch or tag of Git repository at '${_repo_url}' is '"${_branch}"'" ${__VERBOSE_MODE}
fi
return 0
}
Expand All @@ -106,7 +106,7 @@ function get-file-from-archive {
local _repo_url="${1}"
local _ref="${2}"
local _module_file="${3}"
verbose_echo "Getting archive of ${_module_file} from ${_repo_url} with reference ${_ref}" ${__VERBOSE_MODE}
verbose-echo "Getting archive of ${_module_file} from ${_repo_url} with reference ${_ref}" ${__VERBOSE_MODE}
local gmf="$(git archive --remote="${_repo_url}" "${_ref}" "${_module_file}" 2>/dev/null)"
echo "${gmf}" | tar -x
local _ret=$?
Expand All @@ -115,28 +115,28 @@ function get-file-from-archive {

function get-submodules-file {
local _ref="${1:-${__GIT_REF}}"
verbose_echo "Checking if submodule config file exists for Git reference: ${_ref}" ${__VERBOSE_MODE}
verbose-echo "Checking if submodule config file exists for Git reference: ${_ref}" ${__VERBOSE_MODE}
git ls-tree --full-name --name-only -r "${_ref}" | grep '.gitmodules' | head -n1
}


function get-short-hash {
local _repo_url="${1}"
local _ref="${2}"
verbose_echo "Getting short hash of Git reference '${_ref}' from repository at: ${_repo_url}" ${__VERBOSE_MODE}
verbose-echo "Getting short hash of Git reference '${_ref}' from repository at: ${_repo_url}" ${__VERBOSE_MODE}
local _short_ref="$(git ls-remote "${_repo_url}" "${_ref}" | awk '{ print substr($1,1,10) }' | head -n1)"
if [[ -z $_short_ref ]]; then
_short_ref="$(echo "${_ref}" | awk '{ print substr($1,1,10) }')"
verbose_echo "Reference in not symbolic. Shortening '${_ref}' to '${_short_ref}'" ${__VERBOSE_MODE}
verbose-echo "Reference in not symbolic. Shortening '${_ref}' to '${_short_ref}'" ${__VERBOSE_MODE}
fi
verbose_echo "Short hash of Git reference '${_ref}' from repository '${_repo_url}' is: ${_short_ref}" ${__VERBOSE_MODE}
verbose-echo "Short hash of Git reference '${_ref}' from repository '${_repo_url}' is: ${_short_ref}" ${__VERBOSE_MODE}
echo "${_short_ref}"
}

function get-fragment {
local _repo_url="${1}"
local _ref="${2}"
verbose_echo "Getting fragment tag for Git reference '${_ref}' from repository at: ${_repo_url}" ${__VERBOSE_MODE}
verbose-echo "Getting fragment tag for Git reference '${_ref}' from repository at: ${_repo_url}" ${__VERBOSE_MODE}
local _fragment="$(git ls-remote --tags "${_repo_url}" "${_ref}")"
if [[ ! -z $_fragment ]]; then
echo '#tag='"${_ref}"
Expand All @@ -156,26 +156,26 @@ function get-branch-or-tag {
local _ref="${2}"
local _branch=""
if ! check-branch-or-tag "${_repo_url}" "${_ref}"; then
verbose_echo "Cannot get branch or tag name for Git reference '${_ref}' from repository at '${_repo_url}'." ${__VERBOSE_MODE}
verbose-echo "Cannot get branch or tag name for Git reference '${_ref}' from repository at '${_repo_url}'." ${__VERBOSE_MODE}
echo ''
return 1
elif [[ "${_ref}" == "HEAD" ]]; then
_branch="$(git ls-remote --symref "${_repo_url}" HEAD | grep 'ref:' | awk '{sub(/refs\/(heads|tags)\//, "", $2); print $2}' | head -n1)"
verbose_echo "Converting 'HEAD' to default branch of Git repository at '${_repo_url}': '"${_branch}"'" ${__VERBOSE_MODE}
verbose-echo "Converting 'HEAD' to default branch of Git repository at '${_repo_url}': '"${_branch}"'" ${__VERBOSE_MODE}
else
verbose_echo "Getting branch or tag name of Git reference '${_ref}' from repository at: ${_repo_url}" ${__VERBOSE_MODE}
verbose-echo "Getting branch or tag name of Git reference '${_ref}' from repository at: ${_repo_url}" ${__VERBOSE_MODE}
_branch="$(git ls-remote --symref "${_repo_url}" "${_ref}" | grep -v 'ref:' | awk '{sub(/refs\/(heads|tags)\//, "", $2); print $2}' | head -n1)"
fi
verbose_echo "Reference branch or tag of Git repository at '${_repo_url}' is '"${_branch}"'" ${__VERBOSE_MODE}
verbose-echo "Reference branch or tag of Git repository at '${_repo_url}' is '"${_branch}"'" ${__VERBOSE_MODE}
echo "${_branch}"
}

function get-reference-hash {
local _repo_url="${1}"
local _ref="${2}"
verbose_echo "Getting hash of Git reference '${_ref}' from repository at: ${_repo_url}" ${__VERBOSE_MODE}
verbose-echo "Getting hash of Git reference '${_ref}' from repository at: ${_repo_url}" ${__VERBOSE_MODE}
local _hash="$(git ls-remote "${_repo_url}" "${_ref}" | awk '{print $1}' | head -n1)"
verbose_echo "Hash of Git reference '${_ref}' from repository '${_repo_url}' is '"${_hash}"'" ${__VERBOSE_MODE}
verbose-echo "Hash of Git reference '${_ref}' from repository '${_repo_url}' is '"${_hash}"'" ${__VERBOSE_MODE}
echo "${_hash}"
}

Expand Down Expand Up @@ -212,64 +212,64 @@ function get-module-paths {
function get-proto {
# Extract the protocol (includes trailing "://").
local __PROTO="$(echo "${1}" | sed -nr 's,^(.*://).*,\1,p')"
verbose_echo "URL protocol: ${__PROTO}" ${__VERBOSE_MODE}
verbose-echo "URL protocol: ${__PROTO}" ${__VERBOSE_MODE}
echo "${__PROTO}"
}

function get-url-noproto {
# Remove the protocol from the URL.
local __URL="$(echo ${1/$(get-proto "${1}")/})"
verbose_echo "URL without protocol: ${__URL}" ${__VERBOSE_MODE}
verbose-echo "URL without protocol: ${__URL}" ${__VERBOSE_MODE}
echo "${__URL}"
}

function get-url-noproto-nouser {
# Remove the protocol from the URL.
local __noproto="$(get-url-noproto "${1}")"
local __URL="$(echo ${__noproto/$(get-user "${1}")/})"
verbose_echo "URL without protocol and without user: ${__URL}" ${__VERBOSE_MODE}
verbose-echo "URL without protocol and without user: ${__URL}" ${__VERBOSE_MODE}
echo "${__URL}"
}

function get-url-noproto-nouser-noport {
# Remove the protocol from the URL.
local __noproto_nouser="$(get-url-noproto-nouser "${1}")"
local __URL="$(echo ${__noproto_nouser/$(get-port "${1}")/})"
verbose_echo "URL without protocol, user or port: ${__URL}" ${__VERBOSE_MODE}
verbose-echo "URL without protocol, user or port: ${__URL}" ${__VERBOSE_MODE}
echo "${__URL}"
}

function get-user {
# Extract the user (includes trailing "@").
local __USER="$(echo "$(get-url-noproto "${1}")" | sed -nr 's,^(.*@).*,\1,p')"
verbose_echo "USER: ${__USER}" ${__VERBOSE_MODE}
verbose-echo "USER: ${__USER}" ${__VERBOSE_MODE}
echo "${__USER}"
}

function get-port {
local __noproto_nouser="$(get-url-noproto-nouser "${1}")"
local __PORT="$(echo "${__noproto_nouser}" | sed -nr 's,.*(:[0-9]+).*,\1,p')"
verbose_echo "PORT: ${__USER}" ${__VERBOSE_MODE}
verbose-echo "PORT: ${__USER}" ${__VERBOSE_MODE}
echo "${__PORT}"
}

function get-path {
local __PATH="$(echo "$(get-url-noproto-nouser-noport "${1}")" | sed -nr 's,[^/:]*([/:].*),\1,p')"
verbose_echo "PATH: ${__USER}" ${__VERBOSE_MODE}
verbose-echo "PATH: ${__USER}" ${__VERBOSE_MODE}
echo "${__PATH}"
}

function get-host {
local __noproto_nouser_noport="$(get-url-noproto-nouser-noport "${1}")"
local __HOST="$(echo ${__noproto_nouser_noport/$(get-path "${1}")/})"
verbose_echo "HOST: ${__USER}" ${__VERBOSE_MODE}
verbose-echo "HOST: ${__USER}" ${__VERBOSE_MODE}
echo "${__HOST}"
}

function get-repo-name {
local _repo_url="${1}"
local _repo_name="$(basename --suffix='.git' "${_repo_url}")"
verbose_echo "Git repository at ${_repo_url} is named: ${_repo_name}" ${__VERBOSE_MODE}
verbose-echo "Git repository at ${_repo_url} is named: ${_repo_name}" ${__VERBOSE_MODE}
echo "${_repo_name}"
}

Expand All @@ -282,47 +282,66 @@ function get-repo-suffix-path {
echo "${__PATH}"
}

function shallow_clone {
function shallow-clone {
local _repo_url="${1}"
local _ref="${2}"
local _destdir="${3:-"./$(get-repo-name "${_repo_url}")"}"
local _symref="$(get-branch-or-tag "${_repo_url}" "${_ref}")"
verbose_echo "Shallow-cloning ${_repo_url} at reference ${_symref} into ${_destdir}." ${__VERBOSE_MODE}
verbose-echo "Shallow-cloning ${_repo_url} at reference ${_symref} into ${_destdir}." ${__VERBOSE_MODE}
git clone --no-checkout --depth 1 -b "${_symref}" "${_repo_url}" "${_destdir}" &>${__PIPE} || fail "Repository shallow clone failed." 1
}

function full_clone {
function full-clone {
local _repo_url="${1}"
local _ref="${2}"
local _destdir="${3:-"./$(get-repo-name "${_repo_url}")"}"
local _pwd="$(pwd)"
verbose_echo "Cloning ${_repo_url} at reference ${_ref} into ${_destdir}." ${__VERBOSE_MODE}
verbose-echo "Cloning ${_repo_url} at reference ${_ref} into ${_destdir}." ${__VERBOSE_MODE}
git clone "${_repo_url}" "${_destdir}" &>${__PIPE} || fail "Repository clone failed." 1
cd "${_destdir}"
git reset --hard "${_ref}" &>${__PIPE} || fail "Reference ${_ref} checkout failed." 1
cd "${_pwd}"
}

function print_submodule_url {
function git-remote-prefix {
local _repo_url="${1}"
echo "$(get-proto "${_repo_url}")$(get-user "${_repo_url}")$(get-host "${_repo_url}")$(get-port "${_repo_url}")"
}

function git-vcs-url {
local _repo_url="${1}"
local _proto="$(get-proto "${_repo_url}")"
_remote_prefix="$(git-remote-prefix "${_repo_url}")"
_repo_url="${_remote_prefix}${_repo_url##${_remote_prefix}}"
if [[ "${_proto// /}" == 'git://' ]]; then
echo "${_repo_url}"
elif [[ "${_repo_url:0:3}" == 'git@' ]]; then
echo "git+ssh://${_repo_url}"
elif [[ -n "${_proto// /}" ]]; then
echo "git+${_repo_url}"
else
echo "git://${_repo_url}"
fi
}

function print-submodule-url {
local _mod_url="${1}"
local _remote_prefix="${2}"
local _repo_url="${3}"
local _outfile="${4}"
local _repo_url="${2}"
local _outfile="${3}"
local _remote_prefix="$(git-remote-prefix "${_repo_url}")"
local _srcline=""
if ! check-existance "${_mod_url}"; then

if ! check-existence "${_mod_url}"; then
_srcline="${_remote_prefix}$(get-repo-suffix-path "${_repo_url}" "${_mod_url}")"
else
_srcline="${_mod_url}"
fi
if [[ "${_srcline:0:3}" == 'git' ]]; then
_srcline=' "${pkgname}-'"$(get-repo-name "${_mod_url}")::${_srcline}"'"'
else
_srcline=' "${pkgname}-'"$(get-repo-name "${_mod_url}")::git+${_srcline}"'"'
fi
_srcline=' "${pkgname}-'"$(get-repo-name "${_mod_url}")::$(git-vcs-url "${_srcline}")"'"'

echo "${_srcline}" >>"${_outfile}"
}

function print_git_config {
function print-git-config {
local _mod_url="${1}"
local _name="${2}"
local _outfile="${3}"
Expand Down Expand Up @@ -382,21 +401,23 @@ if [ -z $__GIT_REMOTE ]; then
fail "$(help)" 1
fi

# format

# this needs to happen before checking for default branch
if ! check-existance "${__GIT_REMOTE}"; then
if ! check-existence "${__GIT_REMOTE}"; then
fail "Git repository ${__GIT_REMOTE} not found or not accessible." 1
fi

if [ -z $__GIT_REF ]; then
__GIT_REF='HEAD'
fi

verbose_echo "Checking repository ${__GIT_REMOTE} with ref ${__GIT_REF}..." ${__VERBOSE_MODE}
verbose-echo "Checking repository ${__GIT_REMOTE} with ref ${__GIT_REF}..." ${__VERBOSE_MODE}

# can we use git archive
__USE_ARCHIVE=1
if ! check-git-archive "${__GIT_REMOTE}"; then
verbose_echo "Git remote of ${__GIT_REMOTE} does not support git-archive. Cloning entire repository..." ${__VERBOSE_MODE}
verbose-echo "Git remote of ${__GIT_REMOTE} does not support git-archive. Cloning entire repository..." ${__VERBOSE_MODE}
__USE_ARCHIVE=0
fi

Expand All @@ -408,10 +429,10 @@ else
# clone repository locally
if ! check-branch-or-tag "${__GIT_REMOTE}" "${__GIT_REF}" ; then
# shallow clone is not possible
err_echo "Warning: Shallow clone is not possible for exact commit hashes. Preferably use a tag for better performance."
full_clone "${__GIT_REMOTE}" "${__GIT_REF}" "${gcldir}"
err-echo "Warning: Shallow clone is not possible for exact commit hashes. Preferably use a tag for better performance."
full-clone "${__GIT_REMOTE}" "${__GIT_REF}" "${gcldir}"
else
shallow_clone "${__GIT_REMOTE}" "${__GIT_REF}" "${gcldir}"
shallow-clone "${__GIT_REMOTE}" "${__GIT_REF}" "${gcldir}"
fi
cd "${gcldir}"
if ! check-submodules "${__GIT_REF}"; then
Expand All @@ -425,12 +446,11 @@ touch "${outfile}"
__FRAGMENT="$(get-fragment "${__GIT_REMOTE}" "${__GIT_REF}")"
echo '# Your source array should look something like this:
source=(
"${pkgname}::'${__GIT_REMOTE}${__FRAGMENT}'"' >>"${outfile}"
__REMOTE_PREFIX="$(get-proto "${__GIT_REMOTE}")$(get-user "${__GIT_REMOTE}")$(get-host "${__GIT_REMOTE}")$(get-port "${__GIT_REMOTE}")"
"${pkgname}::'$(git-vcs-url "${__GIT_REMOTE}")${__FRAGMENT}'"' >>"${outfile}"
for name in $(get-module-names "${gmdfile}"); do
__mod_url="$(get-module-url "${gmdfile}" "${name}")"
# this is slow so run in parallel, store PIDs and wait on all later
print_submodule_url "${__mod_url}" "${__REMOTE_PREFIX}" "${__GIT_REMOTE}" "${outmodurlfile}" &
print-submodule-url "${__mod_url}" "${__GIT_REMOTE}" "${outmodurlfile}" &
pids="$pids $!"
done
for pid in $pids; do
Expand All @@ -448,7 +468,7 @@ prepare() {
' >>"${outfile}"
for name in $(get-module-names "${gmdfile}"); do
__mod_url="$(get-module-url "${gmdfile}" "${name}")"
print_git_config "${__mod_url}" "${name}" "${outgitconffile}"
print-git-config "${__mod_url}" "${name}" "${outgitconffile}"
done
# print sorted for consistency and more readable diffs
LC_ALL="${__ENV_LC_ALL}" sort "${outgitconffile}" >>"${outfile}"
Expand Down