git-zclone

#!/bin/bash

### Automate cloning of local Git repository stored as a dedicated ZFS dataset.
### Also can clone a remote (https, ssh) repository into a new ZFS dataset.
### This supports creation of the clone mounted into a specified absolute path
### or into a relative path near the original repo. The user must have proper
### "zfs allow" delegated permissions for source and target datasets, list of
### specific required and/or useful permissions is suggested in the code below.
### The new dataset is created (subject to "zfs allow" permissions as well as
### POSIX permissions to create the mountpoint) under the hierarchy node which
### would contain its new mountpoint.
### Currently this clones only one dataset and one repo (non-recursive).
### After the initial cloning, the new repo is adjusted to reference its local
### filesystem parent as the default 'origin' (any initial 'origin' is renamed
### into 'origin-parent') - this can be disabled by '-NR' argument.
### If the NetBeans workspace metadata is found - then the NB Project instance
### is renamed in the metadata files according to new dataset base-name.
### Parameters specify the filesystem paths, just like ordinary "git clone" for
### local filesystem source and destination.
### Note that ZFS-based repository cloning also refers the non-repo cruft in
### the workspace, such as build products if in-tree builds were executed here.
### However, you can use the file:/// URL scheme to name the source, to clone
### an original local git repository into a dedicated dataset using regular
### "git clone" under the hood, after which you have a dedicated dataset with
### just the fresh repository.
### Also note that this did not yet intend to support cloning of bare repos.
###
### Copyright (C) 2013-2022 by Jim Klimov <jim@jimklimov.com>, License: MIT
###     https://github.com/jimklimov/git-scripts

do_git_zfs_clone() {
PATH=/usr/gnu/bin:/usr/sfw/bin:/opt/sfw/bin:/opt/omniti/bin:/bin:\
/sbin:/usr/sbin:$PATH
LANG=C
LC_ALL=C
export PATH LANG LC_ALL

### Default values for command-line controllable flags and options
CHECKOUT_BRANCH=""
CREATE_BRANCH_FLAG=""
REMOTE_UPSTREAM=""
REMOTE_UPSTREAM_CANPUSH="no"
REWRITE_ORIGIN="yes"
WIPE_WORKSPACE="no"

usage() {
	# NOTE: No TABs in the usage markup, so we stay in 80-column formatting
	cat <<EOH
This Git plugin allows to clone a local or remote git repo into a dedicated ZFS
filesystem dataset. It requires that your system does have ZFS support.
Note that local repos are zfs-cloned including any cruft in the workspace, and
remote URLs (including file:/// schema) are git-cloned without untracked files.
To clone a directory which is not a dedicated dataset you must use URL schema.

Usage: $0 FROM [DEST] [-b|--branch BNAME] [-B|--new-branch BNAME]
        [-U|--upstream|--set-upstream|-URW URL] [-NR|--no-rewrite-origin]
    FROM    Required argument - the local (ZFS dataset) or remote (any URL)
            source repository to be cloned into DEST
    DEST    Local path to be created; defaults to basename of FROM (without the
            optional .git extension) if omitted or empty
When both FROM and DEST are defined, further arguments can be passed:
    -b NAME    After cloning, checkout the branch NAME as the current workspace
    -B NAME    After cloning, create+checkout branch NAME as current workspace
    -U URL    Set (maybe replace) the "upstream" repo reference to specified URL
            NOTE: "-U" also explicitly invalidates push_url so even a maintainer
            can not corrupt the common-upstream repository by mistake; use the
            "-URW URL" variant to allow not-invalidating the push_url
            NOTE: local ZFS-cloned repos likely have an upstream already set up
    -NR|--no-rewrite-origin    By default, this script manipulates the "origin"
            and backup "origin-parent" repo references for local clones (where
            FROM is a local ZFS dataset); this option disables this activity
    --wipe { -n | -f | -i } Run "git clean" in the zfs-cloned workspace to try
            removing any build products and other untracked data (except the
            netbeans dir, if present); -n = dry-run, -i = interact, -f = force
EOH
}

# Verify that we have a usable ZFS command
zfs --help 2>&1 | GREP_OPTIONS= egrep -i 'dataset|snapshot|clone|volume' >/dev/null || \
	{ usage; echo "FATAL: No suitable 'zfs' command found!" >&2 && return 2; }

# Currently we accept at least two CLI parameters: FROM and DEST
#[ $# -lt 2 -o -z "$1" -o -z "$2" ] && \
#	echo "Required: FROM and DEST replica paths" >&2 && \
#	echo " FROM must be a Git repo in a dedicated ZFS dataset, and" >&2 && \
#	echo " DEST must not block ZFS mount (may be an empty directory)" >&2 && \
#	return 1

FROM="$1"
### Remove the original CLI parameters such as FROM and DEST
shift

# This only inspects the first argument; a more generic loop is below
case "$FROM" in
	-h|-help|--help)
		usage
		return 0
		;;
	*://*|*@*:*)
		# TODO: This misses the "alternative scp-like syntax"
		# aka   [user@]host.xz:path/to/repo.git/
		echo "INFO: Detected the FROM repository as a remote URL"
		FROM_TYPE=URL
		# No local sanity checks at this moment
		;;
	*)
		FROM_TYPE=LOCAL
		# Other sanity checks
		[ -z "$FROM" -o ! -d "$FROM" ] && \
			echo "Required: FROM must exist as a directory" >&2 && return 1
		FROMPATH="`cd "$FROM" && pwd`" || FROMPATH=""
		[ -z "$FROMPATH" ] && \
			echo "Required: FROM must exist as a directory" >&2 && return 1
		[ ! -d "$FROM/.git" ] && \
			echo "Required: FROM must be a Git repository" >&2 && return 1
		[ ! -d "$FROM/.zfs/snapshot" ] && { \
			echo "Required: FROM must be a ZFS POSIX dataset"
			echo "To clone from arbitrary local directory, please use URL format, e.g.:"
			echo "    $0 file://$FROM $*"
			} >&2 && return 1

		FROMDS="`cd "$FROMPATH" && /bin/df -k . | GREP_OPTIONS= grep / | head -1 | awk '{print $1}'`" || \
			FROMDS=""
		[ -n "$FROMDS" ] && [ "$FROMDS" != '-' ] || \
			{ echo "Required: FROM must be a mounted ZFS POSIX dataset" >&2 && \
			  return 1 ; }

		# Validate both ZFS command and FROM dataset name
		echo "=== FROM dataset:"
		zfs list -o \
			mountpoint,space,refer,lrefer,dedup,compression,compressratio,sharenfs,sharesmb \
			"$FROMDS"
		[ $? != 0 ] && \
			{ echo "FATAL: Couldn't use ZFS to review dataset properties" >&2 && \
			  return 1 ; }
		;;
esac

### If we have another parameter, snatch it as the destination
DEST=""
[ $# -gt 0 ] && \
	DEST="$1" && \
	shift

if [ -z "$DEST" ]; then
	echo "WARN: No explicit DEST was provided, so assigning one as a basename of FROM" >&2
	echo "      Note that this may fail due to conflicts a bit later..." >&2
	DEST="`basename "$FROM" .git`"
fi

# Verify that "DEST" is an empty directory or does not exist
BAD=no
if [ -n "$DEST" ] && [ -e "$DEST" ]; then
	BAD=yes
	[ -d "$DEST" ] && \
		( cd "$DEST" && [ x"`find . 2>/dev/null | GREP_OPTIONS= grep -v ./.zfs`" = x. ] ) && \
		BAD=no
fi
[ x"$BAD" = xyes ] && \
	echo "Required: DEST must not exist as a non-empty directory" >&2 && \
	return 1

### Process optional further CLI parameters
while [ $# -gt 0 ]; do
	case "$1" in
		-h|-help|--help)
			usage
			return 0
			;;
		-b|--branch)
			CHECKOUT_BRANCH="$2"
			shift ;;
		-B|--new-branch)
			### Note: if the named branch exists, it is just checked out and no
			### error propagates into the return-code (that we couldn't create it)
			CREATE_BRANCH_FLAG="-b"
			CHECKOUT_BRANCH="$2"
			shift ;;
		-U|-URO|--upstream|--set-upstream)
			REMOTE_UPSTREAM="$2"
			REMOTE_UPSTREAM_CANPUSH="no"
			shift ;;
		-URW)
			REMOTE_UPSTREAM="$2"
			REMOTE_UPSTREAM_CANPUSH="yes"
			shift ;;
		-NR|--no-rewrite-origin) REWRITE_ORIGIN=no ;;
		--wipe) case "$2" in
				-n|-f|-i) WIPE_WORKSPACE="$2"; shift ;;
				*) echo "FATAL: git-zclone: Unknown argument to --wipe: '$2'" >&2; return 1 ;;
			esac
			;;
		*) echo "FATAL: git-zclone currently does not support parameter '$1'" >&2
			return 1
			;;
	esac
	shift
done

#################################################################################
# TODO: Maybe fallback to some other timestamp if this fails?
# Perhaps a current latest git commit of the original repo?
# Either way, some tag is needed to create the (LOCAL) snapshot
# and then the clone.
TS="initialClone"
if [ x"$FROM_TYPE" = xLOCAL ]; then
	TS="`date -u '+%Y%m%dT%H%M%SZ'`" || TS=""
	[ -z "$TS" ] && \
		echo "FATAL: Can't determine current timestamp" >&2 && return 1
fi

# Prerequisites seem good, begin non-readonly activity...
[ -d "$DEST" ] || mkdir -p "$DEST"
[ $? != 0 ] && \
	echo "FATAL: Couldn't create the paths up to DEST" >&2 && return 1
DESTPATH="`cd "$DEST" && pwd`" || DESTPATH=""
[ -z "$DESTPATH" ] && \
	echo "FATAL: Couldn't change the path into DEST" >&2 && return 1

# Determine the dataset that will hold the new repo and its mountpoint
DESTDS=""
DESTMPT=""
if [ -d "$DESTPATH/.zfs/snapshot" ]; then
	# This seems like a dataset already? Would be fatal later on for LOCAL.
	DESTDS="`cd "$DESTPATH" && /bin/df -k . | GREP_OPTIONS= grep / | head -1 | awk '{print $1}'`" && \
	[ "$DESTDS" != '-' ] || \
	DESTDS=""
fi

if [ -z "$DESTDS" -a -d "$DESTPATH/../.zfs/snapshot" ]; then
	# The (future) direct parent of DEST is a dataset?
	DESTDS="`cd "$DESTPATH/.." && /bin/df -k . | GREP_OPTIONS= grep / | head -1 | awk '{print $1}'`"/"`basename "$DESTPATH"`" || \
	DESTDS=""

	[ x"`basename "$DESTPATH"`" != x"`basename "$DEST"`" ] && \
		DESTMPT="$DESTPATH"
fi

if [ -z "$DESTDS" ]; then
	case "$FROM_TYPE" in
	LOCAL)
		# Finally, try to spawn the clone "near" the original repo
		# Note that if it is mounted in an explicit location
		DESTDS="`dirname "$FROMDS"`/`basename "$DESTPATH"`"
		[ x"`dirname "$FROMPATH"`/`basename "$DEST"`" != x"$DESTPATH" ] && \
			DESTMPT="$DESTPATH"
		;;
	URL)
		# Finally, try to spawn the clone under current directory's dataset
		DESTDS="`/bin/df -k . | GREP_OPTIONS= grep / | head -1 | awk '{print $1}'`/`basename "$DESTPATH"`" || \
		DESTDS=""
		[ x"`pwd`/`basename "$DEST"`" != x"$DESTPATH" ] && \
			DESTMPT="$DESTPATH"
		case "$DESTDS" in
		/*|*:*|swap|"")	# Local non-ZFS path or an NFS mount
			echo "FATAL: The container backing store DESTDS='$DESTDS' does not seem like a local ZFS dataset!" >&2
			echo "Try:   git clone '$FROM' '$DEST'" >&2
			return 1
			;;
		esac
		;;
	esac
fi

case "$FROM_TYPE" in
	LOCAL)
		# Validate both ZFS command and DEST dataset name
		zfs list -o name "$DESTDS" >/dev/null 2>&1 && \
			echo "FATAL: DEST ZFS dataset $DESTDS already exists?" \
			     "Can't clone into it!" >&2 && return 2

		echo "=== Cloning from '$FROMDS@git-auto-snap-clone-$TS'" \
		     "to '$DESTDS' (mountpoint=$DESTMPT)..."

		zfs snapshot -r "$FROMDS@git-auto-snap-clone-$TS" || \
		{ RES=$?; echo "FATAL: Could not snapshot the dataset" \
			"'$FROMDS@git-auto-snap-clone-$TS'" >&2; return $RES; }

		zfs clone -p -o mountpoint=none \
			"$FROMDS@git-auto-snap-clone-$TS" "$DESTDS" || \
		{ RES=$?
		  echo "FATAL: Could not clone the dataset"
		  echo "  from '$FROMDS@git-auto-snap-clone-$TS'"
		  echo "  to '$DESTDS'"
		  echo "Please make sure that dataset name is valid and that proper 'zfs allow'"
		  echo "permissions were set on destination container dataset, e.g. at least:"
		  echo "  sudo zfs allow -ldu $USER clone,create,destroy,diff,mount,promote,rollback,snapshot,share,sharenfs,sharesmb,canmount,mountpoint `dirname $DESTDS`"
		  echo "Maybe also allow: send,receive,dedup,compression,hold,release ..."
		  echo "::: Current set of destination container permissions is:"
		  zfs allow "`dirname $DESTDS`"
		  echo ""
		  echo "::: Current set of original repository dataset permissions is:"
		  zfs allow "$FROMDS"
		  echo ""
		  echo "FATAL: Dataset cloning FAILED"
		  return $RES; } >&2

		echo "=== Trying to carry over some properties of the source dataset..."
		EXCLUDE_ATTRS='canmount|mountpoint'
		zfs get all "$FROMDS" | GREP_OPTIONS= egrep ' (local|received)$' | \
			GREP_OPTIONS= egrep -v "$EXCLUDE_ATTRS" | while read _D A V _T; do \
				echo "$A=$V"; zfs set "$A=$V" "$DESTDS"; \
			done
		# TODO: zfs allow permissions if set directly on FROMDS?
		# TODO: zfs smb share ACL files?
		# TODO: sub-datasets?
		;;
	URL)
		if zfs list -o name "$DESTDS" >/dev/null 2>&1 ; then
			echo "NOTE: DEST ZFS dataset $DESTDS already exists (and is empty)"
		else
			echo "Creating new DEST ZFS dataset $DESTDS for this repository clone..."
			zfs create "$DESTDS" || \
			{ RES=$?
			  echo "FATAL: Could not create the dataset '$DESTDS' !"
			  echo "Please make sure that dataset name is valid and that proper 'zfs allow'"
			  echo "permissions were set on destination container dataset, e.g. at least:"
			  echo "  sudo zfs allow -ldu $USER clone,create,destroy,diff,mount,promote,rollback,snapshot,share,sharenfs,sharesmb,canmount,mountpoint `dirname $DESTDS`"
			  echo "You may also want to ensure that your parent datasets propagate compression"
			  echo "and other useful settings to their descendants."
			  return $RES; } >&2
		fi
		;;
	*)
		echo "FATAL: Unimplemented FROM_TYPE='$FROM_TYPE'!" >&2
		return 3
		;;
esac

RES=0
if [ -n "$DESTMPT" ]; then
	zfs set mountpoint="$DESTMPT" "$DESTDS" || RES=$?
else
	zfs inherit mountpoint "$DESTDS" || RES=$?
fi
[ $RES != 0 ] && \
	echo "FATAL: Could not set '$DESTDS' mountpoint='$DESTMPT'" >&2 && \
	return $RES

echo "=== DEST dataset stats:"
zfs list -o \
	mountpoint,space,refer,lrefer,dedup,compression,compressratio,sharenfs,sharesmb \
	"$DESTDS"
[ $? != 0 ] && \
	echo "FATAL: Couldn't use ZFS to review dataset properties" >&2 && \
	return 1

echo ""
echo "=== Review ZFS ALLOW settings:"
if [ "$FROM_TYPE" = LOCAL ]; then
	echo "===== FROM $FROMDS:"
	zfs allow "$FROMDS"
	echo ""
fi
echo "===== DEST $DESTDS:"
zfs allow "$DESTDS"
echo ""

case "$FROM_TYPE" in
	LOCAL)
		if [ x"$REWRITE_ORIGIN" = xyes ] ; then
			echo "=== Rewriting Git 'origin' of the new repo for automated sync with the parent"
			echo "    (in local filesystem); retaining the original 'origin' as 'origin-parent'"
			ORIGIN="$FROMPATH"
			[ x"`dirname "$FROMPATH"`" = x"`dirname "$DESTPATH"`" ] && \
				ORIGIN="../`basename "$FROMPATH"`"
			( cd "$DESTPATH" && {
			  git remote rm origin-parent || echo "OK_TO_FAIL"
			  git remote rename origin origin-parent || echo "OK_TO_FAIL"
			  git remote add origin "$ORIGIN" && \
			  echo "===== Refresh Git tracking of origin..." && \
			  git pull --all; } ) || \
			{ RES=$?; echo "FATAL: Could not set Git origin URL"; return $RES; } >&2
			### This was fatal because maybe "git" or "cd" failed, or something?
		else
			echo "NOTE: as requested, I did not mangle git origin for the local clone"
		fi
		;;
	URL)
		echo "=== Cloning remote Git repo..." && \
		cd "$DESTPATH" && git clone "$FROM" . || \
		{ RES=$?
		  echo "FATAL: Could not clone remote Git repo '$FROM' to local dataset '$DESTDS' mounted at '$DESTPATH'"
		  return $RES; } >&2
		;;
esac

if [ -n "$REMOTE_UPSTREAM" ]; then
	echo ""
	echo "=== Setting Git 'upstream' of the new repo to be '$REMOTE_UPSTREAM'"
	OLD_UPSTREAM="`cd "$DESTPATH" && git remote -v | GREP_OPTIONS= grep -w upstream`" 2>/dev/null \
		&& [ -n "$OLD_UPSTREAM" ] || OLD_UPSTREAM=""
	[ -n "$OLD_UPSTREAM" ] && \
		echo "=== Old Git 'upstream' setup defined in the new repo was:" && \
		echo "$OLD_UPSTREAM"
	( cd "$DESTPATH" && {
	  git remote rm upstream || echo "OK_TO_FAIL"
	  git remote add upstream "$REMOTE_UPSTREAM" && \
	  if [ x"$REMOTE_UPSTREAM_CANPUSH" = xno ]; then
	    echo "===== Disabling ability to Git push into the upstream..."
	    git remote set-url --push upstream "no_push" || return
	  else : ; fi && \
	  echo "===== Refresh Git tracking of upstream..." && \
	  git pull --all; } ) || \
	{ RES=$?; echo "FATAL: Could not set Git upstream URL and/or pull it"; return $RES; } >&2
	### This was fatal because maybe "git" or "cd" failed, or something?
fi

if [ -n "$CHECKOUT_BRANCH" ]; then
	echo ""
	[ -z "$CREATE_BRANCH_FLAG" ] && \
		echo "=== Checking out Git branch '$CHECKOUT_BRANCH' in the new repo" || \
		echo "=== Creating and checking out Git branch '$CHECKOUT_BRANCH' in the new repo"

	( cd "$DESTPATH" && \
	  { [ -z "$CREATE_BRANCH_FLAG" ] || \
	    git checkout $CREATE_BRANCH_FLAG "$CHECKOUT_BRANCH"; } || \
	  git checkout "$CHECKOUT_BRANCH" ) || \
	{ RES=$?; echo "WARN: Could not change Git branch"; } >&2
fi

echo ""
echo "=== Rewriting Git branch tracking for the current branch of the new repo"
echo "    for automated sync with the parent one"
( cd "$DESTPATH" && {
  BRANCH="`git rev-parse --abbrev-ref HEAD`" || BRANCH=""
  # NOTE: '--set-upstream-to' May be unsupported in Git-1.x
  [ -z "$BRANCH" ] || \
    git branch --set-upstream-to=origin/"$BRANCH" "$BRANCH" 2>/dev/null || \
    git branch --set-upstream "$BRANCH" origin/"$BRANCH"; } ) || \
{ RES_T=$?; echo "WARN: Could not modify Git branch tracker"
  [ -n "$CREATE_BRANCH_FLAG" ] && echo "OK_TO_FAIL" || RES=$RES_T; } >&2

if [ -d "$DESTPATH/nbproject" -a -s "$DESTPATH/nbproject/project.xml" ]; then
	echo ""
	echo "=== Rewriting NetBeans project name..."
	( cd "$DESTPATH/nbproject" && \
	  cp -pf project.xml project.xml.orig && \
	  sed 's|<name>'"`basename "$FROMPATH"`"'</name>|<name>'"`basename "$DESTPATH"`"'</name>|' \
	  < project.xml.orig > project.xml ) || \
	{ RES=$?; echo "WARN: Could not modify NetBeans project name"; } >&2
fi

if [ x"$WIPE_WORKSPACE" != x"no" ]; then
	case "$FROM_TYPE" in
	LOCAL)
		echo ""
		echo "=== Wiping newly cloned workspace from files not tracked by Git:"
		( cd "$DESTPATH" && { \
			echo "    git clean -d -x -e 'nbproject' $WIPE_WORKSPACE" ; \
			git clean -d -x -e 'nbproject' $WIPE_WORKSPACE ; \
		  } ) || \
			{ RES=$?; echo "WARN: Could not clean up workspace"; } >&2
		;;
	*)
		echo ""
		echo "=== SKIP: Wiping of newly cloned workspace is a no-op for a non-ZFS clone"
		;;
	esac
fi

echo ""
echo "=== Resulting known branches and remote repos:"
( cd "$DESTPATH" && git remote -v ) || RES=$?
( cd "$DESTPATH" && git branch -a ) || RES=$?
( echo ""; echo "=== Current branch:"
  cd "$DESTPATH" && git branch -a | GREP_OPTIONS= egrep '^\* ' ) || RES=$?

echo ""
echo "=== SUCCESS: git zclone '$FROM' '$DEST':" \
	"created '$DESTDS' mounted to '$DESTPATH'"
if [ x"$REWRITE_ORIGIN" = xyes ] ; then
	[ "$FROM_TYPE" = LOCAL ] && \
		echo "and changed git origin to '$ORIGIN'"
else
	echo "and did not mangle git origin"
fi
if [ "$RES" != 0 ]; then
	echo "=== WARN: Some non-fatal failures were detected" \
		"and reported above" >&2
fi

return $RES
}

do_git_zfs_clone "$@"