unraid-fast-copy.sh

#!/bin/bash

# The {share_name} and {share_subdir} variables define the
# name of the unraid share you want to copy as well as an
# optional subdirectory path within the share if you want
# to keep the copy scoped to one specific share folder.
#
# The values below would copy the unraid share named "storage" in it's entirety to
# {target_path}/storage/
#    > share_name="storage"
#    > share_subdir=""
#
# These values would copy the "season 1" directory from the share named "media"
# to this location in your {target_path}: {target_path}/media/tv/show name/season 1/
#    > share_name="media"
#    > share_subdir="tv/show name/season 1"
#
# These values would copy the "tv" directory located in the "media" share
# to this location in your {target_path}: {target_path}/media/tv/
#    > share_name="media"
#    > share_subdir="tv"
#
# If {share_subdir} is defined, it must be relative to the root
# of the share, and can not have a leading or trailing slash in the path
share_name="media"
share_subdir="tv"

# The {target_path} variable defines the location that you'll want the unraid share copied to.
#
# The media folder will be coped directly into the target dir, creating this structure:
#     {target_path}/{share_name}/
#
# Or if share_subdir was defined, it'd look like this:
#     {target_path}/{share_name}/{share_subdir}
#
# In either case, the directories will be fully copied and
# preserved exactly as they appeared in the unraid share.
# Any directories that don't exist on the {target_path} will
# automatically be created.
#
# This path is passed directly to rsync so you should also be able to define it as a remote path.
# So far i've only tested it by setting it to the path of locally mounted nfs/smb network share.
target_path="/mnt/cifs-or-nfs/copy/destination/"

# Defines the location that all temp output
# files will be generated by the script
script_output_dir="$(pwd)/output"

# Defines whether or not the script should automatically delete all of the files it created when the script completes.
# The default is 'no' because many of the rsync files (input files, stdout redirected to a file, logfile) will be in
# there and can be useful to review during/after the transfer for general stats/info about the run. Changing this value
# to 'yes' will enable the cleanup functionality on script exit.
#
# Note: if you rerun this script, it will always clean out temp data from past runs when it starts.
# This cleanup option only defines whether or not it should delete temp files when the script exits.
cleanup='no'

setup_env() {
    echo "Setting up script enviornment"
    if [[ -d "$script_output_dir" ]]; then
        # If the script's output dir already exists just
        # delete any old output files from previous runs
        echo "Deleting temp files generated from past runs"
        rm -vrf "$script_output_dir"/*.*
    else
        echo "Creating directory for temp script files"
        mkdir -v "$script_output_dir"
    fi
    echo
}


cleanup_env() {
    # If the cleanup parameter is set to 'yes'
    # the script's output directory will be deleted
    if [[ $cleanup == yes ]]; then
        echo "Cleaning up script environment"
        if [[ -d "$script_output_dir" ]]; then
            rm -vrf "$script_output_dir"
        fi
        echo
    fi
}


terminate() {
    echo -e "\n\nCaught termination signal, killing child rsync processes...\n"

    # Block SIGTERM so it doesn't interfere with killing child rsync
    # processes, then kill all child processes and exit this script
    trap "" SIGTERM
    kill 0

    tput init
    cleanup_env

    exit
}


print_progress() {
    local rsync_process_count=$(ls -lq $script_output_dir/*.out | wc -l)
    printf "\n%.0s" $(seq $((($rsync_process_count * 2) + 5)))

    process_info=()
    for stdout_file in $(find "$script_output_dir" -name rsync-*.out); do
        rsync_id=$( [[ ${stdout_file} =~ .*rsync-([0-9]+).out ]] && echo "${BASH_REMATCH[1]}" )
        process_info+=" $rsync_id;$stdout_file"
    done

    while true; do
        local terminal_rows=$(tput lines)
        tput cup $((terminal_rows - $(((rsync_process_count * 2) + 4)))) 0

        for info in $process_info; do
            IFS=';' read rsync_id stdout_file <<< "${info}"
            local text=$(tail -n 1 "$stdout_file")
            printf "Disk %s rsync progress:\n" "$rsync_id"
            printf "%s\n" "$text"
        done

        sleep 0.1
    done
}


copy_unraid_share() {
    # First we'll want to make sure we trap any SIGINT or SIGTEM signals so that either of
    # those can be poperly handled by killing any child processes that are spawned below.
    trap terminate SIGINT SIGTERM

    for disk_mount_path in $(find /mnt/disk* -type d -maxdepth 0 | sort -t "/" -hk 3.5,3); do
        disk_share_data_dir="$disk_mount_path/$share_name/$share_subdir"
        disk_id=${disk_mount_path//"/mnt/disk"/}
        rsync_file_basename="$script_output_dir"/rsync-"$disk_id"
        rm -vf "$rsync_file_basename.*"

        if [[ ! -d "$disk_share_data_dir" ]]; then
            echo "No data found on disk $disk_id for $share_name/$share_subdir, skipping"
            continue
        fi

        echo "Starting rsync process for disk $disk_id.."
        echo "[rsync-$disk_id] => Copying $share_name/$share_subdir data from: /mnt/disk${disk_id}/$share_name/$share_subdir"

        {
            # Assign a real-time io priority for the
            # child rsync process being spawned below
            ionice -c 1 \
            rsync --recursive \
                  --whole-file \
                  --inplace \
                  --sparse \
                  --no-compress \
                  --max-alloc=8GiB \
                  --size-only \
                  --human-readable \
                  --info=progress2 \
                  --log-file="$rsync_file_basename.log" \
                  --log-file-format="%o=%-7'''b | total=%-7'''l [%i] => %f%L" \
               "/mnt/disk${disk_id}/$share_name/$share_subdir/" \
               "$target_path" \
            >> "$rsync_file_basename.out"
        } &
    done

    # Uncomment function call below to output progress for
    # each child process in the terminal. It's a work in
    # progress and doesn't work very well at the moment so
    # so it's disabled for now.

    # print_progress

    # Once all rsync processes have been spawned, wait for
    # all of them to complete before exiting. Ctrl + C can
    # be used to terminate the script early and terminate
    # all child rsync processes that are still active
    echo "Waiting for rsync child processes to complete..."
    wait

}

setup_env
copy_unraid_share
cleanup_env