diff --git a/.mk/docker.mk b/.mk/docker.mk index fe9d410c..206d11d1 100644 --- a/.mk/docker.mk +++ b/.mk/docker.mk @@ -1,10 +1,16 @@ .PHONY: docker-setup -## Setup environment variables required for docker-compose +## Setup environment variables required for docker-compose if needed docker-setup: @scripts/docker-setup.sh +.PHONY: docker-setup-update + +## Update environment variables required for docker-compose +docker-setup-update: + @scripts/docker-setup.sh --force-update + .PHONY: docker-run @@ -19,11 +25,27 @@ docker-run: docker-setup docker-stop: sudo docker-compose stop +.PHONY: docker-build + ## Build docker images for docker-compose application docker-build: sudo docker-compose build +.PHONY: docker-rebuild + ## Rebuild docker images docker-rebuild: sudo docker-compose rm -s -f sudo docker-compose build + +.PHONY: docker-update + +## Update docker images (rebuild local or pull latest from repository depending on configuration). +docker-update: + @scripts/docker-update.sh + +.PHONY: docker-purge + +## Shut-down docker-compose application and remove all its images and volumes. +docker-purge: + sudo docker-compose down --rmi all -v --remove-orphans --timeout 0 diff --git a/Makefile b/Makefile index 6e2f4ac3..db117b00 100644 --- a/Makefile +++ b/Makefile @@ -18,12 +18,28 @@ stop: docker-stop .PHONY: setup -## Setup docker-compose application (generate .env file) +## Setup docker-compose application (generate .env file in needed) setup: docker-setup +.PHONY: update-setup + +## Update docker-compose application (regenerate .env file) +setup-update: docker-setup-update + +.PHONY: rebuild + ## Rebuild docker-compose images rebuild: docker-rebuild +.PHONY: update + +## Update images of the docker-compose application +update: docker-update + +.PHONY: purge + +## Remove docker-compose application and all its images and volumes. +purge: docker-purge # Define default goal .DEFAULT_GOAL := help diff --git a/docker-compose/prebuilt.cpu.yml b/docker-compose/prebuilt.cpu.yml index 9219e194..74d35e54 100644 --- a/docker-compose/prebuilt.cpu.yml +++ b/docker-compose/prebuilt.cpu.yml @@ -5,5 +5,5 @@ version: '2.3' services: dedup-app: - image: johnhbenetech/videodeduplication:cpu + image: "johnhbenetech/videodeduplication:cpu${BENETECH_MODE}" runtime: runc diff --git a/docker-compose/prebuilt.yml b/docker-compose/prebuilt.yml index 9264b8dd..9ca3abcb 100644 --- a/docker-compose/prebuilt.yml +++ b/docker-compose/prebuilt.yml @@ -5,4 +5,4 @@ version: '2.3' services: dedup-app: - image: johnhbenetech/videodeduplication:gpu + image: "johnhbenetech/videodeduplication:gpu${BENETECH_MODE}" diff --git a/scripts/docker-run.sh b/scripts/docker-run.sh index e0b56560..15f819f4 100755 --- a/scripts/docker-run.sh +++ b/scripts/docker-run.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +# This script runs the docker-compose application +# according to the configuration saved in .env file. + if ! [ -f ".env" ]; then echo -e "\e[31mERROR\e[0m Environment file not found: $(pwd)/.env" echo -e "\e[31mERROR\e[0m Please run script/docker-setup.sh first." @@ -15,11 +18,15 @@ if ! [ -d "$BENETECH_DATA_LOCATION" ] || [ -z "$BENETECH_RUNTIME" ] || [ -z "$BE fi if [ "$BENETECH_RUNTIME" = "GPU" ] && [ "$BENETECH_PREBUILT" = "NO" ]; then + set -x sudo docker-compose up -d elif [ "$BENETECH_RUNTIME" = "CPU" ] && [ "$BENETECH_PREBUILT" = "NO" ]; then + set -x sudo docker-compose -f docker-compose.yml -f docker-compose/build.cpu.yml up -d elif [ "$BENETECH_RUNTIME" = "GPU" ] && [ "$BENETECH_PREBUILT" = "YES" ]; then + set -x sudo docker-compose -f docker-compose.yml -f docker-compose/prebuilt.yml up -d elif [ "$BENETECH_RUNTIME" = "CPU" ] && [ "$BENETECH_PREBUILT" = "YES" ]; then + set -x sudo docker-compose -f docker-compose.yml -f docker-compose/prebuilt.cpu.yml up -d fi diff --git a/scripts/docker-setup.sh b/scripts/docker-setup.sh index e7880bb7..f8e7c17f 100755 --- a/scripts/docker-setup.sh +++ b/scripts/docker-setup.sh @@ -1,46 +1,74 @@ #!/usr/bin/env bash +read -r -d '' HELP << ENDOFHELP +usage: ./docker-setup.sh [--help] [-f | --force-update] + +Generate .env file used by docker-compose tool +and some docker-related scripts under the ./script +directory. + +See also https://docs.docker.com/compose/env-file/ +ENDOFHELP + +FORCE_UPDATE=NO; + +# Read arguments +while (( $# > 0 )); do + case $1 in + -f|--force-update) + FORCE_UPDATE=YES; + shift; + ;; + --help) + echo "$HELP"; + exit 0; + ;; + *) + echo "Unrecognized argument: $1"; + echo "$HELP" + exit 1; + esac +done + + +LIBS="$(realpath "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib" )" +source "$LIBS/ask.choose.sh" +source "$LIBS/ask.confirm.sh" +source "$LIBS/ask.path.sh" + if [ -f ".env" ]; then source .env fi -# Read source data locaion -while ! [ -d "$BENETECH_DATA_LOCATION" ]; do +# Read source data location +if [ "$FORCE_UPDATE" = "YES" ] || ! [ -d "$BENETECH_DATA_LOCATION" ]; then DIRTY=yes - echo -n -e "\e[36mPlease specify the root folder with your video files (use Tab for auto-complete): \e[0m" - read -e -r BENETECH_DATA_LOCATION - if ! [ -d "$BENETECH_DATA_LOCATION" ]; then - echo -e "\e[31mERROR\e[0m No such directory: $BENETECH_DATA_LOCATION" - fi -done + read-dir-path BENETECH_DATA_LOCATION "Please specify the root folder with your video files (use Tab for auto-complete)" + echo +fi # Choose data analysis runtime -while [ -z "$BENETECH_RUNTIME" ]; do +if [ "$FORCE_UPDATE" = "YES" ] || [ -z "$BENETECH_RUNTIME" ]; then DIRTY=yes - echo -n -e "\e[36mWould you like to use GPU for data analysis? [Y/n]: \e[0m" - read -r RUNTIME_ANSWER - if [ -z "$RUNTIME_ANSWER" ] || [[ "$RUNTIME_ANSWER" =~ [Yy] ]]; then - export BENETECH_RUNTIME=GPU - elif [[ "$RUNTIME_ANSWER" =~ [Nn] ]]; then - export BENETECH_RUNTIME=CPU - else - echo -e "\e[31mERROR\e[0m Cannot recognize answer. Please answer 'y' or 'n'" - fi -done + tput setaf 6; echo "Would you like to use GPU for data processing?"; tput sgr0; + choose BENETECH_RUNTIME GPU="Use GPU for data processing." CPU="Use CPU for data processing." + echo +fi # Decide whether to use prebuilt images -while [ -z "$BENETECH_PREBUILT" ]; do +if [ "$FORCE_UPDATE" = "YES" ] || [ -z "$BENETECH_PREBUILT" ]; then DIRTY=yes - echo -n -e "\e[36mWould you like to use pre-built images? [y/N]: \e[0m" - read -r PREBUILT_ANSWER - if [ -z "$PREBUILT_ANSWER" ] || [[ "$PREBUILT_ANSWER" =~ [Nn] ]]; then - export BENETECH_PREBUILT=NO - elif [[ "$PREBUILT_ANSWER" =~ [Yy] ]]; then - export BENETECH_PREBUILT=YES - else - echo -e "\e[31mERROR\e[0m Cannot recognize answer. Please answer 'y' or 'n'" + tput setaf 6; echo "Would you like to use pre-built Docker images?"; tput sgr0; + choose BENETECH_PREBUILT YES="Pull pre-built images from Docker Hub." NO="Build images locally." + echo + + # Ask if user would like to use dev or prod images + if [ "$BENETECH_PREBUILT" = "YES" ]; then + tput setaf 6; echo "Would you like to use production Docker images?"; tput sgr0; + choose BENETECH_MODE ''="Use production images." '-dev'="Use dev-images." + echo fi -done +fi # Write data to the .env file if [ -n "$DIRTY" ]; then @@ -48,7 +76,8 @@ if [ -n "$DIRTY" ]; then echo "BENETECH_DATA_LOCATION=$BENETECH_DATA_LOCATION" echo "BENETECH_RUNTIME=$BENETECH_RUNTIME" echo "BENETECH_PREBUILT=$BENETECH_PREBUILT" + echo "BENETECH_MODE=$BENETECH_MODE" } > .env - - echo -e "\e[1mOK\e[0m Configuration is written to the $(pwd)/.env file" + tput setaf 2; echo -n "OK"; tput sgr0; + echo " Configuration is written to the $(pwd)/.env"; fi diff --git a/scripts/docker-update.sh b/scripts/docker-update.sh new file mode 100755 index 00000000..a1096a2e --- /dev/null +++ b/scripts/docker-update.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# This script updates the docker-images used by the application. + +if ! [ -f ".env" ]; then + echo -e "\e[31mERROR\e[0m Environment file not found: $(pwd)/.env" + echo -e "\e[31mERROR\e[0m Please run script/docker-setup.sh first." + exit 1 +fi + +source .env + +if ! [ -d "$BENETECH_DATA_LOCATION" ] || [ -z "$BENETECH_RUNTIME" ] || [ -z "$BENETECH_PREBUILT" ]; then + echo -e "\e[31mERROR\e[0m Environment file is incomplete." + echo -e "\e[31mERROR\e[0m Please run script/docker-setup.sh first." + exit 1 +fi + + +if [ "$BENETECH_RUNTIME" = "GPU" ] && [ "$BENETECH_PREBUILT" = "NO" ]; then + set -x + sudo docker-compose rm -s -f + sudo docker-compose build +elif [ "$BENETECH_RUNTIME" = "CPU" ] && [ "$BENETECH_PREBUILT" = "NO" ]; then + set -x + sudo docker-compose rm -s -f + sudo docker-compose -f docker-compose.yml -f docker-compose/build.cpu.yml build +elif [ "$BENETECH_RUNTIME" = "GPU" ] && [ "$BENETECH_PREBUILT" = "YES" ]; then + set -x + sudo docker-compose rm -s -f + sudo docker-compose -f docker-compose.yml -f docker-compose/prebuilt.yml pull +elif [ "$BENETECH_RUNTIME" = "CPU" ] && [ "$BENETECH_PREBUILT" = "YES" ]; then + set -x + sudo docker-compose rm -s -f + sudo docker-compose -f docker-compose.yml -f docker-compose/prebuilt.cpu.yml pull +fi diff --git a/scripts/lib/ask.choose.sh b/scripts/lib/ask.choose.sh new file mode 100644 index 00000000..51b703e1 --- /dev/null +++ b/scripts/lib/ask.choose.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# This module provides a 'choose' function which allows +# users to choose from a set of options using arrow keys. +# You should source this module to make it available in +# your script. +# Based on: https://www.bughunter2k.de/blog/cursor-controlled-selectmenu-in-bash + + +# Get choice value. +# Args: +# $1: Choice in the form of value="Display text." +function choose.option-value() { + local entry=$1; + echo "$entry" | grep -Po "^[^=]+" +} + +# Get choice display text. +# Args: +# $1: Choice in the form of value="Display text." +function choose.option-text() { + local entry=$1; + echo "$entry" | grep -Po "(?<==).*$" +} + +# Print choices. +# Args: +# $1 (number): Index of currently selected choice. +# *: Any number of choices in the form of value="Display text". +function choose.print-options() { + local cur=$1; shift 1; + local entries=( "$@" ) + for entry in "${entries[@]}"; do + if [[ ${entries[$cur]} == "$entry" ]]; then + # Print green. See http://linuxcommand.org/lc3_adv_tput.php, "Text Effects" + tput setaf 2; echo ">$(choose.option-text "$entry")"; tput sgr0; + else + echo " $(choose.option-text "$entry")"; + fi + done +} + +# Erase printed choices. +# Args: +# $1 (number): Number of choices. +function choose.erase-options() { + local count=$1; + for _ in $(seq "$count"); do + # Move cursor one line up + # See http://linuxcommand.org/lc3_adv_tput.php, "Controlling The Cursor" + tput cuu1; + done + # Clear from the cursor to the end of the screen + # See http://linuxcommand.org/lc3_adv_tput.php, "Clearing The Screen" + tput ed; +} + +# Ask user to select from the list of options using arrow keys. +# Args: +# $1 (variable reference): Variable to write result to. +# *: Any number of choices in the form value="Display text" +# Example: +# local MODE; +# choose MODE prod="Use production mode" dev="Use development mode" +function choose() { + # Use function name as a var-name prefix + # to prevent circular name reference + # See https://stackoverflow.com/a/33777659 + # See http://mywiki.wooledge.org/BashFAQ/048#line-120 + local -n _choose_result=$1; shift 1 + local entries=( "$@" ) + local cur=0; + + choose.print-options "$cur" "${entries[@]}" + + while read -sN1 key; do + # Catch multi-char special key sequences + # See https://stackoverflow.com/a/11759139 + read -sN1 -t 0.0001 k1 + read -sN1 -t 0.0001 k2 + read -sN1 -t 0.0001 k3 + key+=${k1}${k2}${k3} + + # Enter or Space was pressed + if [ -z "$key" ]; then + # shellcheck disable=SC2034 + # Variable is passed by reference + _choose_result="$(choose.option-value "${entries[$cur]}")"; + return 0; + fi + + case "$key" in + # Arrow up or left: previous item: + $'\e[A'|$'\e0A'|$'\e[D'|$'\e0D') + ((cur > 0)) && ((cur--));; + + # Arrow down or right: next item: + $'\e[B'|$'\e0B'|$'\e[C'|$'\e0C') + ((cur < ${#entries[@]}-1)) && ((cur++));; + + # Home: first item + $'\e[1~'|$'\e0H'|$'\e[H') + cur=0;; + + # End: last item + $'\e[4~'|$'\e0F'|$'\e[F') + ((cur=${#entries[@]}-1));; + + # 'q' or carriage return: Quit + q|$'\e') + return 1;; + esac + + choose.erase-options "${#entries[@]}" + choose.print-options "$cur" "${entries[@]}" + done +} \ No newline at end of file diff --git a/scripts/lib/ask.confirm.sh b/scripts/lib/ask.confirm.sh new file mode 100644 index 00000000..52f9e37f --- /dev/null +++ b/scripts/lib/ask.confirm.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# This module provides a 'confirm' function which allows +# users to confirm or reject the statement. You should +# source this module to make it available in your script. + +# Ask user for confirmation and save result to the variable. +# Args: +# $1 (variable reference): Variable name to save result to (YES or NO). +# $2 (string): Question to ask. +# $3 (optional YES|NO): Default answer (used on empty user input). +function confirm() { + # Use function name as a var-name prefix + # to prevent circular name reference + # See https://stackoverflow.com/a/33777659 + # See http://mywiki.wooledge.org/BashFAQ/048#line-120 + local -n _confirm_result=$1; + local confirm_text=$2; + local default=${3:-YES} + local answer; + + local answer_pattern; + if [ "$default" = "YES" ]; then + answer_pattern="[Y/n]"; + else + answer_pattern="[y/N]"; + fi + + while true; do + tput setaf 6; echo -n "$confirm_text $answer_pattern: "; tput sgr0; + read -r answer; + # shellcheck disable=SC2015 + # Disable false-positive on condition expression. + if [ -z "$answer" ] && [ "$default" = "YES" ] || [[ "$answer" =~ [Yy] ]]; then + # shellcheck disable=SC2034 + # Variable is passed by reference + _confirm_result=YES; + return 0; + elif [ -z "$answer" ] && [ "$default" = "NO" ] || [[ "$answer" =~ [Nn] ]]; then + # shellcheck disable=SC2034 + # Variable is passed by reference + _confirm_result=NO; + return 0; + else + tput setaf 1; echo -n "ERROR"; tput sgr0; echo ": Cannot recognize the answer '$answer'. Please answer 'y' or 'n'" + fi + done +} diff --git a/scripts/lib/ask.path.sh b/scripts/lib/ask.path.sh new file mode 100644 index 00000000..915f6383 --- /dev/null +++ b/scripts/lib/ask.path.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# This module provides a 'read-file-path' and 'read-dir-path' functions +# to ask users for existing file and directory path correspondingly. +# You should source this module to make it available in your script. + +# Ask user to provide an existing directory path. +# Args: +# $1 (variable reference): Variable to write result to. +# $2 (string): Question to ask. +function read-dir-path() { + # Use function name as a var-name prefix + # to prevent circular name reference + # See https://stackoverflow.com/a/33777659 + # See http://mywiki.wooledge.org/BashFAQ/048#line-120 + local -n _read_dir_path_result=$1; + local question=$2; + local answer; + + while true; do + tput setaf 6; echo -n "$question: "; tput sgr0; + read -e -r answer; + if [ -d "$answer" ]; then + # shellcheck disable=SC2034 + # Variable is passed by reference + _read_dir_path_result=$answer; + return 0; + else + tput setaf 1; echo -n "ERROR"; tput sgr0; echo ": No such directory: '$answer'" + fi + done +} + +# Ask user to provide an existing file path. +# Args: +# $1 (variable reference): Variable to write result to. +# $2 (string): Question to ask. +function read-file-path() { + # Use function name as a var-name prefix + # to prevent circular name reference + # See https://stackoverflow.com/a/33777659 + # See http://mywiki.wooledge.org/BashFAQ/048#line-120 + local -n _read_file_path_result=$1; + local question=$2; + local answer; + + while true; do + tput setaf 6; echo -n "$question: "; tput sgr0; + read -e -r answer; + if [ -f "$answer" ]; then + # shellcheck disable=SC2034 + # Variable is passed by reference + _read_file_path_result=$answer; + return 0; + else + tput setaf 1; echo -n "ERROR"; tput sgr0; echo ": No such file: '$answer'" + fi + done +} diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 12b53be7..0a57d1a3 100644 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + echo "Running unit tests" python -m pytest tests/general_tests.py