run

#!/usr/bin/env bash
#

function usage() {
	echo "USAGE:"
	echo "./run logmode CONFIG_FILE=< config.yaml >"
	echo "./run backtesting CONFIG_FILE=< config.yaml >"
	echo "./run testnet CONFIG_FILE=< config.yaml >"
	echo "./run live CONFIG_FILE=< config.yaml >"
	echo "./run compress-logs"
	echo "./run lastfewdays DAYS=3 PAIR=USDT"
	echo "./run download-price-logs FROM=20210101 TO=20211231"
	echo "./run prove-backtesting CONFIG_FILE=myconfig.yaml"
	echo "./run config-endpoint-service BIND=0.0.0.0 CONFIG_FILE=myconfig.yaml"
	echo "./run klines-caching-service BIND=0.0.0.0"
	echo "./run price_log_service BIND=0.0.0.0"
	echo "./run download_price_logs FROM=20220101 TO=20220131 UNIT=1m"
}

function free_port () { # looks for a free TCP port
    LPORT=32768;
    UPORT=60999;
    while true; do
        MPORT=$[$LPORT + ($RANDOM % $UPORT)];
        (echo "" >/dev/tcp/127.0.0.1/${MPORT}) >/dev/null 2>&1
        if [ $? -ne 0 ]; then
            echo $MPORT;
            return 0;
        fi
    done
}

function set_service_ports () { # locks a port to use by each service
	for service in $*
	do
		if [ ! -f $STATE_DIR/.${service}.port ] ; then
			free_port > $STATE_DIR/.${service}.port
		fi
	done
}

function checks() { # runs docker version checks
	if [ "`docker --version | cut -d " " -f3 | tr -d 'v'| cut -c1`" -lt 2 ]; \
		then echo "docker version is too old"; exit 1; fi
}

function build() { # builds docker images
	docker pull ubuntu:focal
	docker buildx build  --build-arg BUILDKIT_INLINE_CACHE=1 -t tempbuild \
		--cache-from=${IMAGE}:latest \
		--cache-from=${IMAGE}:next_release \
		--cache-from=${IMAGE}:${TAG} .
	docker tag tempbuild ${IMAGE}:${TAG}
}

function down() { # power down all containers
	docker ps | grep "${CONTAINER_SUFFIX}" | awk '{print $1}' | xargs -i docker kill {}
}

function latest() { # pulls :latest image tag
	docker pull ${IMAGE}:${TAG} >/dev/null
}

# this should become its own docker image
function download_price_logs() { # downloads klines logs fro binance
	if [ -z "$FROM" ]; then
		echo "FROM env variable not set"
		exit 1
	fi
	if [ -z "$TO" ]; then
		echo "TO env variable not set"
		exit 1
	fi

	if [ -z "$UNIT" ]; then
		export UNIT="1m"
	fi

	docker run --rm \
		${USE_TTY} \
		${DOCKER_RUN_AS} \
		${DOCKER_NAME} \
		${DOCKER_MOUNTS} \
		${DOCKER_NETWORK} \
		${RUN_IN_BACKGROUND} \
		${IMAGE}:${TAG} \
    /cryptobot/.venv/bin/python -u  /cryptobot/utils/pull_klines.py \
		-s ${FROM} -e ${TO} -u ${UNIT}
}

function docker_network() { # creates a docker network
	docker network ls |grep ${CONTAINER_SUFFIX} >/dev/null 2>&1 || docker network create ${CONTAINER_SUFFIX}
}

function logmode() { # runs in logmode
	docker run --rm \
		${USE_TTY} \
		${DOCKER_RUN_AS} \
		${DOCKER_NAME} \
		${DOCKER_MOUNTS} \
		${DOCKER_NETWORK} \
		${RUN_IN_BACKGROUND} \
		${IMAGE}:${TAG} \
		/cryptobot/.venv/bin/python -u app.py \
		-s /cryptobot/secrets/fake.yaml \
		-c /cryptobot/configs/${CONFIG_FILE} \
		-m ${MODE} > ${LOG_DIR}/${MODE}.${CONFIG_FILE}.txt 2>&1
}

function testnet() { # runs in testnet mode
	if [ -z "$PORT" ]; then
		export PORT=$( cat ${STATE_DIR}/.testnet.port)
	fi

	docker run --rm \
		${USE_TTY} \
		${DOCKER_RUN_AS} \
		${DOCKER_NAME} \
		${DOCKER_MOUNTS} \
		${DOCKER_NETWORK} \
		${RUN_IN_BACKGROUND} \
    -p ${BIND}:${PORT}:5555 \
		${IMAGE}:${TAG} \
		/cryptobot/.venv/bin/python -u app.py \
		-s /cryptobot/secrets/${SECRETS_FILE:-binance.testnet.yaml} \
		-c /cryptobot/configs/${CONFIG_FILE} \
		-m ${MODE} > ${LOG_DIR}/${MODE}.${CONFIG_FILE}.txt 2>&1
}

function live() { # runs in live mode
	if [ -z "$PORT" ]; then
		export PORT=$( cat ${STATE_DIR}/.${MODE}.port )
	fi

	docker run --rm \
		${USE_TTY} \
		${DOCKER_RUN_AS} \
		${DOCKER_NAME} \
		${DOCKER_MOUNTS} \
		${DOCKER_NETWORK} \
		${RUN_IN_BACKGROUND} \
    -p ${BIND}:${PORT}:5555 \
		${IMAGE}:${TAG} \
		/cryptobot/.venv/bin/python -u app.py \
		-s /cryptobot/secrets/${SECRETS_FILE:-binance.prod.yaml} \
		-c /cryptobot/configs/${CONFIG_FILE} \
		-m ${MODE} >> ${LOG_DIR}/${MODE}.${CONFIG_FILE}.txt 2>&1
}

function backtesting() { # runs in backtesting mode
	docker run --rm \
		${USE_TTY} \
		${DOCKER_RUN_AS} \
		${DOCKER_NAME} \
		${DOCKER_MOUNTS} \
		${DOCKER_NETWORK} \
		${RUN_IN_BACKGROUND} \
		${IMAGE}:${TAG} \
		/usr/bin/eatmydata /cryptobot/.venv/bin/python -u app.py \
		-s /cryptobot/secrets/${SECRETS_FILE:-fake.yaml} \
		-c /cryptobot/configs/${CONFIG_FILE} \
		-m backtesting > ${RESULTS_DIR}/backtesting.${CONFIG_FILE}.txt 2>&1
}

function prove_backtesting() { # runs the prove backtesting
	if [ -z "$CONFIG_FILE" ]; then
		echo "CONFIG_FILE env variable not set"
		exit 1
	fi

	RESULTS_LOG="${RESULTS_DIR}/prove-backtesting"
	RESULTS_LOG="${RESULTS_LOG}.${CONFIG_FILE}.txt"
	docker run --rm \
		${USE_TTY} \
		${DOCKER_RUN_AS} \
		${DOCKER_NAME} \
		${DOCKER_MOUNTS} \
		${DOCKER_NETWORK} \
		${RUN_IN_BACKGROUND} \
		-e CONFIG_FILE=${CONFIG_FILE} \
		${IMAGE}:${TAG} \
		/usr/bin/eatmydata /cryptobot/utils/prove-backtesting.sh \
		> ${RESULTS_LOG}
}

function config_endpoint_service() { # runs the config endpoint service
	if [ -z "$PORT" ]; then
		export PORT=$( cat ${STATE_DIR}/.${MODE}.port )
	fi

	if [ -z "$CONFIG_FILE" ]; then
		echo "CONFIG_FILE env variable not set"
		exit 1
	fi

	docker run --rm \
		${USE_TTY} \
		${DOCKER_RUN_AS} \
		${DOCKER_NAME} \
		${DOCKER_MOUNTS} \
		${DOCKER_NETWORK} \
		${RUN_IN_BACKGROUND} \
		-e CONFIG_FILE=${CONFIG_FILE} \
    -p ${BIND}:${PORT}:5883 \
		${IMAGE}:${TAG} \
    /cryptobot/utils/config-endpoint-service.sh
}

function klines_caching_service() { # runs the klines caching service
	if [ -z "$PORT" ]; then
		export PORT=$( cat ${STATE_DIR}/.${MODE}.port )
	fi

	if [ -n "${RUN_IN_BACKGROUND}" ]; then
		docker ps | grep "klines_caching_service-${CONTAINER_SUFFIX}" \
			|awk '{ print $1 }' | xargs -i docker kill {} >/dev/null 2>&1
	fi

	docker run --rm \
		${USE_TTY} \
		${DOCKER_RUN_AS} \
		${DOCKER_NAME} \
		${DOCKER_MOUNTS} \
		${DOCKER_NETWORK} \
		--network-alias klines \
		${RUN_IN_BACKGROUND} \
    -p ${BIND}:${PORT}:8999 \
		${IMAGE}:${TAG} \
    /cryptobot/.venv/bin/gunicorn --preload \
			--workers=${N_CPUS} \
			--worker-class=gthread \
			--threads=8 \
			--worker-tmp-dir /dev/shm \
			--bind 0.0.0.0:8999  klines_caching_service:app
}

function price_log_service() { # runs the klines caching service
	if [ -z "$PORT" ]; then
		export PORT=$( cat ${STATE_DIR}/.${MODE}.port )
	fi

	if [ -n "${RUN_IN_BACKGROUND}" ]; then
		docker ps | grep "price-log-service-${CONTAINER_SUFFIX}" \
			|awk '{ print $1 }' | xargs -i docker kill {} >/dev/null 2>&1
	fi

	docker run --rm \
		${USE_TTY} \
		${DOCKER_RUN_AS} \
		${DOCKER_NAME} \
		${DOCKER_MOUNTS} \
		${DOCKER_NETWORK} \
		--network-alias price-log-service \
		${RUN_IN_BACKGROUND} \
    -p ${BIND}:${PORT}:8998 \
		${IMAGE}:${TAG} \
    /cryptobot/.venv/bin/gunicorn --preload \
			--workers=${N_CPUS} \
			--worker-class=gthread \
			--threads=8 \
			--worker-tmp-dir /dev/shm \
			--bind 0.0.0.0:8998  price_log_service:app
}

function setup() { # local setup for development
	which pyenv >/dev/null 2>&1 ||  curl https://pyenv.run | bash
	export PATH=~/.pyenv/bin:$PATH

	if [ ! -e .venv ]; then
		pyenv install -s
		pyenv exec python -m venv .venv
	fi

	source .venv/bin/activate
	pip --disable-pip-version-check install wheel
	pip --disable-pip-version-check install -r requirements.txt
	pip --disable-pip-version-check install -r requirements-dev.txt
	deactivate
}

function tests() { # CI and pre-commit tests
	set -e
	set -o pipefail

	IMAGE=local TAG=tests build
	docker build -f Dockerfile.tests .
}

function github_actions_ci_pr_docker_tests() {
	set -ex
  ./run down
  ./run build TAG=pr
  ./run klines-caching-service RUN_IN_BACKGROUND=yes TAG=pr
  ./run price-log-service RUN_IN_BACKGROUND=yes TAG=pr

	sleep 5

	# don't worry if we can't push, as when running locally this will fail anyway
	docker push ghcr.io/azulinho/cryptobot:pr || true
	# TODO: review where these are being consumed in the tests
	cp tests/fake.yaml secrets/binance.prod.yaml
	cp tests/fake.yaml secrets/fake.yaml
	cp tests/price.log.gz log/tests.price.log.gz

	mkdir -p log/ETHUSDT
	mkdir -p log/BTCUSDT
	for ta in 01 02 03 04 05 06 07 08 09
	do
		cat tests/price.log.gz | grep 2021-12-${ta}  |gzip -1 > log/202112${ta}.log.gz
		cat tests/price.log.gz | grep ETHUSDT | grep 2021-12-${ta}  |gzip -1 > log/ETHUSDT/202112${ta}.log.gz
		cat tests/price.log.gz | grep BTCUSDT | grep 2021-12-${ta}  |gzip -1 > log/BTCUSDT/202112${ta}.log.gz
	done
	cp tests/index.json.gz log/
	cp tests/index_v2.json.gz log/

	export PRICE_LOG_PORT=$( cat ${STATE_DIR}/.price_log_service.port)
	curl --output /dev/null http://${DOCKER_IP}:${PRICE_LOG_PORT}/index.json.gz
	curl --output /dev/null http://${DOCKER_IP}:${PRICE_LOG_PORT}/index_v2.json.gz

	echo BuyMoonSellRecoveryStrategy.yaml
	cp tests/BuyMoonSellRecoveryStrategy.yaml configs/
	./run backtesting CONFIG_FILE=BuyMoonSellRecoveryStrategy.yaml TAG=pr
	grep 'wins:366 losses:98 stales:104 holds:1' results/backtesting.BuyMoonSellRecoveryStrategy.yaml.txt

	echo BuyOnGrowthTrendAfterDropStrategy.yaml
	cp tests/BuyOnGrowthTrendAfterDropStrategy.yaml configs/
	./run backtesting CONFIG_FILE=BuyOnGrowthTrendAfterDropStrategy.yaml TAG=pr
	grep 'wins:23 losses:3 stales:87 holds:2' results/backtesting.BuyOnGrowthTrendAfterDropStrategy.yaml.txt

  echo BuyDropSellRecoveryStrategy.yaml
	cp tests/BuyDropSellRecoveryStrategy.yaml configs/
	./run backtesting CONFIG_FILE=BuyDropSellRecoveryStrategy.yaml TAG=pr
	grep 'wins:4 losses:9 stales:1 holds:0' results/backtesting.BuyDropSellRecoveryStrategy.yaml.txt

	echo BuyDropSellRecoveryStrategyWhenBTCisUp.yaml
	cp tests/BuyDropSellRecoveryStrategyWhenBTCisUp.yaml configs/
	./run backtesting CONFIG_FILE=BuyDropSellRecoveryStrategyWhenBTCisUp.yaml TAG=pr
	grep 'wins:209 losses:2 stales:674 holds:0' results/backtesting.BuyDropSellRecoveryStrategyWhenBTCisUp.yaml.txt

	echo BuyDropSellRecoveryStrategyWhenBTCisDown.yaml
	cp tests/BuyDropSellRecoveryStrategyWhenBTCisDown.yaml configs/
	./run backtesting CONFIG_FILE=BuyDropSellRecoveryStrategyWhenBTCisDown.yaml TAG=pr
	grep 'wins:10 losses:0 stales:125 holds:0' results/backtesting.BuyDropSellRecoveryStrategyWhenBTCisDown.yaml.txt

	echo BuyOnRecoveryAfterDropDuringGrowthTrendStrategy.yaml
	cp tests/BuyOnRecoveryAfterDropDuringGrowthTrendStrategy.yaml configs/
	./run backtesting CONFIG_FILE=BuyOnRecoveryAfterDropDuringGrowthTrendStrategy.yaml TAG=pr
	grep 'wins:131 losses:0 stales:411 holds:0' results/backtesting.BuyOnRecoveryAfterDropDuringGrowthTrendStrategy.yaml.txt

	echo BuyOnRecoveryAfterDropFromAverageStrategy.yaml
	cp tests/BuyOnRecoveryAfterDropFromAverageStrategy.yaml configs/
	./run backtesting CONFIG_FILE=BuyOnRecoveryAfterDropFromAverageStrategy.yaml TAG=pr
	grep 'wins:195 losses:4 stales:621 holds:0' results/backtesting.BuyOnRecoveryAfterDropFromAverageStrategy.yaml.txt

	echo prove-backtesting
	cp tests/prove-backtesting.yaml configs/
	./run prove-backtesting \
		TAG=pr CONFIG_FILE=prove-backtesting.yaml

	wc -l results/prove-backtesting.prove-backtesting.yaml.txt \
		| grep '49'

	for ta in 01 02 03 04 05 06 07 08 09
	do
		rm -f log/202112${ta}.log.gz
		rm -f log/ETHUSDT/202112${ta}.log.gz
		rm -f log/BTCUSDT/202112${ta}.log.gz
	done
	rm -f log/index.json.gz

  ./run down
}

function compress_logs() { # compresses the latest price logs
	find ${LOG_DIR}/ -name "202*.log" | grep -v "$(date  '+%Y%m%d')" | xargs -i gzip -3 {}
}

function last_few_days() { # generates a lastfewdays.log.gz from last n days
	if [ -z "$PAIR" ]; then
		echo "PAIR env variable not set"
		exit 1
	fi
	if [ -z "$DAYS" ]; then
		echo "DAYS env variable not set"
		exit 1
	fi

	rm -f lastfewdays.log.gz; for ta in `find log/ -name '202*.gz' |sort -n \
		| tail -${DAYS}` ; do zcat $ta | grep -a "${PAIR}" \
		| grep -vEa 'DOWN${PAIR}|UP${PAIR}|BULL${PAIR}|BEAR${PAIR}' \
		| gzip -3 >> lastfewdays.log.gz; done
}

function main() { # main innit?
	LOCAL_PATH="$(pwd)"
	export PATH="${LOCAL_PATH}/.venv/bin:$PATH"

	if [ $# -eq 0 ]; then usage; exit 1; fi

	export USE_TTY=""
	test -t 1 && USE_TTY="-it"

	# allow for the same syntax used in makefiles using '-' instead of '_'
	export MODE=$( echo $1 | tr -s '-' '_' )
	shift 1

	# convert CLI into env vars
	for ARG in $*
	do
		export $ARG
	done

	# certain modes require certain env vars set
	if [ "`echo " $MODE " | grep -cE ' logmode | live | testnet '`" -eq 1 ]; then
		if [ -z "$CONFIG_FILE" ]; then
			echo "CONFIG_FILE env variable not set"
			exit 1
		fi
	fi

	if [ -z "$RUN_IN_BACKGROUND" ]; then
		export RUN_IN_BACKGROUND=""
	else
		export RUN_IN_BACKGROUND="-d"
	fi

	if [ -z "$IMAGE" ]; then
		export IMAGE="ghcr.io/azulinho/cryptobot"
	fi

	if [ -z "$TAG" ]; then
		export TAG="latest"
	fi

	if [ -z "$SMP_MULTIPLIER" ]; then
		export SMP_MULTIPLIER=1
	fi

	if [ -z "$BIND_ADDRESS" ]; then
		export DOCKER_IP=$(ip a show docker0 |grep 'inet '| awk '{ print $2 }' | cut -f1 -d '/')
		export BIND_ADDRESS=$DOCKER_IP
	fi

	if [ -z "$LOG_DIR" ]; then
		export LOG_DIR="$(pwd)/log"
	fi

	if [ -z "$CONFIG_DIR" ]; then
		export CONFIG_DIR="$(pwd)/configs"
	fi

	if [ -z "$SECRETS_DIR" ]; then
		export SECRETS_DIR="$(pwd)/secrets"
	fi

	if [ -z "$STATE_DIR" ]; then
		export STATE_DIR="$(pwd)/state"
	fi

	if [ -z "$RESULTS_DIR" ]; then
		export RESULTS_DIR="$(pwd)/results"
	fi

	if [ -z "$CONTROL_DIR" ]; then
		export CONTROL_DIR="$(pwd)/control"
	fi

	if [ -z "$CACHE_DIR" ]; then
		export CACHE_DIR="$(pwd)/cache"
	fi

	# TODO: do I need this?
	if [ -z "$TESTS_DIR" ]; then
		export TESTS_DIR="$(pwd)/tests"
	fi

	if [ -z "$SORTBY" ]; then
		export SORTBY="number_of_clean_wins"
	fi

	if [ -z "$FILTER" ]; then
		export FILTER=''
	fi

	if [ -z "$TMP_DIR" ]; then
		export TMP_DIR="$(pwd)/tmp"
	fi


	export N_CPUS=`grep processor /proc/cpuinfo|wc -l`

	export DOCKER_MOUNTS=""
	export DOCKER_MOUNTS="${DOCKER_MOUNTS} -v /etc/timezone:/etc/timezone:ro"
	export DOCKER_MOUNTS="${DOCKER_MOUNTS} -v /etc/localtime:/etc/localtime:ro"
	export DOCKER_MOUNTS="${DOCKER_MOUNTS} -v $PWD/strategies:/cryptobot/strategies:rw "
	export DOCKER_MOUNTS="${DOCKER_MOUNTS} -v $CONFIG_DIR:/cryptobot/configs:rw "
	export DOCKER_MOUNTS="${DOCKER_MOUNTS} -v $SECRETS_DIR:/cryptobot/secrets:ro"
	export DOCKER_MOUNTS="${DOCKER_MOUNTS} -v $LOG_DIR:/cryptobot/log:rw "
	export DOCKER_MOUNTS="${DOCKER_MOUNTS} -v $STATE_DIR:/cryptobot/state:rw "
	export DOCKER_MOUNTS="${DOCKER_MOUNTS} -v $RESULTS_DIR:/cryptobot/results:rw "
	export DOCKER_MOUNTS="${DOCKER_MOUNTS} -v $CONTROL_DIR:/cryptobot/control:rw "
	export DOCKER_MOUNTS="${DOCKER_MOUNTS} -v $CACHE_DIR:/cryptobot/cache:rw "
	export DOCKER_MOUNTS="${DOCKER_MOUNTS} -v $TESTS_DIR:/cryptobot/tests:rw "
	export DOCKER_MOUNTS="${DOCKER_MOUNTS} -v $TMP_DIR:/cryptobot/tmp:rw "

	export DOCKER_RUN_AS="--user $(id -u):$(id -g)"
	export DOCKER_PREFIX_VARS="U=`id -u` G=`id -g` BIND=${BIND_ADDRESS} "
	export CONTAINER_SUFFIX="$(whoami)-$(pwd |md5sum |cut -c1-8)"
	export DOCKER_NAME="--name $MODE-${CONTAINER_SUFFIX}"
	export DOCKER_NETWORK="--network ${CONTAINER_SUFFIX}"

	checks
	docker_network
  set_service_ports klines_caching_service config_endpoint_service live testnet price_log_service
	${MODE}
}

main $*