Docker setup for localtests (#1479)

* add docker localtest

* add MYSQL_ROOT_HOST

.gitignore

@@ -3,3 +3,4 @@
 /libexec/
 /.vendor/
 .idea/
+*.tmp

localtests/docker-compose.yml

@@ -0,0 +1,27 @@
+services:
+  mysql-primary:
+    image: $TEST_MYSQL_IMAGE
+    container_name: mysql-primary
+    command: --server-id=1 --log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON
+    environment:
+      MYSQL_ROOT_PASSWORD: opensesame
+      MYSQL_ROOT_HOST: '%'
+      MYSQL_DATABASE: test
+      MYSQL_TCP_PORT: 3307
+    ports:
+      - '3307:3307'
+    expose:
+      - '3307'
+  mysql-replica:
+    image: $TEST_MYSQL_IMAGE
+    container_name: mysql-replica
+    command: --server-id=2 --log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON --log-slave-updates=ON
+    environment:
+      MYSQL_ROOT_PASSWORD: opensesame
+      MYSQL_ROOT_HOST: '%'
+      MYSQL_DATABASE: test
+      MYSQL_TCP_PORT: 3308
+    ports:
+      - '3308:3308'
+    expose:
+      - '3308'

localtests/test.sh

@@ -11,6 +11,7 @@ tests_path=$(dirname $0)
 test_logfile=/tmp/gh-ost-test.log
 default_ghost_binary=/tmp/gh-ost-test
 ghost_binary=""
+docker=false
 storage_engine=innodb
 exec_command_file=/tmp/gh-ost-test.bash
 ghost_structure_output_file=/tmp/gh-ost-test.ghost.structure.sql
@@ -25,13 +26,15 @@ replica_port=
 original_sql_mode=
 
 OPTIND=1
-while getopts "b:s:" OPTION
+while getopts "b:s:d" OPTION
 do
   case $OPTION in
     b)
       ghost_binary="$OPTARG";;
     s)
       storage_engine="$OPTARG";;
+    d)
+      docker=true;;
   esac
 done
 shift $((OPTIND-1))
@@ -98,6 +101,13 @@ test_single() {
   local test_name
   test_name="$1"
 
+  if [ "$docker" = true ]; then
+    master_host="0.0.0.0"
+    master_port="3307"
+    replica_host="0.0.0.0"
+    replica_port="3308"
+  fi
+
   if [ -f $tests_path/$test_name/ignore_versions ] ; then
     ignore_versions=$(cat $tests_path/$test_name/ignore_versions)
     mysql_version=$(gh-ost-test-mysql-master -s -s -e "select @@version")
@@ -270,9 +280,10 @@ build_binary() {
 
 test_all() {
   build_binary
-  find $tests_path ! -path . -type d -mindepth 1 -maxdepth 1 | cut -d "/" -f 3 | egrep "$test_pattern" | sort | while read test_name ; do
-    test_single "$test_name"
-    if [ $? -ne 0 ] ; then
+  test_dirs=$(find "$tests_path" -mindepth 1 -maxdepth 1 ! -path . -type d | grep "$test_pattern" | sort)
+  while read -r test_dir; do
+    test_name=$(basename "$test_dir")
+    if ! test_single "$test_name" ; then
       create_statement=$(gh-ost-test-mysql-replica test -t -e "show create table _gh_ost_test_gho \G")
       echo "$create_statement" >> $test_logfile
       echo "+ FAIL"
@@ -282,7 +293,7 @@ test_all() {
       echo "+ pass"
     fi
     gh-ost-test-mysql-replica -e "start slave"
-  done
+  done <<< "$test_dirs"
 }
 
 verify_master_and_replica

script/docker-gh-ost-replica-tests

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+# This script starts two MySQL docker containers in a primary-replica setup
+# which can be used for running the replica tests in localtests/ .
+# Set the environment var TEST_MYSQL_IMAGE to change the docker image.
+#
+# Usage:
+#   docker-gh-ost-replica-tests up      start the containers
+#   docker-gh-ost-replica-tests down    remove the containers
+#   docker-gh-ost-replica-tests run     run replica tests on the containers
+
+set -e
+
+GH_OST_ROOT=$(git rev-parse --show-toplevel)
+if [[ ":$PATH:" != *":$GH_OST_ROOT:"* ]]; then
+    export PATH="${PATH}:${GH_OST_ROOT}/script"
+fi
+
+poll_mysql() {
+    CTR=0
+    cmd="gh-ost-test-mysql-$1"
+    while ! $cmd -e "select 1;" > /dev/null 2>&1
+    do
+        sleep 1
+        CTR=$((CTR + 1))
+        if [ $CTR -gt 30 ]; then
+            echo " ❌ MySQL $1 failed to start"
+            return 1
+        fi
+    done
+    echo " ✔ MySQL $1 OK"
+    return 0
+}
+
+setup() {
+    [ -z "$TEST_MYSQL_IMAGE" ] && TEST_MYSQL_IMAGE="mysql:8.0.39"
+
+    echo "Starting MySQL $TEST_MYSQL_IMAGE containers..."
+    compose_file="$GH_OST_ROOT/localtests/docker-compose.yml"
+    (TEST_MYSQL_IMAGE="$TEST_MYSQL_IMAGE" envsubst < "$compose_file") > "$compose_file.tmp"
+    docker compose -f "$compose_file.tmp" up -d --wait
+
+    echo "Waiting for MySQL..."
+    poll_mysql "master" || exit 1
+    poll_mysql "replica" || exit 1
+
+    echo -n "Setting up replication..."
+    gh-ost-test-mysql-master -e "create user if not exists 'repl'@'%' identified with 'mysql_native_password' by 'repl';"
+    gh-ost-test-mysql-master -e "grant replication slave on *.* to 'repl'@'%'; flush privileges;"
+    gh-ost-test-mysql-master -e "create user if not exists 'gh-ost'@'%' identified by 'gh-ost';"
+    gh-ost-test-mysql-master -e "grant all on *.* to 'gh-ost'@'%';"
+
+    sleep 1
+    gh-ost-test-mysql-replica -e "change master to master_host='mysql-primary', master_port=3307, master_user='repl', master_password='repl', master_auto_position=1;"
+    gh-ost-test-mysql-replica -e "start slave;"
+    echo "OK"
+}
+
+teardown() {
+    echo "Stopping containers..."
+    docker stop mysql-replica
+    docker stop mysql-primary
+    echo "Removing containers..."
+    docker rm mysql-replica
+    docker rm mysql-primary
+}
+
+main() {
+    if [[ "$1" == "up" ]]; then
+        setup
+    elif [[ "$1" == "down" ]]; then
+        teardown
+    elif [[ "$1" == "run" ]]; then
+        shift 1
+        "$GH_OST_ROOT/localtests/test.sh" -d "$@"
+    fi
+}
+
+main "$@"

script/gh-ost-test-mysql-master

@@ -0,0 +1,6 @@
+#!/bin/bash
+#
+# This executes a command on the mysql-primary docker container created
+# from localtests/docker-compose.yml. It's used by localtests/test.sh.
+
+MYSQL_PWD=opensesame mysql -uroot -h0.0.0.0 -P3307 "$@"

script/gh-ost-test-mysql-replica

@@ -0,0 +1,6 @@
+#!/bin/bash
+#
+# This executes a command on the mysql-replica docker container created
+# from localtests/docker-compose.yml. It's used by localtests/test.sh.
+
+MYSQL_PWD=opensesame mysql -uroot -h0.0.0.0 -P3308 "$@"