From b2d59929b23737bce5746780784169249d83ac2b Mon Sep 17 00:00:00 2001 From: meiji163 Date: Thu, 12 Dec 2024 19:21:12 -0800 Subject: [PATCH 1/2] add docker localtest --- .gitignore | 1 + localtests/docker-compose.yml | 25 ++++++++++ localtests/test.sh | 21 ++++++-- script/docker-gh-ost-replica-tests | 78 ++++++++++++++++++++++++++++++ script/gh-ost-test-mysql-master | 6 +++ script/gh-ost-test-mysql-replica | 6 +++ 6 files changed, 132 insertions(+), 5 deletions(-) create mode 100644 localtests/docker-compose.yml create mode 100755 script/docker-gh-ost-replica-tests create mode 100755 script/gh-ost-test-mysql-master create mode 100755 script/gh-ost-test-mysql-replica diff --git a/.gitignore b/.gitignore index 605546d20..5b4676a24 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ /libexec/ /.vendor/ .idea/ +*.tmp diff --git a/localtests/docker-compose.yml b/localtests/docker-compose.yml new file mode 100644 index 000000000..f7739814c --- /dev/null +++ b/localtests/docker-compose.yml @@ -0,0 +1,25 @@ +services: + mysql-primary: + image: $TEST_MYSQL_IMAGE + container_name: mysql-primary + command: --server-id=1 --log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON + environment: + MYSQL_ROOT_PASSWORD: opensesame + MYSQL_DATABASE: test + MYSQL_TCP_PORT: 3307 + ports: + - '3307:3307' + expose: + - '3307' + mysql-replica: + image: $TEST_MYSQL_IMAGE + container_name: mysql-replica + command: --server-id=2 --log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON --log-slave-updates=ON + environment: + MYSQL_ROOT_PASSWORD: opensesame + MYSQL_DATABASE: test + MYSQL_TCP_PORT: 3308 + ports: + - '3308:3308' + expose: + - '3308' diff --git a/localtests/test.sh b/localtests/test.sh index 14ecd83ea..0c8670cb6 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -11,6 +11,7 @@ tests_path=$(dirname $0) test_logfile=/tmp/gh-ost-test.log default_ghost_binary=/tmp/gh-ost-test ghost_binary="" +docker=false storage_engine=innodb exec_command_file=/tmp/gh-ost-test.bash ghost_structure_output_file=/tmp/gh-ost-test.ghost.structure.sql @@ -25,13 +26,15 @@ replica_port= original_sql_mode= OPTIND=1 -while getopts "b:s:" OPTION +while getopts "b:s:d" OPTION do case $OPTION in b) ghost_binary="$OPTARG";; s) storage_engine="$OPTARG";; + d) + docker=true;; esac done shift $((OPTIND-1)) @@ -98,6 +101,13 @@ test_single() { local test_name test_name="$1" + if [ "$docker" = true ]; then + master_host="0.0.0.0" + master_port="3307" + replica_host="0.0.0.0" + replica_port="3308" + fi + if [ -f $tests_path/$test_name/ignore_versions ] ; then ignore_versions=$(cat $tests_path/$test_name/ignore_versions) mysql_version=$(gh-ost-test-mysql-master -s -s -e "select @@version") @@ -270,9 +280,10 @@ build_binary() { test_all() { build_binary - find $tests_path ! -path . -type d -mindepth 1 -maxdepth 1 | cut -d "/" -f 3 | egrep "$test_pattern" | sort | while read test_name ; do - test_single "$test_name" - if [ $? -ne 0 ] ; then + test_dirs=$(find "$tests_path" -mindepth 1 -maxdepth 1 ! -path . -type d | grep "$test_pattern" | sort) + while read -r test_dir; do + test_name=$(basename "$test_dir") + if ! test_single "$test_name" ; then create_statement=$(gh-ost-test-mysql-replica test -t -e "show create table _gh_ost_test_gho \G") echo "$create_statement" >> $test_logfile echo "+ FAIL" @@ -282,7 +293,7 @@ test_all() { echo "+ pass" fi gh-ost-test-mysql-replica -e "start slave" - done + done <<< "$test_dirs" } verify_master_and_replica diff --git a/script/docker-gh-ost-replica-tests b/script/docker-gh-ost-replica-tests new file mode 100755 index 000000000..84f19b8e9 --- /dev/null +++ b/script/docker-gh-ost-replica-tests @@ -0,0 +1,78 @@ +#!/bin/bash + +# This script starts two MySQL docker containers in a primary-replica setup +# which can be used for running the replica tests in localtests/ . +# Set the environment var TEST_MYSQL_IMAGE to change the docker image. +# +# Usage: +# docker-gh-ost-replica-tests up start the containers +# docker-gh-ost-replica-tests down remove the containers +# docker-gh-ost-replica-tests run run replica tests on the containers + +set -e + +GH_OST_ROOT=$(git rev-parse --show-toplevel) +if [[ ":$PATH:" != *":$GH_OST_ROOT:"* ]]; then + export PATH="${PATH}:${GH_OST_ROOT}/script" +fi + +poll_mysql() { + CTR=0 + cmd="gh-ost-test-mysql-$1" + while ! $cmd -e "select 1;" > /dev/null 2>&1 + do + sleep 1 + CTR=$((CTR + 1)) + if [ $CTR -gt 30 ]; then + echo " ❌ MySQL $1 failed to start" + return 1 + fi + done + echo " ✔ MySQL $1 OK" + return 0 +} + +setup() { + [ -z "$TEST_MYSQL_IMAGE" ] && TEST_MYSQL_IMAGE="mysql:8.0.39" + + echo "Starting MySQL $TEST_MYSQL_IMAGE containers..." + compose_file="$GH_OST_ROOT/localtests/docker-compose.yml" + (TEST_MYSQL_IMAGE="$TEST_MYSQL_IMAGE" envsubst < "$compose_file") > "$compose_file.tmp" + docker compose -f "$compose_file.tmp" up -d --wait + + echo "Waiting for MySQL..." + poll_mysql "master" || exit 1 + poll_mysql "replica" || exit 1 + + echo -n "Setting up replication..." + gh-ost-test-mysql-master -e "create user if not exists 'repl'@'%' identified with 'mysql_native_password' by 'repl';" + gh-ost-test-mysql-master -e "grant replication slave on *.* to 'repl'@'%'; flush privileges;" + gh-ost-test-mysql-master -e "create user if not exists 'gh-ost'@'%' identified by 'gh-ost';" + gh-ost-test-mysql-master -e "grant all on *.* to 'gh-ost'@'%';" + + gh-ost-test-mysql-replica -e "change master to master_host='mysql-primary', master_port=3307, master_user='repl', master_password='repl', master_auto_position=1;" + gh-ost-test-mysql-replica -e "start slave;" + echo "OK" +} + +teardown() { + echo "Stopping containers..." + docker stop mysql-replica + docker stop mysql-primary + echo "Removing containers..." + docker rm mysql-replica + docker rm mysql-primary +} + +main() { + if [[ "$1" == "up" ]]; then + setup + elif [[ "$1" == "down" ]]; then + teardown + elif [[ "$1" == "run" ]]; then + shift 1 + "$GH_OST_ROOT/localtests/test.sh" -d "$@" + fi +} + +main "$@" diff --git a/script/gh-ost-test-mysql-master b/script/gh-ost-test-mysql-master new file mode 100755 index 000000000..d517ca77b --- /dev/null +++ b/script/gh-ost-test-mysql-master @@ -0,0 +1,6 @@ +#!/bin/bash +# +# This executes a command on the mysql-primary docker container created +# from localtests/docker-compose.yml. It's used by localtests/test.sh. + +MYSQL_PWD=opensesame mysql -uroot -h0.0.0.0 -P3307 "$@" diff --git a/script/gh-ost-test-mysql-replica b/script/gh-ost-test-mysql-replica new file mode 100755 index 000000000..ae2646bf3 --- /dev/null +++ b/script/gh-ost-test-mysql-replica @@ -0,0 +1,6 @@ +#!/bin/bash +# +# This executes a command on the mysql-replica docker container created +# from localtests/docker-compose.yml. It's used by localtests/test.sh. + +MYSQL_PWD=opensesame mysql -uroot -h0.0.0.0 -P3308 "$@" From 0471288d396642fc9edb7f83331507f171837bfd Mon Sep 17 00:00:00 2001 From: meiji163 Date: Thu, 12 Dec 2024 21:52:55 -0800 Subject: [PATCH 2/2] add MYSQL_ROOT_HOST --- localtests/docker-compose.yml | 2 ++ script/docker-gh-ost-replica-tests | 1 + 2 files changed, 3 insertions(+) diff --git a/localtests/docker-compose.yml b/localtests/docker-compose.yml index f7739814c..845650ed4 100644 --- a/localtests/docker-compose.yml +++ b/localtests/docker-compose.yml @@ -5,6 +5,7 @@ services: command: --server-id=1 --log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON environment: MYSQL_ROOT_PASSWORD: opensesame + MYSQL_ROOT_HOST: '%' MYSQL_DATABASE: test MYSQL_TCP_PORT: 3307 ports: @@ -17,6 +18,7 @@ services: command: --server-id=2 --log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON --log-slave-updates=ON environment: MYSQL_ROOT_PASSWORD: opensesame + MYSQL_ROOT_HOST: '%' MYSQL_DATABASE: test MYSQL_TCP_PORT: 3308 ports: diff --git a/script/docker-gh-ost-replica-tests b/script/docker-gh-ost-replica-tests index 84f19b8e9..15920a683 100755 --- a/script/docker-gh-ost-replica-tests +++ b/script/docker-gh-ost-replica-tests @@ -50,6 +50,7 @@ setup() { gh-ost-test-mysql-master -e "create user if not exists 'gh-ost'@'%' identified by 'gh-ost';" gh-ost-test-mysql-master -e "grant all on *.* to 'gh-ost'@'%';" + sleep 1 gh-ost-test-mysql-replica -e "change master to master_host='mysql-primary', master_port=3307, master_user='repl', master_password='repl', master_auto_position=1;" gh-ost-test-mysql-replica -e "start slave;" echo "OK"