Merge remote-tracking branch 'origin/main' into vrepl_tablet_picker

Signed-off-by: Matt Lord <mattalord@gmail.com>

.github/workflows/cluster_endtoend_vreplication_partial_movetables_simple.yml → .github/workflows/cluster_endtoend_vreplication_partial_movetables_basic.yml

@@ -1,9 +1,9 @@
 # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows"
 
-name: Cluster (vreplication_partial_movetables_simple)
+name: Cluster (vreplication_partial_movetables_basic)
 on: [push, pull_request]
 concurrency:
-  group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vreplication_partial_movetables_simple)')
+  group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vreplication_partial_movetables_basic)')
   cancel-in-progress: true
 
 permissions: read-all
@@ -15,7 +15,7 @@ env:
 
 jobs:
   build:
-    name: Run endtoend tests on Cluster (vreplication_partial_movetables_simple)
+    name: Run endtoend tests on Cluster (vreplication_partial_movetables_basic)
     runs-on: ubuntu-22.04
 
     steps:
@@ -58,7 +58,7 @@ jobs:
             - 'tools/**'
             - 'config/**'
             - 'bootstrap.sh'
-            - '.github/workflows/cluster_endtoend_vreplication_partial_movetables_simple.yml'
+            - '.github/workflows/cluster_endtoend_vreplication_partial_movetables_basic.yml'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
@@ -149,7 +149,7 @@ jobs:
         EOF
         
         # run the tests however you normally do, then produce a JUnit XML file
-        eatmydata -- go run test.go -docker=false -follow -shard vreplication_partial_movetables_simple | tee -a output.txt | go-junit-report -set-exit-code > report.xml
+        eatmydata -- go run test.go -docker=false -follow -shard vreplication_partial_movetables_basic | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 
     - name: Print test output and Record test result in launchable
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always()

.github/workflows/e2e_race.yml

@@ -87,4 +87,4 @@ jobs:
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       timeout-minutes: 30
       run: |
-        make e2e_test_race
+        NOVTADMINBUILD=1 make e2e_test_race

.github/workflows/unit_race.yml

@@ -95,4 +95,5 @@ jobs:
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true'
       timeout-minutes: 45
       run: |
+        export NOVTADMINBUILD=1
         eatmydata -- make unit_test_race

.github/workflows/unit_test_mysql57.yml

@@ -143,6 +143,9 @@ jobs:
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true'
       timeout-minutes: 30
       run: |
+        set -exo pipefail
+
+        export NOVTADMINBUILD=1
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 
     - name: Print test output and Record test result in launchable

.github/workflows/unit_test_mysql80.yml

@@ -140,6 +140,9 @@ jobs:
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true'
       timeout-minutes: 30
       run: |
+        set -exo pipefail
+
+        export NOVTADMINBUILD=1
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 
     - name: Print test output and Record test result in launchable

Makefile

@@ -379,7 +379,7 @@ tools:
 
 minimaltools:
 	echo $$(date): Installing minimal dependencies
-	BUILD_CHROME=0 BUILD_JAVA=0 BUILD_CONSUL=0 ./bootstrap.sh
+	BUILD_JAVA=0 BUILD_CONSUL=0 ./bootstrap.sh
 
 dependency_check:
 	./tools/dependency_check.sh

bootstrap.sh

@@ -25,7 +25,6 @@ source ./dev.env
 
 BUILD_JAVA=${BUILD_JAVA:-1}
 BUILD_CONSUL=${BUILD_CONSUL:-1}
-BUILD_CHROME=${BUILD_CHROME:-1}
 
 VITESS_RESOURCES_DOWNLOAD_BASE_URL="https://github.com/vitessio/vitess-resources/releases/download"
 VITESS_RESOURCES_RELEASE="v4.0"
@@ -198,37 +197,6 @@ install_consul() {
 }
 
 
-# Download chromedriver
-install_chromedriver() {
-  local version="$1"
-  local dist="$2"
-
-  case $(uname) in
-    Linux)  local platform=linux;;
-    *)   echo "Platform not supported for vtctl-web tests. Skipping chromedriver install."; return;;
-  esac
-
-  if [ "$(arch)" == "aarch64" ] ; then
-      os=$(cat /etc/*release | grep "^ID=" | cut -d '=' -f 2)
-      case $os in
-        ubuntu|debian)
-          sudo apt-get update -y && sudo apt install -y --no-install-recommends unzip libglib2.0-0 libnss3 libx11-6
-        ;;
-        centos|fedora)
-          sudo yum update -y && yum install -y libX11 unzip wget
-        ;;
-      esac
-      echo "For Arm64, using prebuilt binary from electron (https://github.com/electron/electron/) of version 76.0.3809.126"
-      "${VTROOT}/tools/wget-retry" https://github.com/electron/electron/releases/download/v6.0.3/chromedriver-v6.0.3-linux-arm64.zip
-      unzip -o -q chromedriver-v6.0.3-linux-arm64.zip -d "$dist"
-      rm chromedriver-v6.0.3-linux-arm64.zip
-  else
-      "${VTROOT}/tools/wget-retry" "https://chromedriver.storage.googleapis.com/$version/chromedriver_linux64.zip"
-      unzip -o -q chromedriver_linux64.zip -d "$dist"
-      rm chromedriver_linux64.zip
-  fi
-}
-
 # Download and install toxiproxy, link toxiproxy binary into our root.
 install_toxiproxy() {
   local version="$1"
@@ -275,11 +243,6 @@ install_all() {
     install_dep "Consul" "1.11.4" "$VTROOT/dist/consul" install_consul
   fi
 
-  # chromedriver
-  if [ "$BUILD_CHROME" == 1 ] ; then
-    install_dep "chromedriver" "90.0.4430.24" "$VTROOT/dist/chromedriver" install_chromedriver
-  fi
-
   # toxiproxy
   install_dep "toxiproxy" "v2.5.0" "$VTROOT/dist/toxiproxy" install_toxiproxy
 

changelog/18.0/18.0.0/summary.md

@@ -55,6 +55,10 @@ Throttler related `vttablet` flags:
 - `--throttle_check_as_check_self` is deprecated and will be removed in `v19.0`
 - `--throttler-config-via-topo` is deprecated after asummed `true` in `v17.0`. It will be removed in a future version.
 
+VTGate flag:
+
+- `--schema_change_signal_user` is deprecated and will be removed in `v19.0`
+
 #### <a id="deleted-v3"/>Deleted `v3` planner
 
 The `Gen4` planner has been the default planner since Vitess 14. The `v3` planner was deprecated in Vitess 15 and has now been removed in this release.

dev.env

@@ -25,9 +25,6 @@ source ./build.env
 export VTPORTSTART=6700
 
 
-# Add chromedriver to path for Selenium tests.
-PATH=$(prepend_path "$PATH" "$VTROOT/dist/chromedriver")
-
 # Node path.
 PATH=$(prepend_path "$PATH" "$VTROOT/dist/node/bin")
 export PATH

docker/bootstrap/Dockerfile.common

@@ -22,7 +22,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
 ENV VTROOT /vt/src/vitess.io/vitess
 ENV VTDATAROOT /vt/vtdataroot
 ENV VTPORTSTART 15000
-ENV PATH $VTROOT/bin:$VTROOT/dist/maven/bin:$VTROOT/dist/chromedriver:$PATH
+ENV PATH $VTROOT/bin:$VTROOT/dist/maven/bin:$PATH
 ENV USER vitess
 
 # Copy files needed for bootstrap

examples/common/env.sh

@@ -26,7 +26,7 @@ fi
 
 # mysqld might be in /usr/sbin which will not be in the default PATH
 PATH="/usr/sbin:$PATH"
-for binary in mysqld etcd etcdctl curl vtctlclient vttablet vtgate vtctld mysqlctl; do
+for binary in mysqld etcd etcdctl curl vtctlclient vtctldclient vttablet vtgate vtctld mysqlctl; do
   command -v "$binary" > /dev/null || fail "${binary} is not installed in PATH. See https://vitess.io/docs/get-started/local/ for install instructions."
 done;
 

examples/common/lib/utils.sh

@@ -139,6 +139,35 @@ function wait_for_healthy_shard() {
 	wait_for_shard_vreplication_engine "${keyspace}" "${shard}"
 }
 
+# Wait for a workflow to reach the running state. Example:
+#  wait_for_workflow_running customer customer2customer
+function wait_for_workflow_running() {
+    if [[ -z ${1} || -z ${2} ]]; then
+        fail "A keyspace and workflow must be specified when waiting for a workflow to reach the running state"
+    fi
+
+    local keyspace=${1}
+    local workflow=${2}
+    local wait_secs=90
+    local result=""
+
+    echo "Waiting for the ${workflow} workflow in the ${keyspace} keyspace to finish the copy phase..." 
+
+    for _ in $(seq 1 ${wait_secs}); do
+        result=$(vtctldclient Workflow --keyspace="${keyspace}" show --workflow="${workflow}" 2>/dev/null | grep "Copy phase completed")
+        if [[ ${result} != "" ]]; then
+            break
+        fi
+        sleep 1
+    done;
+
+    if [[ ${result} == "" ]]; then
+        fail "Timed out after ${wait_secs} seconds waiting for the ${workflow} workflow in the ${keyspace} keyspace to reach the running state"
+    fi
+
+    echo "The ${workflow} workflow in the ${keyspace} keyspace is now running. $(sed -rn 's/.*"(Copy phase.*)".*/\1/p' <<< "${result}")."
+}
+
 # Stop the specified binary name using the provided PID file.
 # Example:
 #  stop_process "vtadmin-web" "$VTDATAROOT/tmp/vtadmin-web.pid"

examples/local/202_move_tables.sh

@@ -19,4 +19,7 @@
 
 source ../common/env.sh
 
-vtctlclient MoveTables -- --source commerce --tables 'customer,corder' Create customer.commerce2customer
+vtctldclient MoveTables --workflow commerce2customer --target-keyspace customer create --source-keyspace commerce --tables "customer,corder"
+
+# Wait for the workflow to reach the running state.
+wait_for_workflow_running customer commerce2customer

examples/local/203_switch_reads.sh

@@ -19,4 +19,4 @@
 
 source ../common/env.sh
 
-vtctlclient MoveTables -- --tablet_types=rdonly,replica SwitchTraffic customer.commerce2customer
+vtctldclient MoveTables --workflow commerce2customer --target-keyspace customer switchtraffic --tablet-types "rdonly,replica"

examples/local/204_switch_writes.sh

@@ -19,4 +19,4 @@
 
 source ../common/env.sh
 
-vtctlclient MoveTables -- --tablet_types=primary SwitchTraffic customer.commerce2customer
+vtctldclient MoveTables --workflow commerce2customer --target-keyspace customer switchtraffic --tablet-types primary

examples/local/205_clean_commerce.sh

@@ -19,5 +19,4 @@
 
 source ../common/env.sh
 
-vtctlclient MoveTables Complete customer.commerce2customer
-
+vtctldclient MoveTables --workflow commerce2customer --target-keyspace customer complete

go/cmd/internal/docgen/docgen.go

@@ -156,7 +156,7 @@ func restructure(rootDir string, dir string, name string, commands []*cobra.Comm
 }
 
 func newParentLinkSedCommand(parent string, file string) *exec.Cmd {
-	return exec.Command("sed", "-i ''", "-e", fmt.Sprintf("s:(./%s/):(../):", parent), file)
+	return exec.Command("sed", "-i", "", "-e", fmt.Sprintf("s:(./%s/):(../):i", parent), file)
 }
 
 func recursivelyDisableAutoGenTags(root *cobra.Command) {

go/cmd/vtctldclient/cli/json.go

@@ -54,3 +54,25 @@ func MarshalJSON(obj any) ([]byte, error) {
 		return data, nil
 	}
 }
+
+// MarshalJSONCompact works the same as MarshalJSON but elides zero value elements.
+func MarshalJSONCompact(obj any) ([]byte, error) {
+	switch obj := obj.(type) {
+	case proto.Message:
+		m := protojson.MarshalOptions{
+			Multiline:       true,
+			Indent:          "  ",
+			UseEnumNumbers:  true,
+			UseProtoNames:   true,
+			EmitUnpopulated: false, // elide zero value elements
+		}
+		return m.Marshal(obj)
+	default:
+		data, err := json.MarshalIndent(obj, "", "  ")
+		if err != nil {
+			return nil, fmt.Errorf("json.Marshal = %v", err)
+		}
+
+		return data, nil
+	}
+}