Gen4: move insert planner to gen4 (vitessio#12934) (vitessio#2318)

* use AliasTableExpr instead of TableName for insert table ast



* gen4: insert unsharded query planner with shortcut support



* gen4: insert sharded initial planning



* added more in insert engine description



* generate next sequence query using ast struct



* gen4: added autogenerate column support



* remove value false from plan description



* added support for insert ignore and insert on duplicate planning



* gen4: added support for comment directive



* error only when column list is empty and values are provided and table's column list is not known



* gen4: added support for insert using select



* gen4: added support for remaining cases in insert select and error on unsupported cases



* fix test expectation and query transformation



* fix sequence table fix for unsharded to ignore column vindex and addressed review comments



* gen4: fix check vindex updates on dup in insert with select, populate prefix and suffix for all the cases



* gen4: added check for non-streaming select when same tables involved



* added new plan test for insert cases



* added additional test for lookup column as auto increment column



* added V3Insert planner version for user to use old v3 planner for insert, added release notes



* addressed review comments in release notes



* added V3Insert in multiple docs



---------

Signed-off-by: Harshit Gangal <harshit@planetscale.com>
Co-authored-by: Harshit Gangal <harshit@planetscale.com>

changelog/17.0/17.0.0/summary.md

@@ -27,6 +27,7 @@
     - [Support for the `noblob` binlog row image mode](#noblob)
   - **[VTGate](#vtgate)
     - [StreamExecute GRPC API](#stream-execute)
+    - [Insert Planner Gen4](#insert-planner)
   - **[Deprecations and Deletions](#deprecations-and-deletions)**
     - [Deprecated Flags](#deprecated-flags)
     - [Deprecated Stats](#deprecated-stats)
@@ -368,6 +369,13 @@ so that it can be persisted with the client and sent back to VTGate on the next
 This does not impact anyone using the mysql client library to connect to VTGate.
 This could be a breaking change for grpc api users based on how they have implemented their grpc clients.
 
+#### <a id="insert-planner"/> Insert Planning with Gen4
+
+Gen4 planner was made default in v14 for `SELECT` queries. In v15 `UPDATE` and `DELETE` queries were moved to Gen4 framework.
+With this release `INSERT` queries are moved to Gen4.
+
+Clients can move to old v3 planner for inserts by using `V3Insert` planner version with `--planner-version` vtgate flag or with comment directive /*vt+ planner=<planner_version>` for individual query.
+
 ### <a id="deprecations-and-deletions"/> Deprecations and Deletions
 
 * The deprecated `automation` and `automationservice` protobuf definitions and associated client and server packages have been removed.

go/cmd/vtcombo/main.go

@@ -61,7 +61,7 @@ var (
 	mysqlPort          = flags.Int("mysql_port", 3306, "mysql port")
 	externalTopoServer = flags.Bool("external_topo_server", false, "Should vtcombo use an external topology server instead of starting its own in-memory topology server. "+
 		"If true, vtcombo will use the flags defined in topo/server.go to open topo server")
-	plannerName = flags.String("planner-version", "", "Sets the default planner to use when the session has not changed it. Valid values are: V3, Gen4, Gen4Greedy and Gen4Fallback. Gen4Fallback tries the gen4 planner and falls back to the V3 planner if the gen4 fails.")
+	plannerName = flags.String("planner-version", "", "Sets the default planner to use when the session has not changed it. Valid values are: V3, V3Insert, Gen4, Gen4Greedy and Gen4Fallback. Gen4Fallback tries the gen4 planner and falls back to the V3 planner if the gen4 fails.")
 
 	tpb             vttestpb.VTTestTopology
 	ts              *topo.Server

go/cmd/vtgate/vtgate.go

@@ -48,7 +48,7 @@ var (
 func registerFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&cell, "cell", cell, "cell to use")
 	fs.Var((*topoproto.TabletTypeListFlag)(&tabletTypesToWait), "tablet_types_to_wait", "Wait till connected for specified tablet types during Gateway initialization. Should be provided as a comma-separated set of tablet types.")
-	fs.StringVar(&plannerName, "planner-version", plannerName, "Sets the default planner to use when the session has not changed it. Valid values are: V3, Gen4, Gen4Greedy and Gen4Fallback. Gen4Fallback tries the gen4 planner and falls back to the V3 planner if the gen4 fails.")
+	fs.StringVar(&plannerName, "planner-version", plannerName, "Sets the default planner to use when the session has not changed it. Valid values are: V3, V3Insert, Gen4, Gen4Greedy and Gen4Fallback. Gen4Fallback tries the gen4 planner and falls back to the V3 planner if the gen4 fails.")
 
 	acl.RegisterFlags(fs)
 }

go/cmd/vttestserver/main.go

@@ -141,7 +141,7 @@ func registerFlags(fs *pflag.FlagSet) {
 
 	fs.StringVar(&config.Charset, "charset", "utf8mb4", "MySQL charset")
 
-	fs.StringVar(&config.PlannerVersion, "planner-version", "", "Sets the default planner to use when the session has not changed it. Valid values are: V3, Gen4, Gen4Greedy and Gen4Fallback. Gen4Fallback tries the new gen4 planner and falls back to the V3 planner if the gen4 fails.")
+	fs.StringVar(&config.PlannerVersion, "planner-version", "", "Sets the default planner to use when the session has not changed it. Valid values are: V3, V3Insert, Gen4, Gen4Greedy and Gen4Fallback. Gen4Fallback tries the new gen4 planner and falls back to the V3 planner if the gen4 fails.")
 
 	fs.StringVar(&config.SnapshotFile, "snapshot_file", "",
 		"A MySQL DB snapshot file")

go/flags/endtoend/vtgate.txt

@@ -148,7 +148,7 @@ Usage of vtgate:
       --onterm_timeout duration                                          wait no more than this for OnTermSync handlers before stopping (default 10s)
       --opentsdb_uri string                                              URI of opentsdb /api/put method
       --pid_file string                                                  If set, the process will write its pid to the named file, and delete it on graceful shutdown.
-      --planner-version string                                           Sets the default planner to use when the session has not changed it. Valid values are: V3, Gen4, Gen4Greedy and Gen4Fallback. Gen4Fallback tries the gen4 planner and falls back to the V3 planner if the gen4 fails.
+      --planner-version string                                           Sets the default planner to use when the session has not changed it. Valid values are: V3, V3Insert, Gen4, Gen4Greedy and Gen4Fallback. Gen4Fallback tries the gen4 planner and falls back to the V3 planner if the gen4 fails.
       --port int                                                         port for the server
       --pprof strings                                                    enable profiling
       --proxy_protocol                                                   Enable HAProxy PROXY protocol on MySQL listener socket

go/flags/endtoend/vttestserver.txt

@@ -85,7 +85,7 @@ Usage of vttestserver:
       --onterm_timeout duration                                          wait no more than this for OnTermSync handlers before stopping (default 10s)
       --persistent_mode                                                  If this flag is set, the MySQL data directory is not cleaned up when LocalCluster.TearDown() is called. This is useful for running vttestserver as a database container in local developer environments. Note that db migration files (--schema_dir option) and seeding of random data (--initialize_with_random_data option) will only run during cluster startup if the data directory does not already exist. vschema migrations are run every time the cluster starts, since persistence for the topology server has not been implemented yet
       --pid_file string                                                  If set, the process will write its pid to the named file, and delete it on graceful shutdown.
-      --planner-version string                                           Sets the default planner to use when the session has not changed it. Valid values are: V3, Gen4, Gen4Greedy and Gen4Fallback. Gen4Fallback tries the new gen4 planner and falls back to the V3 planner if the gen4 fails.
+      --planner-version string                                           Sets the default planner to use when the session has not changed it. Valid values are: V3, V3Insert, Gen4, Gen4Greedy and Gen4Fallback. Gen4Fallback tries the new gen4 planner and falls back to the V3 planner if the gen4 fails.
       --pool_hostname_resolve_interval duration                          if set force an update to all hostnames and reconnect if changed, defaults to 0 (disabled)
       --port int                                                         Port to use for vtcombo. If this is 0, a random port will be chosen.
       --pprof strings                                                    enable profiling

go/test/endtoend/vtgate/queries/dml/insert_test.go

@@ -394,3 +394,60 @@ func TestRedactDupError(t *testing.T) {
 	// inserting same rows, throws error.
 	mcmp.AssertContainsError("insert into order_tbl(region_id, oid, cust_no) select region_id, oid, cust_no from order_tbl", `BindVars: {REDACTED}`)
 }
+
+// TestMixedCases test all the cases for insert when lookup column is also the auto increment column.
+func TestMixedCases(t *testing.T) {
+	mcmp, closer := start(t)
+	defer closer()
+
+	tcases := []struct {
+		insQuery string
+		selQuery string
+		exp      string
+	}{{
+		// values are provided for all columns
+		insQuery: "insert into mixed_tbl(shard_key, lkp_key) values (1, 1000)",
+		selQuery: "select * from mixed_tbl where lkp_key = 1000",
+		exp:      "[[INT64(1000) INT64(1)]]",
+	}, {
+		// lookup column value not provided - auto increment value should be used.
+		insQuery: "insert into mixed_tbl(shard_key) values (2)",
+		selQuery: "select * from mixed_tbl where lkp_key = 1",
+		exp:      "[[INT64(1) INT64(2)]]",
+	}, {
+		// lookup column value not provided in the select - auto increment value should be used.
+		insQuery: "insert into mixed_tbl(shard_key) select 3",
+		selQuery: "select * from mixed_tbl where lkp_key = 2",
+		exp:      "[[INT64(2) INT64(3)]]",
+	}, {
+		// lookup column value provided as NULL in the select - auto increment value should be used.
+		insQuery: "insert into mixed_tbl(shard_key, lkp_key) select 4, null",
+		selQuery: "select * from mixed_tbl where lkp_key = 3",
+		exp:      "[[INT64(3) INT64(4)]]",
+	}, {
+		// values are provided for all column in the select
+		insQuery: "insert into mixed_tbl(shard_key, lkp_key) select 5, 2000",
+		selQuery: "select * from mixed_tbl where lkp_key = 2000",
+		exp:      "[[INT64(2000) INT64(5)]]",
+	}, {
+		// multiple values are inserted - lookup column value not provided - use auto increment value
+		insQuery: "insert into mixed_tbl(shard_key) select shard_key from mixed_tbl order by shard_key desc",
+		selQuery: "select * from mixed_tbl where lkp_key between 4 and 8 order by lkp_key",
+		exp:      "[[INT64(4) INT64(5)] [INT64(5) INT64(4)] [INT64(6) INT64(3)] [INT64(7) INT64(2)] [INT64(8) INT64(1)]]",
+	}, {
+		// partial values are provided from lookup column - use auto increment value where missing.
+		insQuery: "insert into mixed_tbl(shard_key, lkp_key) (select 2, 3000 union select 5, null)",
+		selQuery: "select * from mixed_tbl where lkp_key in (9, 3000) order by lkp_key",
+		exp:      "[[INT64(9) INT64(5)] [INT64(3000) INT64(2)]]",
+	}}
+
+	for _, tc := range tcases {
+		t.Run(tc.insQuery, func(t *testing.T) {
+			utils.Exec(t, mcmp.VtConn, tc.insQuery)
+			utils.AssertMatches(t, mcmp.VtConn, tc.selQuery, tc.exp)
+		})
+	}
+
+	// final check count on the lookup vindex table.
+	utils.AssertMatches(t, mcmp.VtConn, "select count(*) from lkp_mixed_idx", "[[INT64(12)]]")
+}

go/test/endtoend/vtgate/queries/dml/main_test.go

@@ -57,6 +57,9 @@ var (
     },
     "auto_seq": {
        "type":   "sequence"
+    },
+    "mixed_seq": {
+       "type":   "sequence"
     }
   }
 }`
@@ -130,7 +133,7 @@ func start(t *testing.T) (utils.MySQLCompare, func()) {
 
 		tables := []string{
 			"s_tbl", "num_vdx_tbl", "user_tbl", "order_tbl", "oevent_tbl", "oextra_tbl",
-			"auto_tbl", "oid_vdx_tbl", "unq_idx", "nonunq_idx", "u_tbl",
+			"auto_tbl", "oid_vdx_tbl", "unq_idx", "nonunq_idx", "u_tbl", "mixed_tbl", "lkp_map_idx",
 		}
 		for _, table := range tables {
 			// TODO (@frouioui): following assertions produce different results between MySQL and Vitess

go/test/endtoend/vtgate/queries/dml/sharded_schema.sql

@@ -71,4 +71,18 @@ create table nonunq_idx
     id          bigint,
     keyspace_id varbinary(20),
     primary key (nonunq_col, id)
-) Engine = InnoDB;
+) Engine = InnoDB;
+
+create table mixed_tbl
+(
+    lkp_key   bigint,
+    shard_key bigint,
+    primary key (lkp_key)
+) Engine = InnoDB;
+
+create table lkp_mixed_idx
+(
+    lkp_key     bigint,
+    keyspace_id varbinary(20),
+    primary key (lkp_key)
+) Engine = InnoDB;

go/test/endtoend/vtgate/queries/dml/unsharded_schema.sql

@@ -14,6 +14,14 @@ create table auto_seq
     primary key (id)
 ) comment 'vitess_sequence' Engine = InnoDB;
 
+create table mixed_seq
+(
+    id      int    default 0,
+    next_id bigint default null,
+    cache   bigint default null,
+    primary key (id)
+) comment 'vitess_sequence' Engine = InnoDB;
+
 create table u_tbl
 (
     id  bigint,
@@ -25,3 +33,5 @@ insert into user_seq(id, next_id, cache)
 values (0, 1, 1000);
 insert into auto_seq(id, next_id, cache)
 values (0, 666, 1000);
+insert into mixed_seq(id, next_id, cache)
+values (0, 1, 1000);

go/test/endtoend/vtgate/queries/dml/vschema.json

@@ -41,6 +41,16 @@
         "ignore_nulls": "true"
       },
       "owner": "auto_tbl"
+    },
+    "lkp_map_vdx": {
+      "type": "consistent_lookup_unique",
+      "params": {
+        "table": "lkp_mixed_idx",
+        "from": "lkp_key",
+        "to": "keyspace_id",
+        "ignore_nulls": "true"
+      },
+      "owner": "mixed_tbl"
     }
   },
   "tables": {
@@ -154,6 +164,30 @@
           "name": "hash"
         }
       ]
+    },
+    "mixed_tbl": {
+      "auto_increment": {
+        "column": "lkp_key",
+        "sequence": "uks.mixed_seq"
+      },
+      "column_vindexes": [
+        {
+          "column": "shard_key",
+          "name": "hash"
+        },
+        {
+          "column": "lkp_key",
+          "name": "lkp_map_vdx"
+        }
+      ]
+    },
+    "lkp_mixed_idx": {
+      "column_vindexes": [
+        {
+          "column": "lkp_key",
+          "name": "hash"
+        }
+      ]
     }
   }
 }

go/test/endtoend/vtgate/sequence/seq_test.go

@@ -191,7 +191,7 @@ func TestMain(m *testing.M) {
 			SchemaSQL: unshardedSQLSchema,
 			VSchema:   unshardedVSchema,
 		}
-		if err := clusterInstance.StartUnshardedKeyspace(*uKeyspace, 1, false); err != nil {
+		if err := clusterInstance.StartUnshardedKeyspace(*uKeyspace, 0, false); err != nil {
 			return 1
 		}
 
@@ -200,7 +200,7 @@ func TestMain(m *testing.M) {
 			SchemaSQL: shardedSQLSchema,
 			VSchema:   shardedVSchema,
 		}
-		if err := clusterInstance.StartKeyspace(*sKeyspace, []string{"-80", "80-"}, 1, false); err != nil {
+		if err := clusterInstance.StartKeyspace(*sKeyspace, []string{"-80", "80-"}, 0, false); err != nil {
 			return 1
 		}