ysql: implement backfill for index (yugabyte#2301)

Summary:

Implement core functionality for the backfill part of YSQL multi-stage
create index.  Do the following checked items:

- [x] Add `BACKFILL INDEX` grammar for postgres
- [x] Establish basic communication from tserver to postgres
- [x] Use ancient write time for inserting rows for backfill
- [x] Use supplied read time for selecting rows to backfill
- [ ] Establish connection when `yugabyte` role is password protected
- [ ] Handle errors anywhere in the schema migration process
- [ ] Handle multiple indexes backfilling at same time (issue yugabyte#4785)
- [ ] Have postgres respect master to tserver RPC deadline
- [ ] Support create unique index (issue yugabyte#4899)
- [ ] Support nested DDL create index (issue yugabyte#4786)
- [ ] Work on multi-stage drop index

Implement it as follows:

1. Pass database name from master to tserver on `BackfillIndex` request
1. Link libpq to tablet in order to send libpq request from tserver
1. Add `BACKFILL INDEX <index_oids> READ TIME <read_time> PARTITION
   <partition_key> [ FROM <row_key_start> [ TO <row_key_end> ] ]`
   grammar
1. Wire it down a similar path as `index_build`, but pass down read time
   and partition key (don't handle row keys yet) through exec params
1. Pass down hard-coded ancient write time
1. Read from indexed table tablet with specified partition key with
   specified read time
1. Non-transactionally write to index table with specified write time

For now, explicitly error on unique index creation and nested DDL index
creation because they are unstable.  They can later be enabled and wired
to use the fast path (no multi-stage).  Eventually, after some work, we
want to enable them with backfill (multi-stage).

Also, remove support for collecting `reltuples` stats on indexes when
using backfill.  We don't really use this stat, and we don't even
collect it for non-index tables, so it shouldn't be a big deal for now.

This is part 4 of the effort of bringing index backfill to YSQL.

Keep yugabyte#2301 open.

Depends on D8368

Depends on D8578

Test Plan:

`./yb_build.sh --cxx-test pgwrapper_pg_libpq-test --gtest_filter
'PgLibPqTest.Backfill*'`

Reviewers: amitanand, neil, mihnea

Reviewed By: mihnea

Subscribers: yql, bogdan

Differential Revision: https://phabricator.dev.yugabyte.com/D8487

CMakeLists.txt

@@ -1685,6 +1685,8 @@ endif()
 
 message("Linker flags: ${CMAKE_EXE_LINKER_FLAGS}")
 
+set(PQ_SHARED_LIB
+    "${YB_BUILD_ROOT}/postgres/lib/libpq${YB_SHARED_LIBRARY_SUFFIX}")
 set(YB_PGBACKEND_SHARED_LIB 
     "${YB_BUILD_ROOT}/postgres/lib/libyb_pgbackend${YB_SHARED_LIBRARY_SUFFIX}")
 

src/postgres/CMakeLists.txt

@@ -44,7 +44,8 @@ set(build_postgres_args
 # its own flags.
 add_custom_target(configure_postgres ALL COMMAND ${build_postgres_args} --step configure)
 add_custom_target(postgres ALL COMMAND ${build_postgres_args} --step make
-                           BYPRODUCTS "${YB_PGBACKEND_SHARED_LIB}")
+                           BYPRODUCTS "${YB_PGBACKEND_SHARED_LIB}"
+                                      "${PQ_SHARED_LIB}")
 add_dependencies(postgres configure_postgres)
 
 # ------------------------------------------------------------------------------------------------

src/postgres/src/backend/access/ybc/ybcin.c

@@ -41,8 +41,9 @@
 /* Working state for ybcinbuild and its callback */
 typedef struct
 {
-	bool	isprimary;
-	double	index_tuples;
+	bool	isprimary;		/* are we building a primary index? */
+	double	index_tuples;	/* # of tuples inserted into index */
+	bool	is_backfill;	/* are we concurrently backfilling an index? */
 } YBCBuildState;
 
 /*
@@ -93,6 +94,7 @@ ybcinhandler(PG_FUNCTION_ARGS)
 	amroutine->amparallelrescan = NULL;
 	amroutine->yb_aminsert = ybcininsert;
 	amroutine->yb_amdelete = ybcindelete;
+	amroutine->yb_ambackfill = ybcinbackfill;
 
 	PG_RETURN_POINTER(amroutine);
 }
@@ -104,7 +106,11 @@ ybcinbuildCallback(Relation index, HeapTuple heapTuple, Datum *values, bool *isn
 	YBCBuildState  *buildstate = (YBCBuildState *)state;
 
 	if (!buildstate->isprimary)
-		YBCExecuteInsertIndex(index, values, isnull, heapTuple->t_ybctid);
+		YBCExecuteInsertIndex(index,
+							  values,
+							  isnull,
+							  heapTuple->t_ybctid,
+							  buildstate->is_backfill);
 
 	buildstate->index_tuples += 1;
 }
@@ -118,6 +124,7 @@ ybcinbuild(Relation heap, Relation index, struct IndexInfo *indexInfo)
 	/* Do the heap scan */
 	buildstate.isprimary = index->rd_index->indisprimary;
 	buildstate.index_tuples = 0;
+	buildstate.is_backfill = false;
 	heap_tuples = IndexBuildHeapScan(heap, index, indexInfo, true, ybcinbuildCallback,
 									 &buildstate, NULL);
 
@@ -130,6 +137,37 @@ ybcinbuild(Relation heap, Relation index, struct IndexInfo *indexInfo)
 	return result;
 }
 
+IndexBuildResult *
+ybcinbackfill(Relation heap,
+			  Relation index,
+			  struct IndexInfo *indexInfo,
+			  uint64_t *read_time,
+			  RowBounds *row_bounds)
+{
+	YBCBuildState	buildstate;
+	double			heap_tuples = 0;
+
+	/* Do the heap scan */
+	buildstate.isprimary = index->rd_index->indisprimary;
+	buildstate.index_tuples = 0;
+	buildstate.is_backfill = true;
+	heap_tuples = IndexBackfillHeapRangeScan(heap,
+											 index,
+											 indexInfo,
+											 ybcinbuildCallback,
+											 &buildstate,
+											 read_time,
+											 row_bounds);
+
+	/*
+	 * Return statistics
+	 */
+	IndexBuildResult *result = (IndexBuildResult *) palloc(sizeof(IndexBuildResult));
+	result->heap_tuples  = heap_tuples;
+	result->index_tuples = buildstate.index_tuples;
+	return result;
+}
+
 void
 ybcinbuildempty(Relation index)
 {
@@ -141,7 +179,11 @@ ybcininsert(Relation index, Datum *values, bool *isnull, Datum ybctid, Relation
 			IndexUniqueCheck checkUnique, struct IndexInfo *indexInfo)
 {
 	if (!index->rd_index->indisprimary)
-		YBCExecuteInsertIndex(index, values, isnull, ybctid);
+		YBCExecuteInsertIndex(index,
+							  values,
+							  isnull,
+							  ybctid,
+							  false /* is_backfill */);
 
 	return index->rd_index->indisunique ? true : false;
 }

src/postgres/src/backend/catalog/index.c

@@ -131,6 +131,18 @@ static void UpdateIndexRelation(Oid indexoid, Oid heapoid,
 static void index_update_stats(Relation rel,
 				   bool hasindex,
 				   double reltuples);
+static double IndexBuildHeapRangeScanInternal(Relation heapRelation,
+											  Relation indexRelation,
+											  IndexInfo *indexInfo,
+											  bool allow_sync,
+											  bool anyvisible,
+											  BlockNumber start_blockno,
+											  BlockNumber numblocks,
+											  IndexBuildCallback callback,
+											  void *callback_state,
+											  HeapScanDesc scan,
+											  uint64_t *read_time,
+											  RowBounds *row_bounds);
 static void IndexCheckExclusion(Relation heapRelation,
 					Relation indexRelation,
 					IndexInfo *indexInfo);
@@ -2453,6 +2465,88 @@ index_build(Relation heapRelation,
 	SetUserIdAndSecContext(save_userid, save_sec_context);
 }
 
+/*
+ * index_backfill - invoke access-method-specific index backfill procedure
+ *
+ * This is mainly a copy of index_build.  index_build is used for
+ * non-multi-stage index creation; index_backfill is used for multi-stage index
+ * creation.
+ */
+void
+index_backfill(Relation heapRelation,
+			   Relation indexRelation,
+			   IndexInfo *indexInfo,
+			   bool isprimary,
+			   uint64_t *read_time,
+			   RowBounds *row_bounds)
+{
+	IndexBuildResult *stats;
+	Oid			save_userid;
+	int			save_sec_context;
+	int			save_nestlevel;
+
+	/*
+	 * sanity checks
+	 */
+	Assert(RelationIsValid(indexRelation));
+	Assert(PointerIsValid(indexRelation->rd_amroutine));
+	Assert(PointerIsValid(indexRelation->rd_amroutine->yb_ambackfill));
+
+	ereport(DEBUG1,
+			(errmsg("backfilling index \"%s\" on table \"%s\"",
+					RelationGetRelationName(indexRelation),
+					RelationGetRelationName(heapRelation))));
+
+	/*
+	 * Switch to the table owner's userid, so that any index functions are run
+	 * as that user.  Also lock down security-restricted operations and
+	 * arrange to make GUC variable changes local to this command.
+	 */
+	GetUserIdAndSecContext(&save_userid, &save_sec_context);
+	SetUserIdAndSecContext(heapRelation->rd_rel->relowner,
+						   save_sec_context | SECURITY_RESTRICTED_OPERATION);
+	save_nestlevel = NewGUCNestLevel();
+
+	/*
+	 * Call the access method's build procedure
+	 */
+	stats = indexRelation->rd_amroutine->yb_ambackfill(heapRelation,
+													   indexRelation,
+													   indexInfo,
+													   read_time,
+													   row_bounds);
+	Assert(PointerIsValid(stats));
+
+	/*
+	 * I don't think we should be backfilling unlogged indexes.
+	 */
+	Assert(indexRelation->rd_rel->relpersistence != RELPERSISTENCE_UNLOGGED);
+
+	/*
+	 * Update heap and index pg_class rows
+	 * TODO(jason): properly update reltuples.  They can't be set here because
+	 * this backfill func is called for each backfill chunk request from
+	 * master, and we need some way to sum up the tuple numbers.  We also don't
+	 * even collect stats properly for heapRelation anyway, at the moment.
+	 */
+	index_update_stats(heapRelation,
+					   true,
+					   -1);
+
+	index_update_stats(indexRelation,
+					   false,
+					   -1);
+
+	/* Make the updated catalog row versions visible */
+	CommandCounterIncrement();
+
+	/* Roll back any GUC changes executed by index functions */
+	AtEOXact_GUC(false, save_nestlevel);
+
+	/* Restore userid and security context */
+	SetUserIdAndSecContext(save_userid, save_sec_context);
+}
+
 
 /*
  * IndexBuildHeapScan - scan the heap relation to find tuples to be indexed
@@ -2493,16 +2587,6 @@ IndexBuildHeapScan(Relation heapRelation,
 								   callback, callback_state, scan);
 }
 
-/*
- * As above, except that instead of scanning the complete heap, only the given
- * number of blocks are scanned.  Scan to end-of-rel can be signalled by
- * passing InvalidBlockNumber as numblocks.  Note that restricting the range
- * to scan cannot be done when requesting syncscan.
- *
- * When "anyvisible" mode is requested, all tuples visible to any transaction
- * are indexed and counted as live, including those inserted or deleted by
- * transactions that are still in progress.
- */
 double
 IndexBuildHeapRangeScan(Relation heapRelation,
 						Relation indexRelation,
@@ -2514,6 +2598,67 @@ IndexBuildHeapRangeScan(Relation heapRelation,
 						IndexBuildCallback callback,
 						void *callback_state,
 						HeapScanDesc scan)
+{
+	return IndexBuildHeapRangeScanInternal(heapRelation,
+										   indexRelation,
+										   indexInfo,
+										   allow_sync,
+										   anyvisible,
+										   start_blockno,
+										   numblocks,
+										   callback,
+										   callback_state,
+										   scan,
+										   NULL /* read_time */,
+										   NULL /* row_bounds */);
+}
+
+double
+IndexBackfillHeapRangeScan(Relation heapRelation,
+						   Relation indexRelation,
+						   IndexInfo *indexInfo,
+						   IndexBuildCallback callback,
+						   void *callback_state,
+						   uint64_t *read_time,
+						   RowBounds *row_bounds)
+{
+	return IndexBuildHeapRangeScanInternal(heapRelation,
+										   indexRelation,
+										   indexInfo,
+										   true /* allow_sync */,
+										   false /* any_visible */,
+										   0 /* start_blockno */,
+										   InvalidBlockNumber /* num_blocks */,
+										   callback,
+										   callback_state,
+										   NULL /* scan */,
+										   read_time,
+										   row_bounds);
+}
+
+/*
+ * As above, except that instead of scanning the complete heap, only the given
+ * number of blocks are scanned.  Scan to end-of-rel can be signalled by
+ * passing InvalidBlockNumber as numblocks.  Note that restricting the range
+ * to scan cannot be done when requesting syncscan.
+ *
+ * When "anyvisible" mode is requested, all tuples visible to any transaction
+ * are indexed and counted as live, including those inserted or deleted by
+ * transactions that are still in progress.
+ */
+static double
+IndexBuildHeapRangeScanInternal(Relation heapRelation,
+								Relation indexRelation,
+								IndexInfo *indexInfo,
+								bool allow_sync,
+								bool anyvisible,
+								BlockNumber start_blockno,
+								BlockNumber numblocks,
+								IndexBuildCallback callback,
+								void *callback_state,
+								HeapScanDesc scan,
+								uint64_t *read_time,
+								RowBounds *row_bounds)
 {
 	bool		is_system_catalog;
 	bool		checking_uniqueness;
@@ -2557,6 +2702,15 @@ IndexBuildHeapRangeScan(Relation heapRelation,
 	econtext = GetPerTupleExprContext(estate);
 	slot = MakeSingleTupleTableSlot(RelationGetDescr(heapRelation));
 
+	/*
+	 * Set some exec params.
+	 */
+	YBCPgExecParameters *exec_params = &estate->yb_exec_params;
+	if (read_time)
+		exec_params->read_time = *read_time;
+	if (row_bounds)
+		exec_params->partition_key = pstrdup(row_bounds->partition_key);
+
 	/* Arrange for econtext's scan tuple to be the tuple under test */
 	econtext->ecxt_scantuple = slot;
 
@@ -2598,6 +2752,8 @@ IndexBuildHeapRangeScan(Relation heapRelation,
 									NULL,	/* scan key */
 									true,	/* buffer access strategy OK */
 									allow_sync);	/* syncscan OK? */
+		if (IsYBRelation(heapRelation))
+			scan->ybscan->exec_params = exec_params;
 	}
 	else
 	{
@@ -2616,8 +2772,8 @@ IndexBuildHeapRangeScan(Relation heapRelation,
 	/*
 	 * Must call GetOldestXmin() with SnapshotAny.  Should never call
 	 * GetOldestXmin() with MVCC snapshot. (It's especially worth checking
-	 * this for parallel builds, since ambuild routines that support parallel
-	 * builds must work these details out for themselves.)
+	 * this for parallel builds, since yb_ambackfill routines that support
+	 * parallel builds must work these details out for themselves.)
 	 */
 	Assert(snapshot == SnapshotAny || IsMVCCSnapshot(snapshot));
 	Assert(snapshot == SnapshotAny ? TransactionIdIsValid(OldestXmin) :