[#10509][#10510] YSQL: Adding support for CREATE, DROP and REFRESH MA…

…TERIALIZED VIEW

Summary:
`CREATE MATERIALIZED VIEW`:
- Created like any other YB relation.

`REFRESH MATERIALIZED VIEW`:

- In vanilla PG, a non-concurrent refresh on the materialized view entails the following:
   #  Load the new matview data into a transient relation.
   #  Refresh the matview by swapping it with the transient relation. The swap is done by swapping the `relfilenode` of the two relations. (Therefore the `relfilenode` of the matview is now the `OID` of the transient relation).
   #  Reindex the new matview.

- In YB mode we do the following:
  # Load the new matview data into a transient relation, and swap the `relfilenode` (as in vanilla PG).
  # We implement a `YBGetStorageRelid` utility function that returns, if valid, the `relfilenode` (which will be different from the `OID` if there has been a swap) or the `OID` itself.
  # Any subsequent scans/drops on the matview will now use `YBGetStorageRelid` to retrieve the `OID` of the appropriate relation in DocDB.
  # We reindex the matview by dropping and recreating the matview's indexes.

`REFRESH MATERIALIZED VIEW CONCURRENTLY`:

- In PG, a concurrent refresh entails the following:
  # Ensure there's a unique index on the matview.
  # Load the new data into a temporary table.
  # Ensure there are no duplicate rows without any null values (i.e., duplicate rows are only allowed when at least one column value is null).
  # Create a temporary diff table which tells us what rows are present in the old data and not in the new data and vice versa.
       - The diff table is computed as a full join of the old matview and the new temporary table on the unique index.
  # Delete rows in the matview where row.ctid = any old matview ctid in diff where the newdata is null. (i,e,, if the join did not find a match for this row amidst the new data, this row has been deleted).
  # Insert rows from the new data in the diff table where old matview ctid is null. (i,e., if the join did not find a match amidst the old data, this row has been newly inserted).

- In YB mode:
  # We do essentially the same thing, except that since YB doesn't have ctids we use all the column values instead.
  # Additionally, since there aren't any ctids, if the new data has rows with **all** null columns, the joined diff table will have duplicate entries for such rows.
  # Therefore, in YB mode, we disallow rows with all null columns.

`DROP MATERIALIZED VIEW`:
- The syscatalog tuples are dropped like any other YB relation.
- `YBGetStorageRelid` is used to drop the appropriate relation in DocDB.

Test Plan:
Jenkins: urgent, test regex: .*TestPgRegressFeature.*|.*TestPgRegressExtension.*|.*TestPgRegressPartitions.*

Run the `yb_feature_matview` test using the `TestPgRegressFeature` test suite.

Reviewers: myang, mihnea

Reviewed By: myang, mihnea

Subscribers: bogdan, yql

Differential Revision: https://phabricator.dev.yugabyte.com/D13792

src/postgres/contrib/earthdistance/expected/yb_pg_earthdistance.out

@@ -1049,8 +1049,8 @@ DETAIL:  column f1 of table foo depends on type c.cube
 HINT:  Use DROP ... CASCADE to drop the dependent objects too.
 drop schema c;  -- fail, cube requires it
 ERROR:  cannot drop schema c because other objects depend on it
-DETAIL:  extension cube depends on schema c
-column f1 of table foo depends on type c.cube
+DETAIL:  column f1 of table foo depends on type c.cube
+extension cube depends on schema c
 HINT:  Use DROP ... CASCADE to drop the dependent objects too.
 drop extension cube cascade;
 NOTICE:  drop cascades to column f1 of table foo

src/postgres/src/backend/access/transam/xact.c

@@ -6078,14 +6078,23 @@ xact_redo(XLogReaderState *record)
 		elog(PANIC, "xact_redo: unknown op code %u", info);
 }
 
-void YBSaveDdlHandle(YBCPgStatement handle) {
+void YBSaveDdlHandle(YBCPgStatement handle)
+{
 	CurrentTransactionState->YBPostponedDdlOps = lappend(CurrentTransactionState->YBPostponedDdlOps, handle);
 }
 
-List* YBGetDdlHandles() {
+List* YBGetDdlHandles()
+{
 	return CurrentTransactionState->YBPostponedDdlOps;
 }
 
-void YBClearDdlHandles() {
+void YBClearDdlHandles()
+{
 	CurrentTransactionState->YBPostponedDdlOps = NULL;
 }
+
+void YbClearCurrentTransactionId()
+{
+	CurrentTransactionState->transactionId = InvalidTransactionId;
+	MyPgXact->xid = InvalidTransactionId;
+}

src/postgres/src/backend/access/yb_access/yb_scan.c

@@ -123,10 +123,9 @@ static void ybcCheckPrimaryKeyAttribute(YbScanPlan      scan_plan,
 static void ybcLoadTableInfo(Relation relation, YbScanPlan scan_plan)
 {
 	Oid            dboid          = YBCGetDatabaseOid(relation);
-	Oid            relid          = RelationGetRelid(relation);
 	YBCPgTableDesc ybc_table_desc = NULL;
 
-	HandleYBStatus(YBCPgGetTableDesc(dboid, relid, &ybc_table_desc));
+	HandleYBStatus(YBCPgGetTableDesc(dboid, YbGetStorageRelid(relation), &ybc_table_desc));
 
 	for (AttrNumber attnum = 1; attnum <= relation->rd_att->natts; attnum++)
 	{
@@ -450,7 +449,7 @@ ybcSetupScanPlan(bool xs_want_itup, YbScanDesc ybScan, YbScanPlan scan_plan)
 		bool colocated = false;
 		bool notfound;
 		HandleYBStatusIgnoreNotFound(YBCPgIsTableColocated(MyDatabaseId,
-														   RelationGetRelid(relation),
+														   YbGetStorageRelid(relation),
 														   &colocated),
 									 &notfound);
 		ybScan->prepare_params.querying_colocated_table |= colocated;
@@ -742,10 +741,9 @@ static void
 ybcBindScanKeys(YbScanDesc ybScan, YbScanPlan scan_plan) {
 	Relation relation = ybScan->relation;
 	Relation index = ybScan->index;
-	Oid		dboid    = YBCGetDatabaseOid(relation);
-	Oid		relid    = RelationGetRelid(relation);
+	Oid		 dboid = YBCGetDatabaseOid(relation);
 
-	HandleYBStatus(YBCPgNewSelect(dboid, relid, &ybScan->prepare_params, &ybScan->handle));
+	HandleYBStatus(YBCPgNewSelect(dboid, YbGetStorageRelid(relation), &ybScan->prepare_params, &ybScan->handle));
 
 	if (IsSystemRelation(relation))
 	{
@@ -1846,9 +1844,9 @@ HeapTuple YBCFetchTuple(Relation relation, Datum ybctid)
 	TupleDesc      tupdesc = RelationGetDescr(relation);
 
 	HandleYBStatus(YBCPgNewSelect(YBCGetDatabaseOid(relation),
-																RelationGetRelid(relation),
-																NULL /* prepare_params */,
-																&ybc_stmt));
+								  YbGetStorageRelid(relation),
+								  NULL /* prepare_params */,
+								  &ybc_stmt));
 
 	/* Bind ybctid to identify the current row. */
 	YBCPgExpr ybctid_expr = YBCNewConstant(ybc_stmt,

src/postgres/src/backend/access/ybgin/ybginscan.c

@@ -115,7 +115,7 @@ ybginrescan(IndexScanDesc scan, ScanKey scankey, int nscankeys,
 		.querying_colocated_table = isColocated(RelationGetRelid(scan->heapRelation)),
 	};
 	HandleYBStatus(YBCPgNewSelect(YBCGetDatabaseOid(scan->heapRelation),
-								  RelationGetRelid(scan->heapRelation),
+								  YbGetStorageRelid(scan->heapRelation),
 								  &prepare_params,
 								  &ybso->handle));
 

src/postgres/src/backend/catalog/dependency.c

@@ -969,6 +969,9 @@ reportDependentObjects(const ObjectAddresses *targetObjects,
 												 numNotReportedClient),
 						 numNotReportedClient);
 
+	if (IsYugaByteEnabled() && clientdetail.data != NULL)
+		clientdetail.data = YBDetailSorted(clientdetail.data);
+
 	if (!ok)
 	{
 		if (origObject)

src/postgres/src/backend/catalog/index.c

@@ -50,6 +50,7 @@
 #include "catalog/pg_type.h"
 #include "catalog/storage.h"
 #include "commands/tablecmds.h"
+#include "commands/defrem.h"
 #include "commands/event_trigger.h"
 #include "commands/trigger.h"
 #include "executor/executor.h"
@@ -59,6 +60,7 @@
 #include "optimizer/clauses.h"
 #include "optimizer/planner.h"
 #include "parser/parser.h"
+#include "parser/parse_utilcmd.h"
 #include "rewrite/rewriteManip.h"
 #include "storage/bufmgr.h"
 #include "storage/lmgr.h"
@@ -2413,7 +2415,7 @@ index_build(Relation heapRelation,
 	 * Call the access method's build procedure
 	 */
 	stats = indexRelation->rd_amroutine->ambuild(heapRelation, indexRelation,
-	                        indexInfo);
+							indexInfo);
 	Assert(PointerIsValid(stats));
 
 	/*
@@ -4267,11 +4269,24 @@ reindex_relation(Oid relid, int flags, int options)
 		{
 			Oid			indexOid = lfirst_oid(indexId);
 
-			if (is_pg_class)
-				RelationSetIndexList(rel, doneIndexes, InvalidOid);
+			if (IsYBRelationById(indexOid))
+			{
+				Relation new_rel = heap_open(YbGetStorageRelid(rel), AccessExclusiveLock);
+				AttrNumber *new_to_old_attmap = convert_tuples_by_name_map(RelationGetDescr(new_rel),
+											  	RelationGetDescr(rel),
+											  	gettext_noop("could not convert row type"));
+				heap_close(new_rel, AccessExclusiveLock);
+				YbDropAndRecreateIndex(indexOid, relid, rel, new_to_old_attmap);
+				RemoveReindexPending(indexOid);
+			}
+			else
+			{
+				if (is_pg_class)
+					RelationSetIndexList(rel, doneIndexes, InvalidOid);
 
-			reindex_index(indexOid, !(flags & REINDEX_REL_CHECK_CONSTRAINTS),
+				reindex_index(indexOid, !(flags & REINDEX_REL_CHECK_CONSTRAINTS),
 						  persistence, options);
+			}
 
 			CommandCounterIncrement();
 

src/postgres/src/backend/commands/cluster.c

@@ -36,7 +36,9 @@
 #include "commands/cluster.h"
 #include "commands/tablecmds.h"
 #include "commands/vacuum.h"
+#include "commands/ybccmds.h"
 #include "miscadmin.h"
+#include "nodes/makefuncs.h"
 #include "optimizer/planner.h"
 #include "storage/bufmgr.h"
 #include "storage/lmgr.h"
@@ -697,6 +699,13 @@ make_new_heap(Oid OIDOldHeap, Oid NewTableSpace, char relpersistence,
 										  false);
 	Assert(OIDNewHeap != InvalidOid);
 
+	if (IsYugaByteEnabled() && relpersistence != RELPERSISTENCE_TEMP)
+	{
+		CreateStmt *dummyStmt = makeNode(CreateStmt);
+		dummyStmt->relation = makeRangeVar(NULL, NewHeapName, -1);
+		YBCCreateTable(dummyStmt, RELKIND_RELATION, OldHeapDesc, OIDNewHeap, namespaceid, InvalidOid, NewTableSpace);
+	}
+
 	ReleaseSysCache(tuple);
 
 	/*
@@ -1227,6 +1236,19 @@ swap_relation_files(Oid r1, Oid r2, bool target_is_pg_class,
 		relform1->relpersistence = relform2->relpersistence;
 		relform2->relpersistence = swptmpchr;
 
+		if (IsYugaByteEnabled())
+		{
+			/* 
+			 * If this swap is happening during a REFRESH MATVIEW,
+			 * correctly mark the transient relation as a MATVIEW
+			 * so that it is dropped in YB mode.
+			 */
+			if (relform1->relkind == RELKIND_MATVIEW) 
+				relform2->relkind = RELKIND_MATVIEW;
+			else if (relform2->relkind == RELKIND_MATVIEW) 
+				relform1->relkind = RELKIND_MATVIEW;
+		}
+
 		/* Also swap toast links, if we're swapping by links */
 		if (!swap_toast_by_content)
 		{

src/postgres/src/backend/commands/indexcmds.c

@@ -419,7 +419,7 @@ DefineIndex(Oid relationId,
 		IsYBRelation(rel))
 	{
 		HandleYBStatus(YBCPgIsTableColocated(databaseId,
-											 relationId,
+											 YbGetStorageRelid(rel),
 											 &is_indexed_table_colocated));
 	}
 
@@ -1570,7 +1570,7 @@ ComputeIndexAttrs(IndexInfo *indexInfo,
 		use_yb_ordering = IsYBRelation(rel) && !IsSystemRelation(rel);
 		if (IsYBRelation(rel))
 			HandleYBStatus(YBCPgIsTableColocated(YBCGetDatabaseOid(rel),
-												 relId,
+												 YbGetStorageRelid(rel),
 												 &colocated));
 		RelationClose(rel);
 	}