From 1af6779cd6ad0ccd47ad6ac11c18884436bf8642 Mon Sep 17 00:00:00 2001 From: ashish-kumar-sharma Date: Tue, 6 Apr 2021 18:07:44 +0530 Subject: [PATCH] HIVE-23820: Add all function signature changes to accept tableId --- .../hive/metastore/HiveMetaStoreClient.java | 12 ++++++++ .../hive/metastore/IMetaStoreClient.java | 28 +++++++++++++++++++ .../hadoop/hive/metastore/HMSHandler.java | 14 +++++----- .../hadoop/hive/metastore/ObjectStore.java | 6 ++++ .../hadoop/hive/metastore/RawStore.java | 13 +++++++++ .../hive/metastore/cache/CachedStore.java | 6 +++- .../DummyRawStoreControlledCommit.java | 6 ++++ .../DummyRawStoreForJdoConnection.java | 6 ++++ .../HiveMetaStoreClientPreCatalog.java | 12 ++++++++ 9 files changed, 95 insertions(+), 8 deletions(-) diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 40a82af89dec5..2056e0e465df8 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -2367,10 +2367,21 @@ public Table getTable(String catName, String dbName, String tableName, return getTable(catName, dbName, tableName, validWriteIdList, false, null); } + @Override + public Table getTable(String catName, String dbName, String tableName, + String validWriteIdList, long tableId) throws TException { + return getTable(catName, dbName, tableName, validWriteIdList, false, null, tableId); + } @Override public Table getTable(String catName, String dbName, String tableName, String validWriteIdList, boolean getColumnStats, String engine) throws TException { + return getTable(catName, dbName, tableName, validWriteIdList, false, null, -1); + } + + @Override + public Table getTable(String catName, String dbName, String tableName, String validWriteIdList, + boolean getColumnStats, String engine, long tableId) throws TException { long t1 = System.currentTimeMillis(); try { @@ -2379,6 +2390,7 @@ public Table getTable(String catName, String dbName, String tableName, String va req.setCapabilities(version); req.setValidWriteIdList(validWriteIdList); req.setGetColumnStats(getColumnStats); + req.setId(tableId); if (getColumnStats) { req.setEngine(engine); } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java index 6b9be3dda04fd..85e6f1661fd01 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java @@ -696,6 +696,19 @@ Table getTable(String dbName, String tableName, boolean getColumnStats, String e Table getTable(String catName, String dbName, String tableName, String validWriteIdList) throws TException; + /** + * Get a table object. + * @param catName catalog the table is in. + * @param dbName database the table is in. + * @param tableName table name. + * @param validWriteIdList applicable snapshot + * @return table object. + * @throws MetaException Something went wrong, usually in the RDBMS. + * @throws TException general thrift error. + */ + Table getTable(String catName, String dbName, String tableName, + String validWriteIdList, long tableId) throws TException; + /** * Get a table object. * @param catName catalog the table is in. @@ -711,6 +724,21 @@ Table getTable(String catName, String dbName, String tableName, Table getTable(String catName, String dbName, String tableName, String validWriteIdList, boolean getColumnStats, String engine) throws TException; + /** + * Get a table object. + * @param catName catalog the table is in. + * @param dbName database the table is in. + * @param tableName table name. + * @param validWriteIdList applicable snapshot + * @param getColumnStats get the column stats, if available, when true + * @param engine engine sending the request + * @return table object. + * @throws MetaException Something went wrong, usually in the RDBMS. + * @throws TException general thrift error. + */ + Table getTable(String catName, String dbName, String tableName, + String validWriteIdList, boolean getColumnStats, String engine, long tableId) throws TException; + /** * Get tables as objects (rather than just fetching their names). This is more expensive and * should only be used if you actually need all the information about the tables. diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java index f6754528361da..57dc49c034179 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java @@ -3334,7 +3334,7 @@ public Table get_table(final String dbname, final String name) throws MetaExcept NoSuchObjectException { String[] parsedDbName = parseDbName(dbname, conf); return getTableInternal( - parsedDbName[CAT_NAME], parsedDbName[DB_NAME], name, null, null, false, null, null, null); + parsedDbName[CAT_NAME], parsedDbName[DB_NAME], name, null, null, false, null, null, null, -1); } @Override @@ -3412,7 +3412,7 @@ public GetTableResult get_table_req(GetTableRequest req) throws MetaException, String catName = req.isSetCatName() ? req.getCatName() : getDefaultCatalog(conf); return new GetTableResult(getTableInternal(catName, req.getDbName(), req.getTblName(), req.getCapabilities(), req.getValidWriteIdList(), req.isGetColumnStats(), req.getEngine(), - req.getProcessorCapabilities(), req.getProcessorIdentifier())); + req.getProcessorCapabilities(), req.getProcessorIdentifier(), req.getId())); } /** @@ -3422,7 +3422,7 @@ public GetTableResult get_table_req(GetTableRequest req) throws MetaException, */ private Table getTableInternal(String catName, String dbname, String name, ClientCapabilities capabilities, String writeIdList, boolean getColumnStats, String engine, - List processorCapabilities, String processorId) + List processorCapabilities, String processorId, long tableId) throws MetaException, NoSuchObjectException { Preconditions.checkArgument(!getColumnStats || engine != null, "To retrieve column statistics with a table, engine parameter cannot be null"); @@ -3436,7 +3436,7 @@ private Table getTableInternal(String catName, String dbname, String name, startTableFunction("get_table", catName, dbname, name); Exception ex = null; try { - t = get_table_core(catName, dbname, name, writeIdList, getColumnStats, engine); + t = get_table_core(catName, dbname, name, writeIdList, getColumnStats, engine, tableId); if (MetaStoreUtils.isInsertOnlyTableParam(t.getParameters())) { assertClientHasCapability(capabilities, ClientCapability.INSERT_ONLY_TABLES, "insert-only tables", "get_table_req"); @@ -3504,7 +3504,7 @@ public Table get_table_core( final String name, final String writeIdList) throws MetaException, NoSuchObjectException { - return get_table_core(catName, dbname, name, writeIdList, false, null); + return get_table_core(catName, dbname, name, writeIdList, false, null, -1); } /** @@ -3516,14 +3516,14 @@ public Table get_table_core(final String catName, final String dbname, final String name, final String writeIdList, - boolean getColumnStats, String engine) + boolean getColumnStats, String engine, long tableId) throws MetaException, NoSuchObjectException { Preconditions.checkArgument(!getColumnStats || engine != null, "To retrieve column statistics with a table, engine parameter cannot be null"); Table t = null; try { - t = getMS().getTable(catName, dbname, name, writeIdList); + t = getMS().getTable(catName, dbname, name, writeIdList, tableId); if (t == null) { throw new NoSuchObjectException(TableName.getQualified(catName, dbname, name) + " table not found"); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index d6653fe377ec7..ab2f034efccff 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -1419,6 +1419,12 @@ public Table getTable(String catName, String dbName, String tableName, String wr return tbl; } + @Override + public Table getTable(String catalogName, String dbName, String tableName, String writeIdList, long tableId) + throws MetaException { + return getTable( catalogName, dbName, tableName, writeIdList); + } + @Override public List getTables(String catName, String dbName, String pattern) throws MetaException { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index 37ea268c7d547..75ba29a875586 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -295,6 +295,19 @@ boolean dropTable(String catalogName, String dbName, String tableName) Table getTable(String catalogName, String dbName, String tableName, String writeIdList) throws MetaException; + /** + * Get a table object. + * @param catalogName catalog the table is in. + * @param dbName database the table is in. + * @param tableName table name. + * @param writeIdList string format of valid writeId transaction list + * @return table object, or null if no such table exists (wow it would be nice if we either + * consistently returned null or consistently threw NoSuchObjectException). + * @throws MetaException something went wrong in the RDBMS + */ + Table getTable(String catalogName, String dbName, String tableName, + String writeIdList, long tableId) throws MetaException; + /** * Add a partition. * @param part partition to add diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 7cdff16a0232d..8b0dffd071f97 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -1227,7 +1227,11 @@ private void validateTableType(Table tbl) { return getTable(catName, dbName, tblName, null); } - @Override public Table getTable(String catName, String dbName, String tblName, String validWriteIds) + @Override public Table getTable(String catName, String dbName, String tblName, String validWriteIds) throws MetaException { + return getTable(catName, dbName, tblName, null, -1); + } + + @Override public Table getTable(String catName, String dbName, String tblName, String validWriteIds, long tableId) throws MetaException { catName = normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index d6458a690e734..970b572502194 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -271,6 +271,12 @@ public Table getTable(String catName, String dbName, String tableName, String wr return objectStore.getTable(catName, dbName, tableName, writeIdList); } + @Override + public Table getTable(String catalogName, String dbName, String tableName, String writeIdList, long tableId) + throws MetaException { + return objectStore.getTable(catalogName, dbName, tableName, writeIdList, tableId); + } + @Override public boolean addPartition(Partition part) throws InvalidObjectException, MetaException { diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index 0ae294ee43ecd..ac2b306ba5475 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -267,6 +267,12 @@ public Table getTable(String catalogName, String dbName, String tableName, return null; } + @Override + public Table getTable(String catalogName, String dbName, String tableName, String writeIdList, long tableId) + throws MetaException { + return null; + } + @Override public boolean addPartition(Partition part) throws InvalidObjectException, MetaException { diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index d571b93dca687..7ab6239b2ca1d 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -3260,12 +3260,24 @@ public Table getTable(String catName, String dbName, String tableName, throw new UnsupportedOperationException(); } + @Override + public Table getTable(String catName, String dbName, String tableName, String validWriteIdList, long tableId) + throws TException { + throw new UnsupportedOperationException(); + } + @Override public Table getTable(String catName, String dbName, String tableName, String validWriteIdList, boolean getColumnStats, String engine) throws TException { throw new UnsupportedOperationException(); } + @Override + public Table getTable(String catName, String dbName, String tableName, String validWriteIdList, + boolean getColumnStats, String engine, long tableId) throws TException { + throw new UnsupportedOperationException(); + } + @Override public List getTableObjectsByName(String catName, String dbName, List tableNames) throws MetaException,