From a407be410cf5507954f30199029a912d09ade8a4 Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Fri, 12 Aug 2022 13:33:54 +0200 Subject: [PATCH 1/2] Improve test time for testReadMetadataWithRelationsConcurrentModifications in Iceberg Profiling of `testReadMetadataWithRelationsConcurrentModifications` with `TestIcebergParquetConnectorTest` shows that most time is spent listing metadata, and of which, most time is spending listing tables in the TESTING_FILE_METASTORE. This commit improves this by adding simple listing cache on top of the file system. --- .../metastore/file/FileHiveMetastore.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java index 42a36909dcfc..e8543f2a65a3 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java @@ -14,6 +14,8 @@ package io.trino.plugin.hive.metastore.file; import com.google.common.annotations.VisibleForTesting; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -21,6 +23,7 @@ import com.google.common.collect.Sets; import com.google.common.io.ByteStreams; import io.airlift.json.JsonCodec; +import io.trino.collect.cache.EvictableCacheBuilder; import io.trino.plugin.hive.HdfsConfig; import io.trino.plugin.hive.HdfsConfiguration; import io.trino.plugin.hive.HdfsConfigurationInitializer; @@ -120,6 +123,7 @@ import static io.trino.spi.security.PrincipalType.USER; import static java.lang.String.format; import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.SECONDS; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toSet; import static org.apache.hadoop.hive.common.FileUtils.unescapePathName; @@ -156,6 +160,8 @@ public class FileHiveMetastore private final JsonCodec> rolesCodec = JsonCodec.listJsonCodec(String.class); private final JsonCodec> roleGrantsCodec = JsonCodec.listJsonCodec(RoleGrant.class); + private final LoadingCache> listTablesCache; + @VisibleForTesting public static FileHiveMetastore createTestingFileHiveMetastore(File catalogDirectory) { @@ -186,6 +192,10 @@ public FileHiveMetastore(NodeVersion nodeVersion, HdfsEnvironment hdfsEnvironmen catch (IOException e) { throw new TrinoException(HIVE_METASTORE_ERROR, e); } + + listTablesCache = EvictableCacheBuilder.newBuilder() + .expireAfterWrite(10, SECONDS) + .build(CacheLoader.from(this::doListAllTables)); } @Override @@ -499,6 +509,12 @@ public synchronized List getTablesWithParameter(String databaseName, Str @GuardedBy("this") private List listAllTables(String databaseName) + { + return listTablesCache.getUnchecked(databaseName); + } + + @GuardedBy("this") + private List doListAllTables(String databaseName) { requireNonNull(databaseName, "databaseName is null"); @@ -1333,6 +1349,8 @@ private void writeFile(String type, Path path, JsonCodec codec, T value, catch (Exception e) { throw new TrinoException(HIVE_METASTORE_ERROR, "Could not write " + type, e); } + + listTablesCache.invalidateAll(); } private void renameSchemaFile(SchemaType type, Path oldMetadataDirectory, Path newMetadataDirectory) @@ -1345,6 +1363,8 @@ private void renameSchemaFile(SchemaType type, Path oldMetadataDirectory, Path n catch (IOException e) { throw new TrinoException(HIVE_METASTORE_ERROR, "Could not rename " + type + " schema", e); } + + listTablesCache.invalidateAll(); } private void deleteSchemaFile(SchemaType type, Path metadataDirectory) @@ -1357,6 +1377,8 @@ private void deleteSchemaFile(SchemaType type, Path metadataDirectory) catch (IOException e) { throw new TrinoException(HIVE_METASTORE_ERROR, "Could not delete " + type + " schema", e); } + + listTablesCache.invalidateAll(); } private Path getDatabaseMetadataDirectory(String databaseName) From f8c9cbdc41eb280e6f14a11932be703698e33b45 Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Fri, 12 Aug 2022 18:51:27 +0200 Subject: [PATCH 2/2] empty