Skip to content

Commit

Permalink
Simplify CachingHiveMetastore constructor
Browse files Browse the repository at this point in the history
  • Loading branch information
sopel39 committed Dec 22, 2022
1 parent e1be8ce commit 437a677
Showing 1 changed file with 46 additions and 38 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -151,53 +151,31 @@ protected CachingHiveMetastore(HiveMetastore delegate, OptionalLong expiresAfter
this.delegate = requireNonNull(delegate, "delegate is null");
requireNonNull(executor, "executor is null");

databaseNamesCache = buildCache(expiresAfterWriteMillis, refreshMills, executor, maximumSize, statsRecording, ignored -> loadAllDatabases());

databaseCache = buildCache(expiresAfterWriteMillis, refreshMills, executor, maximumSize, statsRecording, this::loadDatabase);

tableNamesCache = buildCache(expiresAfterWriteMillis, refreshMills, executor, maximumSize, statsRecording, this::loadAllTables);

tablesWithParameterCache = buildCache(expiresAfterWriteMillis, refreshMills, executor, maximumSize, statsRecording, this::loadTablesMatchingParameter);

tableStatisticsCache = buildCache(expiresAfterWriteMillis, refreshMills, executor, maximumSize, statsRecording, this::loadTableColumnStatistics);
CacheFactory cacheFactory = cacheFactory(expiresAfterWriteMillis, refreshMills, executor, maximumSize, statsRecording);
databaseNamesCache = cacheFactory.buildCache(ignored -> loadAllDatabases());
databaseCache = cacheFactory.buildCache(this::loadDatabase);
tableNamesCache = cacheFactory.buildCache(this::loadAllTables);
tablesWithParameterCache = cacheFactory.buildCache(this::loadTablesMatchingParameter);
tableStatisticsCache = cacheFactory.buildCache(this::loadTableColumnStatistics);
tableCache = cacheFactory.buildCache(this::loadTable);
viewNamesCache = cacheFactory.buildCache(this::loadAllViews);
tablePrivilegesCache = cacheFactory.buildCache(key -> loadTablePrivileges(key.getDatabase(), key.getTable(), key.getOwner(), key.getPrincipal()));
rolesCache = cacheFactory.buildCache(ignored -> loadRoles());
roleGrantsCache = cacheFactory.buildCache(this::loadRoleGrants);
grantedPrincipalsCache = cacheFactory.buildCache(this::loadPrincipals);
configValuesCache = cacheFactory.buildCache(this::loadConfigValue);

if (partitionCacheEnabled) {
// disable refresh since it can't use the bulk loading and causes too many requests
partitionStatisticsCache = buildCache(expiresAfterWriteMillis, maximumSize, statsRecording, this::loadPartitionColumnStatistics, this::loadPartitionsColumnStatistics);
partitionStatisticsCache = cacheFactory.buildCache(this::loadPartitionColumnStatistics, this::loadPartitionsColumnStatistics);
partitionFilterCache = cacheFactory.buildCache(this::loadPartitionNamesByFilter);
partitionCache = cacheFactory.buildCache(this::loadPartitionByName, this::loadPartitionsByNames);
}
else {
partitionStatisticsCache = neverCache(this::loadPartitionColumnStatistics, this::loadPartitionsColumnStatistics);
}

tableCache = buildCache(expiresAfterWriteMillis, refreshMills, executor, maximumSize, statsRecording, this::loadTable);

viewNamesCache = buildCache(expiresAfterWriteMillis, refreshMills, executor, maximumSize, statsRecording, this::loadAllViews);

if (partitionCacheEnabled) {
partitionFilterCache = buildCache(expiresAfterWriteMillis, refreshMills, executor, maximumSize, statsRecording, this::loadPartitionNamesByFilter);
}
else {
partitionFilterCache = neverCache(this::loadPartitionNamesByFilter);
}

if (partitionCacheEnabled) {
// disable refresh since it can't use the bulk loading and causes too many requests
partitionCache = buildCache(expiresAfterWriteMillis, maximumSize, statsRecording, this::loadPartitionByName, this::loadPartitionsByNames);
}
else {
partitionCache = neverCache(this::loadPartitionByName, this::loadPartitionsByNames);
}

tablePrivilegesCache = buildCache(expiresAfterWriteMillis, refreshMills, executor, maximumSize, statsRecording, key ->
loadTablePrivileges(key.getDatabase(), key.getTable(), key.getOwner(), key.getPrincipal()));

rolesCache = buildCache(expiresAfterWriteMillis, refreshMills, executor, maximumSize, statsRecording, ignored -> loadRoles());

roleGrantsCache = buildCache(expiresAfterWriteMillis, refreshMills, executor, maximumSize, statsRecording, this::loadRoleGrants);

grantedPrincipalsCache = buildCache(expiresAfterWriteMillis, refreshMills, executor, maximumSize, statsRecording, this::loadPrincipals);

configValuesCache = buildCache(expiresAfterWriteMillis, refreshMills, executor, maximumSize, statsRecording, this::loadConfigValue);
}

private static <K, V> LoadingCache<K, V> neverCache(com.google.common.base.Function<K, V> loader)
Expand Down Expand Up @@ -1032,6 +1010,36 @@ public void alterTransactionalTable(Table table, long transactionId, long writeI
}
}

private interface CacheFactory
{
<K, V> LoadingCache<K, V> buildCache(com.google.common.base.Function<K, V> loader);

<K, V> LoadingCache<K, V> buildCache(com.google.common.base.Function<K, V> loader, Function<Iterable<K>, Map<K, V>> bulkLoader);
}

private static CacheFactory cacheFactory(
OptionalLong expiresAfterWriteMillis,
OptionalLong refreshMillis,
Optional<Executor> refreshExecutor,
long maximumSize,
StatsRecording statsRecording)
{
return new CacheFactory()
{
@Override
public <K, V> LoadingCache<K, V> buildCache(com.google.common.base.Function<K, V> loader)
{
return CachingHiveMetastore.buildCache(expiresAfterWriteMillis, refreshMillis, refreshExecutor, maximumSize, statsRecording, loader);
}

@Override
public <K, V> LoadingCache<K, V> buildCache(com.google.common.base.Function<K, V> loader, Function<Iterable<K>, Map<K, V>> bulkLoader)
{
return CachingHiveMetastore.buildCache(expiresAfterWriteMillis, maximumSize, statsRecording, loader, bulkLoader);
}
};
}

private static <K, V> LoadingCache<K, V> buildCache(
OptionalLong expiresAfterWriteMillis,
OptionalLong refreshMillis,
Expand Down

0 comments on commit 437a677

Please sign in to comment.