diff --git a/tcmalloc/global_stats.cc b/tcmalloc/global_stats.cc index 1a68aecb9..dc0f3a413 100644 --- a/tcmalloc/global_stats.cc +++ b/tcmalloc/global_stats.cc @@ -58,6 +58,8 @@ void ExtractStats(TCMallocStats* r, uint64_t* class_count, for (int size_class = 0; size_class < kNumClasses; ++size_class) { const size_t length = tc_globals.central_freelist(size_class).length(); const size_t tc_length = tc_globals.transfer_cache().tc_length(size_class); + const size_t sharded_tc_length = + tc_globals.sharded_transfer_cache().TotalObjectsOfClass(size_class); const size_t cache_overhead = tc_globals.central_freelist(size_class).OverheadBytes(); const size_t size = tc_globals.sizemap().class_to_size(size_class); @@ -66,7 +68,7 @@ void ExtractStats(TCMallocStats* r, uint64_t* class_count, if (class_count) { // Sum the lengths of all per-class freelists, except the per-thread // freelists, which get counted when we call GetThreadStats(), below. - class_count[size_class] = length + tc_length; + class_count[size_class] = length + tc_length + sharded_tc_length; if (UsePerCpuCache(tc_globals)) { class_count[size_class] += tc_globals.cpu_cache().TotalObjectsOfClass(size_class); diff --git a/tcmalloc/transfer_cache.h b/tcmalloc/transfer_cache.h index 14d9ae27b..9237cea72 100644 --- a/tcmalloc/transfer_cache.h +++ b/tcmalloc/transfer_cache.h @@ -194,6 +194,16 @@ class ShardedTransferCacheManagerBase { return out; } + int TotalObjectsOfClass(int size_class) const { + if (shards_ == nullptr) return 0; + int objects = 0; + for (int shard = 0; shard < num_shards_; ++shard) { + if (!shard_initialized(shard)) continue; + objects += shards_[shard].transfer_caches[size_class].tc_length(); + } + return objects; + } + void *Pop(int size_class) { ASSERT(subtle::percpu::IsFastNoInit()); void *batch[1]; @@ -613,6 +623,7 @@ struct ShardedTransferCacheManager { static constexpr size_t TotalBytes() { return 0; } static constexpr void Plunder() {} static int tc_length(int cpu, int size_class) { return 0; } + static int TotalObjectsOfClass(int size_class) { return 0; } static constexpr TransferCacheStats GetStats(int size_class) { return {}; } bool UseGenericCache() const { return false; } bool UseCacheForLargeClassesOnly() const { return false; } diff --git a/tcmalloc/transfer_cache_test.cc b/tcmalloc/transfer_cache_test.cc index 3474b1ba5..d5f0d4072 100644 --- a/tcmalloc/transfer_cache_test.cc +++ b/tcmalloc/transfer_cache_test.cc @@ -516,6 +516,7 @@ TEST(ShardedTransferCacheManagerTest, MinimumNumShards) { EXPECT_TRUE(manager.shard_initialized(0)); EXPECT_FALSE(manager.shard_initialized(1)); EXPECT_EQ(manager.tc_length(0, kSizeClass), 0); + EXPECT_EQ(manager.TotalObjectsOfClass(kSizeClass), 0); EXPECT_EQ(manager.GetStats(kSizeClass).capacity, 0); EXPECT_EQ(manager.GetStats(kSizeClass).max_capacity, 0); EXPECT_GT(env.MetadataAllocated(), metadata); @@ -531,6 +532,7 @@ TEST(ShardedTransferCacheManagerTest, MinimumNumShards) { EXPECT_TRUE(manager.shard_initialized(0)); EXPECT_FALSE(manager.shard_initialized(1)); EXPECT_EQ(manager.tc_length(0, kSizeClass), 0); + EXPECT_EQ(manager.TotalObjectsOfClass(kSizeClass), 0); EXPECT_EQ(manager.GetStats(kSizeClass).capacity, 0); EXPECT_EQ(manager.GetStats(kSizeClass).max_capacity, 0); EXPECT_EQ(env.MetadataAllocated(), metadata); @@ -546,6 +548,7 @@ TEST(ShardedTransferCacheManagerTest, MinimumNumShards) { EXPECT_TRUE(manager.shard_initialized(0)); EXPECT_FALSE(manager.shard_initialized(1)); EXPECT_EQ(manager.tc_length(1, kSizeClass), 0); + EXPECT_EQ(manager.TotalObjectsOfClass(kSizeClass), 0); EXPECT_EQ(manager.GetStats(kSizeClass).capacity, 0); EXPECT_EQ(manager.GetStats(kSizeClass).max_capacity, 0); // No new metadata allocated @@ -563,6 +566,7 @@ TEST(ShardedTransferCacheManagerTest, MinimumNumShards) { EXPECT_TRUE(manager.shard_initialized(0)); EXPECT_TRUE(manager.shard_initialized(1)); EXPECT_EQ(manager.tc_length(2, kSizeClass), 0); + EXPECT_EQ(manager.TotalObjectsOfClass(kSizeClass), 0); EXPECT_EQ(manager.GetStats(kSizeClass).capacity, 0); EXPECT_EQ(manager.GetStats(kSizeClass).max_capacity, 0); EXPECT_GT(env.MetadataAllocated(), metadata); @@ -599,6 +603,7 @@ TEST(ShardedTransferCacheManagerTest, ShardsOnDemand) { manager.Push(kSizeClass, ptr); EXPECT_TRUE(manager.shard_initialized(0)); EXPECT_EQ(manager.tc_length(0, kSizeClass), 1); + EXPECT_EQ(manager.TotalObjectsOfClass(kSizeClass), 1); EXPECT_FALSE(manager.shard_initialized(1)); EXPECT_GT(env.MetadataAllocated(), metadata); metadata = env.MetadataAllocated(); @@ -611,6 +616,7 @@ TEST(ShardedTransferCacheManagerTest, ShardsOnDemand) { env.central_freelist().FreeBatch({&ptr, 1}); EXPECT_TRUE(manager.shard_initialized(0)); EXPECT_EQ(manager.tc_length(0, kSizeClass), 0); + EXPECT_EQ(manager.TotalObjectsOfClass(kSizeClass), 0); EXPECT_FALSE(manager.shard_initialized(1)); EXPECT_EQ(env.MetadataAllocated(), metadata); } @@ -623,6 +629,7 @@ TEST(ShardedTransferCacheManagerTest, ShardsOnDemand) { manager.Push(kSizeClass, ptr); EXPECT_TRUE(manager.shard_initialized(0)); EXPECT_EQ(manager.tc_length(1, kSizeClass), 1); + EXPECT_EQ(manager.TotalObjectsOfClass(kSizeClass), 1); EXPECT_FALSE(manager.shard_initialized(1)); // No new metadata allocated EXPECT_EQ(env.MetadataAllocated(), metadata); @@ -637,6 +644,7 @@ TEST(ShardedTransferCacheManagerTest, ShardsOnDemand) { EXPECT_TRUE(manager.shard_initialized(0)); EXPECT_TRUE(manager.shard_initialized(1)); EXPECT_EQ(manager.tc_length(2, kSizeClass), 1); + EXPECT_EQ(manager.TotalObjectsOfClass(kSizeClass), 2); EXPECT_GT(env.MetadataAllocated(), metadata); } }