(cloud-merge) Fix index data cannot be writed into ttl dir (#37485)

When write index data in ttl dir, it will cannot find the dir because expiration_time is 0. Reset context.expiration_time if the file is ttl file.
apache · Jul 19, 2024 · a011630 · a011630
1 parent 97b3dba
commit a011630
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 5 deletions.
diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp
@@ -637,10 +637,15 @@ void BlockFileCache::fill_holes_with_empty_file_blocks(FileBlocks& file_blocks,
 }
 
 FileBlocksHolder BlockFileCache::get_or_set(const UInt128Wrapper& hash, size_t offset, size_t size,
-                                            const CacheContext& context) {
+                                            CacheContext& context) {
     FileBlock::Range range(offset, offset + size - 1);
 
     std::lock_guard cache_lock(_mutex);
+    if (auto iter = _key_to_time.find(hash);
+        context.cache_type == FileCacheType::INDEX && iter != _key_to_time.end()) {
+        context.cache_type = FileCacheType::TTL;
+        context.expiration_time = iter->second;
+    }
 
     /// Get all blocks which intersect with the given range.
     auto file_blocks = get_impl(hash, context, range, cache_lock);

diff --git a/be/src/io/cache/block_file_cache.h b/be/src/io/cache/block_file_cache.h
@@ -86,7 +86,7 @@ class BlockFileCache {
          * it is guaranteed that these file blocks are not removed from cache.
          */
     FileBlocksHolder get_or_set(const UInt128Wrapper& hash, size_t offset, size_t size,
-                                const CacheContext& context);
+                                CacheContext& context);
 
     /**
      * Clear all cached data for this cache instance async

diff --git a/be/src/io/cache/cached_remote_file_reader.cpp b/be/src/io/cache/cached_remote_file_reader.cpp
@@ -224,7 +224,7 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset, Slice result, size_t*
                 st = block->finalize();
             }
             if (!st.ok()) {
-                LOG_WARNING("Write data to file cache failed").error(st);
+                LOG_EVERY_N(WARNING, 100) << "Write data to file cache failed. err=" << st.msg();
             } else {
                 _insert_file_reader(block);
             }

diff --git a/be/test/io/cache/block_file_cache_test.cpp b/be/test/io/cache/block_file_cache_test.cpp
@@ -1555,9 +1555,9 @@ TEST_F(BlockFileCacheTest, ttl_normal) {
     query_id.hi = 1;
     query_id.lo = 1;
     io::FileCacheSettings settings;
-    settings.query_queue_size = 30;
+    settings.query_queue_size = 50;
     settings.query_queue_elements = 5;
-    settings.capacity = 30;
+    settings.capacity = 50;
     settings.max_file_block_size = 30;
     settings.max_query_cache_size = 30;
     io::CacheContext context;
@@ -1596,6 +1596,18 @@ TEST_F(BlockFileCacheTest, ttl_normal) {
         assert_range(1, blocks[0], io::FileBlock::Range(50, 59), io::FileBlock::State::DOWNLOADED);
         EXPECT_EQ(blocks[0]->cache_type(), io::FileCacheType::TTL);
     }
+    {
+        context.cache_type = io::FileCacheType::INDEX;
+        context.expiration_time = 0;
+        auto holder = cache.get_or_set(key2, 60, 10, context); /// Add range [60, 69]
+        auto blocks = fromHolder(holder);
+        ASSERT_EQ(blocks.size(), 1);
+        assert_range(1, blocks[0], io::FileBlock::Range(60, 69), io::FileBlock::State::EMPTY);
+        ASSERT_TRUE(blocks[0]->get_or_set_downloader() == io::FileBlock::get_caller_id());
+        download(blocks[0]);
+        assert_range(1, blocks[0], io::FileBlock::Range(60, 69), io::FileBlock::State::DOWNLOADED);
+        EXPECT_EQ(blocks[0]->cache_type(), io::FileCacheType::TTL);
+    }
     {
         cache.modify_expiration_time(key2, modify_time);
         context.expiration_time = modify_time;