From 1f5aa0200a3dd49bf4c96546c9f21b0a5fb1fb16 Mon Sep 17 00:00:00 2001 From: b41sh Date: Thu, 30 May 2024 12:54:10 +0800 Subject: [PATCH 1/8] feat(query): show indexes add inverted indexes --- src/common/exception/src/exception_code.rs | 1 + .../interpreter_table_drop_column.rs | 16 ++- .../interpreter_table_modify_column.rs | 16 ++- .../service/src/test_kits/block_writer.rs | 2 + .../it/servers/http/http_query_handlers.rs | 6 +- .../it/storages/fuse/bloom_index_meta_size.rs | 1 + .../operations/mutation/recluster_mutator.rs | 1 + .../mutation/segments_compact_mutator.rs | 1 + .../it/storages/fuse/operations/read_plan.rs | 2 + .../tests/it/storages/fuse/statistics.rs | 2 + .../common/table_meta/src/meta/v2/segment.rs | 5 + .../src/meta/v3/frozen/block_meta.rs | 2 + .../fuse/src/io/write/block_writer.rs | 7 ++ .../storages/fuse/src/statistics/reducers.rs | 1 + .../storages/system/src/indexes_table.rs | 109 +++++++++++++++++- .../04_0000_inverted_index_base.test | 18 ++- 16 files changed, 177 insertions(+), 13 deletions(-) diff --git a/src/common/exception/src/exception_code.rs b/src/common/exception/src/exception_code.rs index 4ef3d3152d16b..57e2a660315e1 100644 --- a/src/common/exception/src/exception_code.rs +++ b/src/common/exception/src/exception_code.rs @@ -145,6 +145,7 @@ build_exceptions! { VirtualColumnNotFound(1115), VirtualColumnAlreadyExists(1116), ColumnReferencedByComputedColumn(1117), + ColumnReferencedByInvertedIndex(1118), // The table is not a clustered table. UnclusteredTable(1118), UnknownCatalog(1119), diff --git a/src/query/service/src/interpreters/interpreter_table_drop_column.rs b/src/query/service/src/interpreters/interpreter_table_drop_column.rs index eec91354c3e61..c93d2eaa90bf9 100644 --- a/src/query/service/src/interpreters/interpreter_table_drop_column.rs +++ b/src/query/service/src/interpreters/interpreter_table_drop_column.rs @@ -86,9 +86,10 @@ impl Interpreter for DropTableColumnInterpreter { ))); } - let mut schema: DataSchema = table_info.schema().into(); - let field = schema.field_with_name(self.plan.column.as_str())?; + let table_schema = table_info.schema(); + let field = table_schema.field_with_name(self.plan.column.as_str())?; if field.computed_expr().is_none() { + let mut schema: DataSchema = table_info.schema().into(); schema.drop_column(self.plan.column.as_str())?; // Check if this column is referenced by computed columns. check_referenced_computed_columns( @@ -97,6 +98,17 @@ impl Interpreter for DropTableColumnInterpreter { self.plan.column.as_str(), )?; } + // If the column is inverted index column, the column can't be dropped. + if !table_info.meta.indexes.is_empty() { + for (index_name, index) in &table_info.meta.indexes { + if index.column_ids.contains(&field.column_id) { + return Err(ErrorCode::ColumnReferencedByInvertedIndex(format!( + "column `{}` is referenced by inverted index `{}`", + field.name, index_name, + ))); + } + } + } let catalog = self.ctx.get_catalog(catalog_name).await?; let mut new_table_meta = table.get_table_info().meta.clone(); diff --git a/src/query/service/src/interpreters/interpreter_table_modify_column.rs b/src/query/service/src/interpreters/interpreter_table_modify_column.rs index 5c39ccc9db2c0..e4ec0d58a13e7 100644 --- a/src/query/service/src/interpreters/interpreter_table_modify_column.rs +++ b/src/query/service/src/interpreters/interpreter_table_modify_column.rs @@ -183,7 +183,7 @@ impl ModifyTableColumnInterpreter { for (field, comment) in field_and_comments { let column = &field.name.to_string(); let data_type = &field.data_type; - if let Some((i, _)) = schema.column_with_name(column) { + if let Some((i, old_field)) = schema.column_with_name(column) { if data_type != &new_schema.fields[i].data_type { // Check if this column is referenced by computed columns. let mut data_schema: DataSchema = table_info.schema().into(); @@ -204,6 +204,20 @@ impl ModifyTableColumnInterpreter { data_type ))); } + // If the column is inverted index column, the type can't be changed. + if !table_info.meta.indexes.is_empty() { + for (index_name, index) in &table_info.meta.indexes { + if index.column_ids.contains(&old_field.column_id) + && old_field.data_type.remove_nullable() + != field.data_type.remove_nullable() + { + return Err(ErrorCode::ColumnReferencedByInvertedIndex(format!( + "column `{}` is referenced by inverted index `{}`", + column, index_name, + ))); + } + } + } new_schema.fields[i].data_type = data_type.clone(); } if table_info.meta.field_comments[i] != *comment { diff --git a/src/query/service/src/test_kits/block_writer.rs b/src/query/service/src/test_kits/block_writer.rs index 2654c246f5c25..da64d44124aa1 100644 --- a/src/query/service/src/test_kits/block_writer.rs +++ b/src/query/service/src/test_kits/block_writer.rs @@ -77,6 +77,7 @@ impl<'a> BlockWriter<'a> { let mut buf = Vec::with_capacity(DEFAULT_BLOCK_BUFFER_SIZE); let col_metas = serialize_block(&write_settings, schema, block, &mut buf)?; let file_size = buf.len() as u64; + let inverted_index_size = None; data_accessor.write(&location.0, buf).await?; @@ -90,6 +91,7 @@ impl<'a> BlockWriter<'a> { location, bloom_filter_index_location, bloom_filter_index_size, + inverted_index_size, Compression::Lz4Raw, Some(Utc::now()), ); diff --git a/src/query/service/tests/it/servers/http/http_query_handlers.rs b/src/query/service/tests/it/servers/http/http_query_handlers.rs index f9a54bb2a2d5d..8183c9acae544 100644 --- a/src/query/service/tests/it/servers/http/http_query_handlers.rs +++ b/src/query/service/tests/it/servers/http/http_query_handlers.rs @@ -346,7 +346,7 @@ async fn test_show_databases() -> Result<()> { let _fixture = TestFixture::setup().await?; let sql = "show databases"; - let (status, result) = post_sql(sql, 1).await?; + let (status, result) = post_sql(sql, 3).await?; assert_eq!(status, StatusCode::OK, "{:?}", result); assert!(result.error.is_none(), "{:?}", result); // has only one field: name @@ -566,7 +566,7 @@ async fn test_pagination() -> Result<()> { let ep = create_endpoint().await?; let sql = "select * from numbers(10)"; - let json = serde_json::json!({"sql": sql.to_string(), "pagination": {"wait_time_secs": 1, "max_rows_per_page": 2}, "session": { "settings": {}}}); + let json = serde_json::json!({"sql": sql.to_string(), "pagination": {"wait_time_secs": 3, "max_rows_per_page": 2}, "session": { "settings": {}}}); let (status, result) = post_json_to_endpoint(&ep, &json, HeaderMap::default()).await?; assert_eq!(status, StatusCode::OK, "{:?}", result); @@ -695,7 +695,7 @@ async fn test_system_tables() -> Result<()> { let sql = "select name from system.tables where database='system' order by name"; - let (status, result) = post_sql_to_endpoint(&ep, sql, 1).await?; + let (status, result) = post_sql_to_endpoint(&ep, sql, 3).await?; assert_eq!(status, StatusCode::OK, "{:?}", result); assert!(!result.data.is_empty(), "{:?}", result); diff --git a/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs b/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs index bdb737adff2e0..fe28c3677b0c3 100644 --- a/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs +++ b/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs @@ -340,6 +340,7 @@ fn build_test_segment_info( location: block_location, bloom_filter_index_location: Some(location_gen.block_bloom_index_location(&block_uuid)), bloom_filter_index_size: 0, + inverted_index_size: None, compression: Compression::Lz4, create_on: Some(Utc::now()), }; diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/recluster_mutator.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/recluster_mutator.rs index a2e6c9dd659ea..eee18e656d2f1 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/recluster_mutator.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/recluster_mutator.rs @@ -75,6 +75,7 @@ async fn test_recluster_mutator_block_select() -> Result<()> { location.clone(), None, 0, + None, meta::Compression::Lz4Raw, Some(Utc::now()), )); diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/segments_compact_mutator.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/segments_compact_mutator.rs index 2f8ee2261e120..d910915cc9cea 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/segments_compact_mutator.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/segments_compact_mutator.rs @@ -774,6 +774,7 @@ impl CompactSegmentTestFixture { location, None, 0, + None, Compression::Lz4Raw, Some(Utc::now()), ); diff --git a/src/query/service/tests/it/storages/fuse/operations/read_plan.rs b/src/query/service/tests/it/storages/fuse/operations/read_plan.rs index 53f75b3801db3..1f713f3204f25 100644 --- a/src/query/service/tests/it/storages/fuse/operations/read_plan.rs +++ b/src/query/service/tests/it/storages/fuse/operations/read_plan.rs @@ -89,6 +89,7 @@ fn test_to_partitions() -> Result<()> { let bloom_filter_location = None; let bloom_filter_size = 0; + let inverted_index_size = None; let block_meta = Arc::new(BlockMeta::new( 0, block_size, @@ -99,6 +100,7 @@ fn test_to_partitions() -> Result<()> { location, bloom_filter_location, bloom_filter_size, + inverted_index_size, meta::Compression::Lz4Raw, Some(Utc::now()), )); diff --git a/src/query/service/tests/it/storages/fuse/statistics.rs b/src/query/service/tests/it/storages/fuse/statistics.rs index bea0c92245616..9426b0915f71f 100644 --- a/src/query/service/tests/it/storages/fuse/statistics.rs +++ b/src/query/service/tests/it/storages/fuse/statistics.rs @@ -610,6 +610,7 @@ fn test_reduce_block_meta() -> databend_common_exception::Result<()> { let block_size = rng.gen::() / size; let file_size = rng.gen::() / size; let bloom_filter_index_size = rng.gen::() / size; + let inverted_index_size = None; acc_row_count += row_count; acc_block_size += block_size; acc_file_size += file_size; @@ -624,6 +625,7 @@ fn test_reduce_block_meta() -> databend_common_exception::Result<()> { location.clone(), None, bloom_filter_index_size, + inverted_index_size, Compression::Lz4Raw, Some(Utc::now()), ); diff --git a/src/query/storages/common/table_meta/src/meta/v2/segment.rs b/src/query/storages/common/table_meta/src/meta/v2/segment.rs index 532fbfb46a08a..b050a5377cac7 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/segment.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/segment.rs @@ -77,6 +77,7 @@ pub struct BlockMeta { #[serde(default)] pub bloom_filter_index_size: u64, + pub inverted_index_size: Option, pub compression: Compression, // block create_on @@ -95,6 +96,7 @@ impl BlockMeta { location: Location, bloom_filter_index_location: Option, bloom_filter_index_size: u64, + inverted_index_size: Option, compression: Compression, create_on: Option>, ) -> Self { @@ -108,6 +110,7 @@ impl BlockMeta { location, bloom_filter_index_location, bloom_filter_index_size, + inverted_index_size, compression, create_on, } @@ -255,6 +258,7 @@ impl BlockMeta { bloom_filter_index_location: None, bloom_filter_index_size: 0, compression: Compression::Lz4, + inverted_index_size: None, create_on: None, } } @@ -287,6 +291,7 @@ impl BlockMeta { bloom_filter_index_location: s.bloom_filter_index_location.clone(), bloom_filter_index_size: s.bloom_filter_index_size, compression: s.compression, + inverted_index_size: None, create_on: None, } } diff --git a/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs b/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs index f778fbaebb5d2..772254905d8b0 100644 --- a/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs +++ b/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs @@ -39,6 +39,7 @@ pub struct BlockMeta { #[serde(default)] pub bloom_filter_index_size: u64, pub compression: Compression, + pub inverted_index_size: Option, } impl From for crate::meta::BlockMeta { @@ -61,6 +62,7 @@ impl From for crate::meta::BlockMeta { location: value.location, bloom_filter_index_location: value.bloom_filter_index_location, bloom_filter_index_size: value.bloom_filter_index_size, + inverted_index_size: value.inverted_index_size, compression: value.compression.into(), create_on: None, } diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index 63ce9df33e82e..dd0afb2eafc41 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -310,6 +310,12 @@ impl BlockBuilder { &mut buffer, )?; let file_size = buffer.len() as u64; + let inverted_index_size = if !inverted_index_states.is_empty() { + let size = inverted_index_states.iter().map(|v| v.size).sum(); + Some(size) + } else { + None + }; let block_meta = BlockMeta { row_count, block_size, @@ -324,6 +330,7 @@ impl BlockBuilder { .map(|v| v.size) .unwrap_or_default(), compression: self.write_settings.table_compression.into(), + inverted_index_size, create_on: Some(Utc::now()), }; diff --git a/src/query/storages/fuse/src/statistics/reducers.rs b/src/query/storages/fuse/src/statistics/reducers.rs index f6db36560e309..3229497bfc69b 100644 --- a/src/query/storages/fuse/src/statistics/reducers.rs +++ b/src/query/storages/fuse/src/statistics/reducers.rs @@ -211,6 +211,7 @@ pub fn reduce_block_metas>( uncompressed_byte_size += b.block_size; compressed_byte_size += b.file_size; index_size += b.bloom_filter_index_size; + index_size += b.inverted_index_size.unwrap_or_default(); if thresholds.check_large_enough(b.row_count as usize, b.block_size as usize) || b.cluster_stats.as_ref().is_some_and(|v| v.level != 0) { diff --git a/src/query/storages/system/src/indexes_table.rs b/src/query/storages/system/src/indexes_table.rs index 0c7ac5af77d17..b6d33326efd63 100644 --- a/src/query/storages/system/src/indexes_table.rs +++ b/src/query/storages/system/src/indexes_table.rs @@ -30,6 +30,7 @@ use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_storages_fuse::TableContext; +use log::warn; use crate::table::AsyncOneBlockSystemTable; use crate::table::AsyncSystemTable; @@ -57,12 +58,15 @@ impl AsyncSystemTable for IndexesTable { .list_indexes(ListIndexesReq::new(&tenant, None)) .await?; - let mut names = Vec::with_capacity(indexes.len()); - let mut types = Vec::with_capacity(indexes.len()); - let mut originals = Vec::with_capacity(indexes.len()); - let mut defs = Vec::with_capacity(indexes.len()); - let mut created_on = Vec::with_capacity(indexes.len()); - let mut updated_on = Vec::with_capacity(indexes.len()); + let inverted_index_tables = self.list_inverted_index_tables(ctx.clone()).await?; + + let len = indexes.len() + inverted_index_tables.len(); + let mut names = Vec::with_capacity(len); + let mut types = Vec::with_capacity(len); + let mut originals = Vec::with_capacity(len); + let mut defs = Vec::with_capacity(len); + let mut created_on = Vec::with_capacity(len); + let mut updated_on = Vec::with_capacity(len); for (_, name, index) in indexes { names.push(name.clone()); @@ -73,6 +77,37 @@ impl AsyncSystemTable for IndexesTable { updated_on.push(index.updated_on.map(|u| u.timestamp_micros())); } + for table in inverted_index_tables { + for (name, index) in &table.meta.indexes { + names.push(name.clone()); + types.push("INVERTED".to_string()); + originals.push("".to_string()); + + let schema = table.schema(); + let columns = index + .column_ids + .iter() + .map(|id| { + let field = schema.field_of_column_id(*id).unwrap(); + field.name.clone() + }) + .collect::>() + .join(", "); + + let options = index + .options + .iter() + .map(|(key, val)| format!("{}='{}'", key, val)) + .collect::>() + .join(", "); + + let def = format!("{}({}) {}", table.name, columns, options); + defs.push(def); + created_on.push(table.meta.created_on.timestamp_micros()); + updated_on.push(None); + } + } + Ok(DataBlock::new_from_columns(vec![ StringType::from_data(names), StringType::from_data(types), @@ -113,4 +148,66 @@ impl IndexesTable { AsyncOneBlockSystemTable::create(Self { table_info }) } + + async fn list_inverted_index_tables( + &self, + ctx: Arc, + ) -> Result> { + let tenant = ctx.get_tenant(); + let visibility_checker = ctx.get_visibility_checker().await?; + let catalog = ctx.get_catalog(CATALOG_DEFAULT).await?; + + let ctl_name = catalog.name(); + let dbs = match catalog.list_databases(&tenant).await { + Ok(dbs) => dbs + .into_iter() + .filter(|db| { + visibility_checker.check_database_visibility( + &ctl_name, + db.name(), + db.get_db_info().ident.db_id, + ) + }) + .collect::>(), + Err(err) => { + let msg = format!("List databases failed on catalog {}: {}", ctl_name, err); + warn!("{}", msg); + ctx.push_warning(msg); + + vec![] + } + }; + + let mut index_tables = Vec::new(); + for db in dbs { + let db_id = db.get_db_info().ident.db_id; + let db_name = db.name(); + + let tables = match catalog.list_tables(&tenant, db_name).await { + Ok(tables) => tables, + Err(err) => { + let msg = format!("Failed to list tables in database: {}, {}", db_name, err); + warn!("{}", msg); + ctx.push_warning(msg); + continue; + } + }; + for table in tables { + let table_info = table.get_table_info(); + if table_info.meta.indexes.is_empty() { + continue; + } + if visibility_checker.check_table_visibility( + &ctl_name, + db_name, + table.name(), + db_id, + table.get_id(), + ) { + index_tables.push(table_info.clone()); + } + } + } + Ok(index_tables) + } } diff --git a/tests/sqllogictests/suites/ee/04_ee_inverted_index/04_0000_inverted_index_base.test b/tests/sqllogictests/suites/ee/04_ee_inverted_index/04_0000_inverted_index_base.test index 213b061f9e730..a443bc84bbdc0 100644 --- a/tests/sqllogictests/suites/ee/04_ee_inverted_index/04_0000_inverted_index_base.test +++ b/tests/sqllogictests/suites/ee/04_ee_inverted_index/04_0000_inverted_index_base.test @@ -191,7 +191,7 @@ CREATE TABLE books( ) statement ok -CREATE INVERTED INDEX IF NOT EXISTS idx1 ON books(title, author, description) tokenizer = 'chinese' filters = 'english_stop,english_stemmer,chinese_stop' +CREATE INVERTED INDEX IF NOT EXISTS idx2 ON books(title, author, description) tokenizer = 'chinese' filters = 'english_stop,english_stemmer,chinese_stop' statement ok INSERT INTO books VALUES @@ -413,6 +413,22 @@ SELECT id, score(), body FROM t1 WHERE query('body.metadata.tags:技术') 6 2.4057739 {"metadata":{"author":"张三","publishedDate":"2023-10-23","tags":["人工智能","机器学习","技术"]},"title":"人工智能与机器学习"} 10 2.4057739 {"metadata":{"author":"刘七","publishedDate":"2023-06-25","tags":["网络安全","隐私保护","信息技术"]},"title":"网络安全与隐私保护"} +statement error 1118 +ALTER TABLE t1 DROP COLUMN body + +statement error 1118 +ALTER TABLE books MODIFY COLUMN title int; + +statement ok +ALTER TABLE books MODIFY COLUMN title string not null + +query TTT +SELECT name, type, definition FROM system.indexes order by name +---- +idx INVERTED t1(body) tokenizer='chinese' +idx1 INVERTED t(content) filters='english_stop,english_stemmer,chinese_stop', tokenizer='chinese' +idx2 INVERTED books(title, author, description) filters='english_stop,english_stemmer,chinese_stop', tokenizer='chinese' + statement ok use default From 8b73a6d21b197d7b1f80f6f2c594ed5f8a945663 Mon Sep 17 00:00:00 2001 From: b41sh Date: Wed, 5 Jun 2024 16:17:19 +0800 Subject: [PATCH 2/8] add tests --- .../interpreters/access/privilege_access.rs | 3 +- .../service/src/test_kits/block_writer.rs | 3 +- .../tests/it/storages/fuse/statistics.rs | 3 +- .../src/meta/v3/frozen/block_meta.rs | 2 +- .../table_functions/fuse_blocks/fuse_block.rs | 10 +++++++ .../18_rbac/18_0011_index_visibility.result | 3 ++ .../18_rbac/18_0011_index_visibility.sh | 29 +++++++++++++++++++ 7 files changed, 47 insertions(+), 6 deletions(-) create mode 100644 tests/suites/0_stateless/18_rbac/18_0011_index_visibility.result create mode 100755 tests/suites/0_stateless/18_rbac/18_0011_index_visibility.sh diff --git a/src/query/service/src/interpreters/access/privilege_access.rs b/src/query/service/src/interpreters/access/privilege_access.rs index 5003ae5e27e28..1b82a3a0bac1e 100644 --- a/src/query/service/src/interpreters/access/privilege_access.rs +++ b/src/query/service/src/interpreters/access/privilege_access.rs @@ -53,7 +53,7 @@ enum ObjectId { // some statements like `SELECT 1`, `SHOW USERS`, `SHOW ROLES`, `SHOW TABLES` will be // rewritten to the queries on the system tables, we need to skip the privilege check on // these tables. -const SYSTEM_TABLES_ALLOW_LIST: [&str; 18] = [ +const SYSTEM_TABLES_ALLOW_LIST: [&str; 19] = [ "catalogs", "columns", "databases", @@ -72,6 +72,7 @@ const SYSTEM_TABLES_ALLOW_LIST: [&str; 18] = [ "processes", "user_functions", "functions", + "indexes", ]; impl PrivilegeAccess { diff --git a/src/query/service/src/test_kits/block_writer.rs b/src/query/service/src/test_kits/block_writer.rs index da64d44124aa1..8fff99b506f28 100644 --- a/src/query/service/src/test_kits/block_writer.rs +++ b/src/query/service/src/test_kits/block_writer.rs @@ -77,7 +77,6 @@ impl<'a> BlockWriter<'a> { let mut buf = Vec::with_capacity(DEFAULT_BLOCK_BUFFER_SIZE); let col_metas = serialize_block(&write_settings, schema, block, &mut buf)?; let file_size = buf.len() as u64; - let inverted_index_size = None; data_accessor.write(&location.0, buf).await?; @@ -91,7 +90,7 @@ impl<'a> BlockWriter<'a> { location, bloom_filter_index_location, bloom_filter_index_size, - inverted_index_size, + None, Compression::Lz4Raw, Some(Utc::now()), ); diff --git a/src/query/service/tests/it/storages/fuse/statistics.rs b/src/query/service/tests/it/storages/fuse/statistics.rs index 9426b0915f71f..246b2799658dd 100644 --- a/src/query/service/tests/it/storages/fuse/statistics.rs +++ b/src/query/service/tests/it/storages/fuse/statistics.rs @@ -610,7 +610,6 @@ fn test_reduce_block_meta() -> databend_common_exception::Result<()> { let block_size = rng.gen::() / size; let file_size = rng.gen::() / size; let bloom_filter_index_size = rng.gen::() / size; - let inverted_index_size = None; acc_row_count += row_count; acc_block_size += block_size; acc_file_size += file_size; @@ -625,7 +624,7 @@ fn test_reduce_block_meta() -> databend_common_exception::Result<()> { location.clone(), None, bloom_filter_index_size, - inverted_index_size, + None, Compression::Lz4Raw, Some(Utc::now()), ); diff --git a/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs b/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs index 772254905d8b0..c9663006580c8 100644 --- a/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs +++ b/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs @@ -38,8 +38,8 @@ pub struct BlockMeta { #[serde(default)] pub bloom_filter_index_size: u64, - pub compression: Compression, pub inverted_index_size: Option, + pub compression: Compression, } impl From for crate::meta::BlockMeta { diff --git a/src/query/storages/fuse/src/table_functions/fuse_blocks/fuse_block.rs b/src/query/storages/fuse/src/table_functions/fuse_blocks/fuse_block.rs index 65653201fa7c2..c0658d350e354 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_blocks/fuse_block.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_blocks/fuse_block.rs @@ -111,6 +111,7 @@ impl<'a> FuseBlock<'a> { let mut row_count = Vec::with_capacity(len); let mut bloom_filter_location = vec![]; let mut bloom_filter_size = Vec::with_capacity(len); + let mut inverted_index_size = Vec::with_capacity(len); let segments_io = SegmentsIO::create( self.ctx.clone(), @@ -143,6 +144,7 @@ impl<'a> FuseBlock<'a> { .map(|s| s.0.clone()), ); bloom_filter_size.push(block.bloom_filter_index_size); + inverted_index_size.push(block.inverted_index_size); row_num += 1; if row_num >= limit { @@ -188,6 +190,10 @@ impl<'a> FuseBlock<'a> { DataType::Number(NumberDataType::UInt64), Value::Column(UInt64Type::from_data(bloom_filter_size)), ), + BlockEntry::new( + DataType::Nullable(Box::new(DataType::Number(NumberDataType::UInt64))), + Value::Column(UInt64Type::from_opt_data(inverted_index_size)), + ), ], row_num, )) @@ -209,6 +215,10 @@ impl<'a> FuseBlock<'a> { "bloom_filter_size", TableDataType::Number(NumberDataType::UInt64), ), + TableField::new( + "inverted_index_size", + TableDataType::Nullable(Box::new(TableDataType::Number(NumberDataType::UInt64))), + ), ]) } } diff --git a/tests/suites/0_stateless/18_rbac/18_0011_index_visibility.result b/tests/suites/0_stateless/18_rbac/18_0011_index_visibility.result new file mode 100644 index 0000000000000..01441c53bb072 --- /dev/null +++ b/tests/suites/0_stateless/18_rbac/18_0011_index_visibility.result @@ -0,0 +1,3 @@ +=== test u1 with role1 === +idx1 INVERTED t1(title) +=== test u2 with role1 === diff --git a/tests/suites/0_stateless/18_rbac/18_0011_index_visibility.sh b/tests/suites/0_stateless/18_rbac/18_0011_index_visibility.sh new file mode 100755 index 0000000000000..03f28c7fc1b68 --- /dev/null +++ b/tests/suites/0_stateless/18_rbac/18_0011_index_visibility.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../../../shell_env.sh + +echo "drop role if exists role1" | $BENDSQL_CLIENT_CONNECT +echo "drop role if exists role2" | $BENDSQL_CLIENT_CONNECT +echo "drop user if exists u1" | $BENDSQL_CLIENT_CONNECT +echo "drop user if exists u2" | $BENDSQL_CLIENT_CONNECT +echo "drop database if exists db1" | $BENDSQL_CLIENT_CONNECT +echo "create database db1" | $BENDSQL_CLIENT_CONNECT +echo "create table db1.t1(id int, title string, inverted index idx1(title))" | $BENDSQL_CLIENT_CONNECT +echo "insert into db1.t1 values(1, 'hello world')" | $BENDSQL_CLIENT_CONNECT +echo "create role role1;" | $BENDSQL_CLIENT_CONNECT +echo "create role role2;" | $BENDSQL_CLIENT_CONNECT +echo "grant select on db1.t1 to role role1;" | $BENDSQL_CLIENT_CONNECT +echo "create user u1 identified by '123' with DEFAULT_ROLE='role1';" | $BENDSQL_CLIENT_CONNECT +echo "create user u2 identified by '123' with DEFAULT_ROLE='role2';" | $BENDSQL_CLIENT_CONNECT +echo "grant role role1 to u1;" | $BENDSQL_CLIENT_CONNECT +echo "grant role role2 to u2;" | $BENDSQL_CLIENT_CONNECT + +echo "=== test u1 with role1 ===" +export TEST_U1_CONNECT="bendsql --user=u1 --password=123 --host=${QUERY_MYSQL_HANDLER_HOST} --port ${QUERY_HTTP_HANDLER_PORT}" +echo "select name, type, definition from system.indexes order by name" | $TEST_U1_CONNECT + +echo "=== test u2 with role1 ===" +export TEST_U2_CONNECT="bendsql --user=u2 --password=123 --host=${QUERY_MYSQL_HANDLER_HOST} --port ${QUERY_HTTP_HANDLER_PORT}" +echo "select name, type, definition from system.indexes order by name" | $TEST_U2_CONNECT + From c0e6ea4f0e3666565d8c5c7549422e3f6c77d015 Mon Sep 17 00:00:00 2001 From: b41sh Date: Wed, 5 Jun 2024 16:48:38 +0800 Subject: [PATCH 3/8] fix --- .../src/interpreters/interpreter_table_drop_column.rs | 2 +- .../src/interpreters/interpreter_table_modify_column.rs | 2 +- .../04_ee_inverted_index/04_0000_inverted_index_base.test | 6 +++--- .../0_stateless/18_rbac/18_0011_index_visibility.result | 2 +- .../suites/0_stateless/18_rbac/18_0011_index_visibility.sh | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/query/service/src/interpreters/interpreter_table_drop_column.rs b/src/query/service/src/interpreters/interpreter_table_drop_column.rs index c93d2eaa90bf9..02e62ec3628c2 100644 --- a/src/query/service/src/interpreters/interpreter_table_drop_column.rs +++ b/src/query/service/src/interpreters/interpreter_table_drop_column.rs @@ -103,7 +103,7 @@ impl Interpreter for DropTableColumnInterpreter { for (index_name, index) in &table_info.meta.indexes { if index.column_ids.contains(&field.column_id) { return Err(ErrorCode::ColumnReferencedByInvertedIndex(format!( - "column `{}` is referenced by inverted index `{}`", + "column `{}` is referenced by inverted index, drop inverted index `{}` first", field.name, index_name, ))); } diff --git a/src/query/service/src/interpreters/interpreter_table_modify_column.rs b/src/query/service/src/interpreters/interpreter_table_modify_column.rs index e4ec0d58a13e7..bdaa111ba2a00 100644 --- a/src/query/service/src/interpreters/interpreter_table_modify_column.rs +++ b/src/query/service/src/interpreters/interpreter_table_modify_column.rs @@ -212,7 +212,7 @@ impl ModifyTableColumnInterpreter { != field.data_type.remove_nullable() { return Err(ErrorCode::ColumnReferencedByInvertedIndex(format!( - "column `{}` is referenced by inverted index `{}`", + "column `{}` is referenced by inverted index, drop inverted index `{}` first", column, index_name, ))); } diff --git a/tests/sqllogictests/suites/ee/04_ee_inverted_index/04_0000_inverted_index_base.test b/tests/sqllogictests/suites/ee/04_ee_inverted_index/04_0000_inverted_index_base.test index a443bc84bbdc0..d31739cd505a6 100644 --- a/tests/sqllogictests/suites/ee/04_ee_inverted_index/04_0000_inverted_index_base.test +++ b/tests/sqllogictests/suites/ee/04_ee_inverted_index/04_0000_inverted_index_base.test @@ -342,10 +342,10 @@ SELECT id, score(), title FROM books WHERE query('title:(设计 实现)^5 descri # index without optional filters and index rocord is basic statement ok -CREATE OR REPLACE INVERTED INDEX idx1 ON books(title, author, description) tokenizer = 'chinese' index_record='basic' +CREATE OR REPLACE INVERTED INDEX idx2 ON books(title, author, description) tokenizer = 'chinese' index_record='basic' statement ok -REFRESH INVERTED INDEX idx1 ON books +REFRESH INVERTED INDEX idx2 ON books query IFT SELECT id, score(), title FROM books WHERE match('title^5, description^1.2', 'python') ORDER BY score() DESC @@ -427,7 +427,7 @@ SELECT name, type, definition FROM system.indexes order by name ---- idx INVERTED t1(body) tokenizer='chinese' idx1 INVERTED t(content) filters='english_stop,english_stemmer,chinese_stop', tokenizer='chinese' -idx2 INVERTED books(title, author, description) filters='english_stop,english_stemmer,chinese_stop', tokenizer='chinese' +idx2 INVERTED books(title, author, description) index_record='"basic"', tokenizer='chinese' statement ok use default diff --git a/tests/suites/0_stateless/18_rbac/18_0011_index_visibility.result b/tests/suites/0_stateless/18_rbac/18_0011_index_visibility.result index 01441c53bb072..7a26d3273b7b3 100644 --- a/tests/suites/0_stateless/18_rbac/18_0011_index_visibility.result +++ b/tests/suites/0_stateless/18_rbac/18_0011_index_visibility.result @@ -1,3 +1,3 @@ === test u1 with role1 === idx1 INVERTED t1(title) -=== test u2 with role1 === +=== test u2 with role2 === diff --git a/tests/suites/0_stateless/18_rbac/18_0011_index_visibility.sh b/tests/suites/0_stateless/18_rbac/18_0011_index_visibility.sh index 03f28c7fc1b68..f0010bbe4c8c2 100755 --- a/tests/suites/0_stateless/18_rbac/18_0011_index_visibility.sh +++ b/tests/suites/0_stateless/18_rbac/18_0011_index_visibility.sh @@ -23,7 +23,7 @@ echo "=== test u1 with role1 ===" export TEST_U1_CONNECT="bendsql --user=u1 --password=123 --host=${QUERY_MYSQL_HANDLER_HOST} --port ${QUERY_HTTP_HANDLER_PORT}" echo "select name, type, definition from system.indexes order by name" | $TEST_U1_CONNECT -echo "=== test u2 with role1 ===" +echo "=== test u2 with role2 ===" export TEST_U2_CONNECT="bendsql --user=u2 --password=123 --host=${QUERY_MYSQL_HANDLER_HOST} --port ${QUERY_HTTP_HANDLER_PORT}" echo "select name, type, definition from system.indexes order by name" | $TEST_U2_CONNECT From fc73ee3ace8ac437a3f20dc981253242ae1d9156 Mon Sep 17 00:00:00 2001 From: b41sh Date: Wed, 5 Jun 2024 18:10:30 +0800 Subject: [PATCH 4/8] fix tests --- src/query/storages/system/src/indexes_table.rs | 13 ++++++++++++- .../04_0000_inverted_index_base.test | 4 ++-- .../06_0001_index_visibility.result} | 0 .../06_inverted_index/06_0001_index_visibility.sh} | 0 4 files changed, 14 insertions(+), 3 deletions(-) rename tests/suites/{0_stateless/18_rbac/18_0011_index_visibility.result => 5_ee/06_inverted_index/06_0001_index_visibility.result} (100%) rename tests/suites/{0_stateless/18_rbac/18_0011_index_visibility.sh => 5_ee/06_inverted_index/06_0001_index_visibility.sh} (100%) diff --git a/src/query/storages/system/src/indexes_table.rs b/src/query/storages/system/src/indexes_table.rs index b6d33326efd63..4899adcd04d05 100644 --- a/src/query/storages/system/src/indexes_table.rs +++ b/src/query/storages/system/src/indexes_table.rs @@ -25,6 +25,8 @@ use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; +use databend_common_license::license::Feature; +use databend_common_license::license_manager::get_license_manager; use databend_common_meta_app::schema::ListIndexesReq; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; @@ -99,7 +101,7 @@ impl AsyncSystemTable for IndexesTable { .iter() .map(|(key, val)| format!("{}='{}'", key, val)) .collect::>() - .join(", "); + .join(" "); let def = format!("{}({}) {}", table.name, columns, options); defs.push(def); @@ -153,6 +155,15 @@ impl IndexesTable { &self, ctx: Arc, ) -> Result> { + let license_manager = get_license_manager(); + if license_manager + .manager + .check_enterprise_enabled(ctx.get_license_key(), Feature::InvertedIndex) + .is_err() + { + return Ok(vec![]); + } + let tenant = ctx.get_tenant(); let visibility_checker = ctx.get_visibility_checker().await?; let catalog = ctx.get_catalog(CATALOG_DEFAULT).await?; diff --git a/tests/sqllogictests/suites/ee/04_ee_inverted_index/04_0000_inverted_index_base.test b/tests/sqllogictests/suites/ee/04_ee_inverted_index/04_0000_inverted_index_base.test index d31739cd505a6..2e60089ea3156 100644 --- a/tests/sqllogictests/suites/ee/04_ee_inverted_index/04_0000_inverted_index_base.test +++ b/tests/sqllogictests/suites/ee/04_ee_inverted_index/04_0000_inverted_index_base.test @@ -426,8 +426,8 @@ query TTT SELECT name, type, definition FROM system.indexes order by name ---- idx INVERTED t1(body) tokenizer='chinese' -idx1 INVERTED t(content) filters='english_stop,english_stemmer,chinese_stop', tokenizer='chinese' -idx2 INVERTED books(title, author, description) index_record='"basic"', tokenizer='chinese' +idx1 INVERTED t(content) index_record='"basic"' tokenizer='chinese' +idx2 INVERTED books(title, author, description) index_record='"basic"' tokenizer='chinese' statement ok use default diff --git a/tests/suites/0_stateless/18_rbac/18_0011_index_visibility.result b/tests/suites/5_ee/06_inverted_index/06_0001_index_visibility.result similarity index 100% rename from tests/suites/0_stateless/18_rbac/18_0011_index_visibility.result rename to tests/suites/5_ee/06_inverted_index/06_0001_index_visibility.result diff --git a/tests/suites/0_stateless/18_rbac/18_0011_index_visibility.sh b/tests/suites/5_ee/06_inverted_index/06_0001_index_visibility.sh similarity index 100% rename from tests/suites/0_stateless/18_rbac/18_0011_index_visibility.sh rename to tests/suites/5_ee/06_inverted_index/06_0001_index_visibility.sh From 0a42f4a4bb642b9ad36b40520091d3cccc494633 Mon Sep 17 00:00:00 2001 From: b41sh Date: Wed, 5 Jun 2024 18:21:30 +0800 Subject: [PATCH 5/8] fix --- Cargo.lock | 1 + src/query/storages/system/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 24c5cee173591..0b946aa0a1dbd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4254,6 +4254,7 @@ dependencies = [ "databend-common-exception", "databend-common-expression", "databend-common-functions", + "databend-common-license", "databend-common-meta-api", "databend-common-meta-app", "databend-common-meta-types", diff --git a/src/query/storages/system/Cargo.toml b/src/query/storages/system/Cargo.toml index 83f5bff815909..87fd592fda91c 100644 --- a/src/query/storages/system/Cargo.toml +++ b/src/query/storages/system/Cargo.toml @@ -20,6 +20,7 @@ databend-common-config = { path = "../../config" } databend-common-exception = { path = "../../../common/exception" } databend-common-expression = { path = "../../expression" } databend-common-functions = { path = "../../functions" } +databend-common-license = { path = "../../../common/license" } databend-common-meta-api = { path = "../../../meta/api" } databend-common-meta-app = { path = "../../../meta/app" } databend-common-meta-types = { path = "../../../meta/types" } From d2f105c2b815eaf27be8d7ffaa59cca3e21b8c71 Mon Sep 17 00:00:00 2001 From: b41sh Date: Wed, 5 Jun 2024 20:57:33 +0800 Subject: [PATCH 6/8] fix tests --- src/query/storages/system/src/indexes_table.rs | 2 +- .../04_ee_inverted_index/04_0000_inverted_index_base.test | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/query/storages/system/src/indexes_table.rs b/src/query/storages/system/src/indexes_table.rs index 4899adcd04d05..88eb3e6d67344 100644 --- a/src/query/storages/system/src/indexes_table.rs +++ b/src/query/storages/system/src/indexes_table.rs @@ -103,7 +103,7 @@ impl AsyncSystemTable for IndexesTable { .collect::>() .join(" "); - let def = format!("{}({}) {}", table.name, columns, options); + let def = format!("{}({}){}", table.name, columns, options); defs.push(def); created_on.push(table.meta.created_on.timestamp_micros()); updated_on.push(None); diff --git a/tests/sqllogictests/suites/ee/04_ee_inverted_index/04_0000_inverted_index_base.test b/tests/sqllogictests/suites/ee/04_ee_inverted_index/04_0000_inverted_index_base.test index 2e60089ea3156..05a3b1e10a33d 100644 --- a/tests/sqllogictests/suites/ee/04_ee_inverted_index/04_0000_inverted_index_base.test +++ b/tests/sqllogictests/suites/ee/04_ee_inverted_index/04_0000_inverted_index_base.test @@ -425,9 +425,9 @@ ALTER TABLE books MODIFY COLUMN title string not null query TTT SELECT name, type, definition FROM system.indexes order by name ---- -idx INVERTED t1(body) tokenizer='chinese' -idx1 INVERTED t(content) index_record='"basic"' tokenizer='chinese' -idx2 INVERTED books(title, author, description) index_record='"basic"' tokenizer='chinese' +idx INVERTED t1(body)tokenizer='chinese' +idx1 INVERTED t(content)index_record='"basic"' tokenizer='chinese' +idx2 INVERTED books(title, author, description)index_record='"basic"' tokenizer='chinese' statement ok use default From 98e3064a8c2257e56cbb3f634c74b4f50879941d Mon Sep 17 00:00:00 2001 From: b41sh Date: Thu, 6 Jun 2024 12:19:03 +0800 Subject: [PATCH 7/8] fix --- Cargo.lock | 1 - src/query/storages/system/Cargo.toml | 1 - src/query/storages/system/src/indexes_table.rs | 11 ----------- 3 files changed, 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0b946aa0a1dbd..24c5cee173591 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4254,7 +4254,6 @@ dependencies = [ "databend-common-exception", "databend-common-expression", "databend-common-functions", - "databend-common-license", "databend-common-meta-api", "databend-common-meta-app", "databend-common-meta-types", diff --git a/src/query/storages/system/Cargo.toml b/src/query/storages/system/Cargo.toml index 87fd592fda91c..83f5bff815909 100644 --- a/src/query/storages/system/Cargo.toml +++ b/src/query/storages/system/Cargo.toml @@ -20,7 +20,6 @@ databend-common-config = { path = "../../config" } databend-common-exception = { path = "../../../common/exception" } databend-common-expression = { path = "../../expression" } databend-common-functions = { path = "../../functions" } -databend-common-license = { path = "../../../common/license" } databend-common-meta-api = { path = "../../../meta/api" } databend-common-meta-app = { path = "../../../meta/app" } databend-common-meta-types = { path = "../../../meta/types" } diff --git a/src/query/storages/system/src/indexes_table.rs b/src/query/storages/system/src/indexes_table.rs index 88eb3e6d67344..61b5cea9671cd 100644 --- a/src/query/storages/system/src/indexes_table.rs +++ b/src/query/storages/system/src/indexes_table.rs @@ -25,8 +25,6 @@ use databend_common_expression::FromData; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRefExt; -use databend_common_license::license::Feature; -use databend_common_license::license_manager::get_license_manager; use databend_common_meta_app::schema::ListIndexesReq; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; @@ -155,15 +153,6 @@ impl IndexesTable { &self, ctx: Arc, ) -> Result> { - let license_manager = get_license_manager(); - if license_manager - .manager - .check_enterprise_enabled(ctx.get_license_key(), Feature::InvertedIndex) - .is_err() - { - return Ok(vec![]); - } - let tenant = ctx.get_tenant(); let visibility_checker = ctx.get_visibility_checker().await?; let catalog = ctx.get_catalog(CATALOG_DEFAULT).await?; From 5935c786a374a251381d24f1756b6115885fd520 Mon Sep 17 00:00:00 2001 From: b41sh Date: Thu, 6 Jun 2024 18:07:26 +0800 Subject: [PATCH 8/8] remove inverted index size --- src/query/service/src/test_kits/block_writer.rs | 1 - .../tests/it/storages/fuse/bloom_index_meta_size.rs | 1 - .../fuse/operations/mutation/recluster_mutator.rs | 1 - .../operations/mutation/segments_compact_mutator.rs | 1 - .../tests/it/storages/fuse/operations/read_plan.rs | 2 -- src/query/service/tests/it/storages/fuse/statistics.rs | 1 - .../storages/common/table_meta/src/meta/v2/segment.rs | 5 ----- .../common/table_meta/src/meta/v3/frozen/block_meta.rs | 2 -- src/query/storages/fuse/src/io/write/block_writer.rs | 7 ------- src/query/storages/fuse/src/statistics/reducers.rs | 1 - .../fuse/src/table_functions/fuse_blocks/fuse_block.rs | 10 ---------- 11 files changed, 32 deletions(-) diff --git a/src/query/service/src/test_kits/block_writer.rs b/src/query/service/src/test_kits/block_writer.rs index 8fff99b506f28..2654c246f5c25 100644 --- a/src/query/service/src/test_kits/block_writer.rs +++ b/src/query/service/src/test_kits/block_writer.rs @@ -90,7 +90,6 @@ impl<'a> BlockWriter<'a> { location, bloom_filter_index_location, bloom_filter_index_size, - None, Compression::Lz4Raw, Some(Utc::now()), ); diff --git a/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs b/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs index fe28c3677b0c3..bdb737adff2e0 100644 --- a/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs +++ b/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs @@ -340,7 +340,6 @@ fn build_test_segment_info( location: block_location, bloom_filter_index_location: Some(location_gen.block_bloom_index_location(&block_uuid)), bloom_filter_index_size: 0, - inverted_index_size: None, compression: Compression::Lz4, create_on: Some(Utc::now()), }; diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/recluster_mutator.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/recluster_mutator.rs index eee18e656d2f1..a2e6c9dd659ea 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/recluster_mutator.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/recluster_mutator.rs @@ -75,7 +75,6 @@ async fn test_recluster_mutator_block_select() -> Result<()> { location.clone(), None, 0, - None, meta::Compression::Lz4Raw, Some(Utc::now()), )); diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/segments_compact_mutator.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/segments_compact_mutator.rs index d910915cc9cea..2f8ee2261e120 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/segments_compact_mutator.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/segments_compact_mutator.rs @@ -774,7 +774,6 @@ impl CompactSegmentTestFixture { location, None, 0, - None, Compression::Lz4Raw, Some(Utc::now()), ); diff --git a/src/query/service/tests/it/storages/fuse/operations/read_plan.rs b/src/query/service/tests/it/storages/fuse/operations/read_plan.rs index 1f713f3204f25..53f75b3801db3 100644 --- a/src/query/service/tests/it/storages/fuse/operations/read_plan.rs +++ b/src/query/service/tests/it/storages/fuse/operations/read_plan.rs @@ -89,7 +89,6 @@ fn test_to_partitions() -> Result<()> { let bloom_filter_location = None; let bloom_filter_size = 0; - let inverted_index_size = None; let block_meta = Arc::new(BlockMeta::new( 0, block_size, @@ -100,7 +99,6 @@ fn test_to_partitions() -> Result<()> { location, bloom_filter_location, bloom_filter_size, - inverted_index_size, meta::Compression::Lz4Raw, Some(Utc::now()), )); diff --git a/src/query/service/tests/it/storages/fuse/statistics.rs b/src/query/service/tests/it/storages/fuse/statistics.rs index 246b2799658dd..bea0c92245616 100644 --- a/src/query/service/tests/it/storages/fuse/statistics.rs +++ b/src/query/service/tests/it/storages/fuse/statistics.rs @@ -624,7 +624,6 @@ fn test_reduce_block_meta() -> databend_common_exception::Result<()> { location.clone(), None, bloom_filter_index_size, - None, Compression::Lz4Raw, Some(Utc::now()), ); diff --git a/src/query/storages/common/table_meta/src/meta/v2/segment.rs b/src/query/storages/common/table_meta/src/meta/v2/segment.rs index b050a5377cac7..532fbfb46a08a 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/segment.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/segment.rs @@ -77,7 +77,6 @@ pub struct BlockMeta { #[serde(default)] pub bloom_filter_index_size: u64, - pub inverted_index_size: Option, pub compression: Compression, // block create_on @@ -96,7 +95,6 @@ impl BlockMeta { location: Location, bloom_filter_index_location: Option, bloom_filter_index_size: u64, - inverted_index_size: Option, compression: Compression, create_on: Option>, ) -> Self { @@ -110,7 +108,6 @@ impl BlockMeta { location, bloom_filter_index_location, bloom_filter_index_size, - inverted_index_size, compression, create_on, } @@ -258,7 +255,6 @@ impl BlockMeta { bloom_filter_index_location: None, bloom_filter_index_size: 0, compression: Compression::Lz4, - inverted_index_size: None, create_on: None, } } @@ -291,7 +287,6 @@ impl BlockMeta { bloom_filter_index_location: s.bloom_filter_index_location.clone(), bloom_filter_index_size: s.bloom_filter_index_size, compression: s.compression, - inverted_index_size: None, create_on: None, } } diff --git a/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs b/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs index c9663006580c8..f778fbaebb5d2 100644 --- a/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs +++ b/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs @@ -38,7 +38,6 @@ pub struct BlockMeta { #[serde(default)] pub bloom_filter_index_size: u64, - pub inverted_index_size: Option, pub compression: Compression, } @@ -62,7 +61,6 @@ impl From for crate::meta::BlockMeta { location: value.location, bloom_filter_index_location: value.bloom_filter_index_location, bloom_filter_index_size: value.bloom_filter_index_size, - inverted_index_size: value.inverted_index_size, compression: value.compression.into(), create_on: None, } diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index dd0afb2eafc41..63ce9df33e82e 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -310,12 +310,6 @@ impl BlockBuilder { &mut buffer, )?; let file_size = buffer.len() as u64; - let inverted_index_size = if !inverted_index_states.is_empty() { - let size = inverted_index_states.iter().map(|v| v.size).sum(); - Some(size) - } else { - None - }; let block_meta = BlockMeta { row_count, block_size, @@ -330,7 +324,6 @@ impl BlockBuilder { .map(|v| v.size) .unwrap_or_default(), compression: self.write_settings.table_compression.into(), - inverted_index_size, create_on: Some(Utc::now()), }; diff --git a/src/query/storages/fuse/src/statistics/reducers.rs b/src/query/storages/fuse/src/statistics/reducers.rs index 3229497bfc69b..f6db36560e309 100644 --- a/src/query/storages/fuse/src/statistics/reducers.rs +++ b/src/query/storages/fuse/src/statistics/reducers.rs @@ -211,7 +211,6 @@ pub fn reduce_block_metas>( uncompressed_byte_size += b.block_size; compressed_byte_size += b.file_size; index_size += b.bloom_filter_index_size; - index_size += b.inverted_index_size.unwrap_or_default(); if thresholds.check_large_enough(b.row_count as usize, b.block_size as usize) || b.cluster_stats.as_ref().is_some_and(|v| v.level != 0) { diff --git a/src/query/storages/fuse/src/table_functions/fuse_blocks/fuse_block.rs b/src/query/storages/fuse/src/table_functions/fuse_blocks/fuse_block.rs index c0658d350e354..65653201fa7c2 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_blocks/fuse_block.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_blocks/fuse_block.rs @@ -111,7 +111,6 @@ impl<'a> FuseBlock<'a> { let mut row_count = Vec::with_capacity(len); let mut bloom_filter_location = vec![]; let mut bloom_filter_size = Vec::with_capacity(len); - let mut inverted_index_size = Vec::with_capacity(len); let segments_io = SegmentsIO::create( self.ctx.clone(), @@ -144,7 +143,6 @@ impl<'a> FuseBlock<'a> { .map(|s| s.0.clone()), ); bloom_filter_size.push(block.bloom_filter_index_size); - inverted_index_size.push(block.inverted_index_size); row_num += 1; if row_num >= limit { @@ -190,10 +188,6 @@ impl<'a> FuseBlock<'a> { DataType::Number(NumberDataType::UInt64), Value::Column(UInt64Type::from_data(bloom_filter_size)), ), - BlockEntry::new( - DataType::Nullable(Box::new(DataType::Number(NumberDataType::UInt64))), - Value::Column(UInt64Type::from_opt_data(inverted_index_size)), - ), ], row_num, )) @@ -215,10 +209,6 @@ impl<'a> FuseBlock<'a> { "bloom_filter_size", TableDataType::Number(NumberDataType::UInt64), ), - TableField::new( - "inverted_index_size", - TableDataType::Nullable(Box::new(TableDataType::Number(NumberDataType::UInt64))), - ), ]) } }