From 9b2994475f8d9ec2c17fa38b505f8165a55265eb Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Mon, 11 Mar 2024 13:57:34 +0800 Subject: [PATCH] feat(query): support settings admin api for global level settings --- src/query/service/src/api/http/v1/mod.rs | 1 + src/query/service/src/api/http/v1/settings.rs | 143 ++++++++++++++++++ src/query/service/src/api/http_service.rs | 26 +++- src/query/service/src/sessions/session_mgr.rs | 3 +- .../table_functions/others/license_info.rs | 2 +- src/query/settings/src/settings_global.rs | 27 +++- .../00_0000_target_tables.result | 2 - .../00_target_tables/00_0000_target_tables.sh | 8 - 8 files changed, 193 insertions(+), 19 deletions(-) create mode 100644 src/query/service/src/api/http/v1/settings.rs diff --git a/src/query/service/src/api/http/v1/mod.rs b/src/query/service/src/api/http/v1/mod.rs index 143006893d4cd..ac3a50720959f 100644 --- a/src/query/service/src/api/http/v1/mod.rs +++ b/src/query/service/src/api/http/v1/mod.rs @@ -19,6 +19,7 @@ pub mod instance_status; pub mod logs; pub mod processes; pub mod queries_queue; +pub mod settings; pub mod stream_status; pub mod system; pub mod tenant_tables; diff --git a/src/query/service/src/api/http/v1/settings.rs b/src/query/service/src/api/http/v1/settings.rs new file mode 100644 index 0000000000000..8a7e83629e713 --- /dev/null +++ b/src/query/service/src/api/http/v1/settings.rs @@ -0,0 +1,143 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; +use databend_common_meta_types::NonEmptyString; +use databend_common_settings::Settings; +use poem::web::Json; +use poem::web::Path; +use poem::IntoResponse; + +#[derive(serde::Serialize, serde::Deserialize)] +pub struct SettingsItem { + pub name: String, + pub desc: &'static str, + pub user_value: String, + pub default_value: String, + pub range: Option, +} + +async fn list_settings_impl(tenant: &str) -> Result> { + if tenant.is_empty() { + return Err(ErrorCode::TenantIsEmpty( + "Tenant can not empty(while list settings)", + )); + } + + let settings = Settings::create(NonEmptyString::new(tenant)?); + settings.load_changes().await?; + + Ok(settings + .into_iter() + .map(|item| SettingsItem { + name: item.name, + desc: item.desc, + user_value: item.user_value.to_string(), + default_value: item.default_value.to_string(), + range: item.range.map(|x| x.to_string()), + }) + .collect::>()) +} + +async fn set_setting_impl(tenant: &str, key: &str, value: String) -> Result> { + if tenant.is_empty() { + return Err(ErrorCode::TenantIsEmpty( + "Tenant can not empty(while set setting)", + )); + } + + if key.is_empty() || key.len() > 1024 { + return Err(ErrorCode::BadArguments( + "Setting key is empty or large length(while set setting)", + )); + } + + let settings = Settings::create(NonEmptyString::new(tenant)?); + settings.set_global_setting(key.to_string(), value).await?; + + Ok(settings + .into_iter() + .map(|item| SettingsItem { + name: item.name, + desc: item.desc, + user_value: item.user_value.to_string(), + default_value: item.default_value.to_string(), + range: item.range.map(|x| x.to_string()), + }) + .collect::>()) +} + +async fn unset_setting_impl(tenant: &str, key: &str) -> Result> { + if tenant.is_empty() { + return Err(ErrorCode::TenantIsEmpty( + "Tenant can not empty(while unset setting)", + )); + } + + if key.is_empty() || key.len() > 1024 { + return Err(ErrorCode::BadArguments( + "Setting key is empty or large length(while unset setting)", + )); + } + + let settings = Settings::create(NonEmptyString::new(tenant)?); + settings.try_drop_global_setting(key).await?; + + Ok(settings + .into_iter() + .map(|item| SettingsItem { + name: item.name, + desc: item.desc, + user_value: item.user_value.to_string(), + default_value: item.default_value.to_string(), + range: item.range.map(|x| x.to_string()), + }) + .collect::>()) +} + +#[poem::handler] +#[async_backtrace::framed] +pub async fn list_settings(Path(tenant): Path) -> poem::Result { + Ok(Json( + list_settings_impl(&tenant) + .await + .map_err(poem::error::InternalServerError)?, + )) +} + +#[poem::handler] +#[async_backtrace::framed] +pub async fn set_settings( + Path((tenant, key)): Path<(String, String)>, + value: Json, +) -> poem::Result { + Ok(Json( + set_setting_impl(&tenant, &key, value.0) + .await + .map_err(poem::error::InternalServerError)?, + )) +} + +#[poem::handler] +#[async_backtrace::framed] +pub async fn unset_settings( + Path((tenant, key)): Path<(String, String)>, +) -> poem::Result { + Ok(Json( + unset_setting_impl(&tenant, &key) + .await + .map_err(poem::error::InternalServerError)?, + )) +} diff --git a/src/query/service/src/api/http_service.rs b/src/query/service/src/api/http_service.rs index ae1b517320fcf..df12db4501807 100644 --- a/src/query/service/src/api/http_service.rs +++ b/src/query/service/src/api/http_service.rs @@ -32,6 +32,7 @@ use log::warn; use poem::get; use poem::listener::RustlsCertificate; use poem::listener::RustlsConfig; +use poem::post; use poem::Endpoint; use poem::Route; @@ -77,11 +78,9 @@ impl HttpService { ) .at("/debug/home", get(debug_home_handler)) .at("/debug/pprof/profile", get(debug_pprof_handler)) - .at("/debug/async_tasks/dump", get(debug_dump_stack)) - .at( - "/v1/background/:tenant/background_tasks", - get(super::http::v1::background_tasks::list_background_tasks), - ); + .at("/debug/async_tasks/dump", get(debug_dump_stack)); + + // Multiple tenants admin api if self.config.query.management_mode { route = route .at( @@ -91,6 +90,23 @@ impl HttpService { .at( "v1/tenants/:tenant/stream_status", get(super::http::v1::stream_status::stream_status_handler), + ) + .at( + "/v1/background/:tenant/background_tasks", + get(super::http::v1::background_tasks::list_background_tasks), + ) + .at( + "/v1/tenants/:tenant/background_tasks", + get(super::http::v1::background_tasks::list_background_tasks), + ) + .at( + "/v1/tenants/:tenant/settings", + get(super::http::v1::settings::list_settings), + ) + .at( + "/v1/tenants/:tenant/settings/:key", + post(super::http::v1::settings::set_settings) + .delete(super::http::v1::settings::unset_settings), ); } diff --git a/src/query/service/src/sessions/session_mgr.rs b/src/query/service/src/sessions/session_mgr.rs index ef9f6398b2d1a..6593c6636df9f 100644 --- a/src/query/service/src/sessions/session_mgr.rs +++ b/src/query/service/src/sessions/session_mgr.rs @@ -90,8 +90,7 @@ impl SessionManager { let tenant = GlobalConfig::instance().query.tenant_id.clone(); let settings = Settings::create(tenant); - self.load_config_changes(&settings)?; - settings.load_global_changes().await?; + settings.load_changes().await?; self.create_with_settings(typ, settings) } diff --git a/src/query/service/src/table_functions/others/license_info.rs b/src/query/service/src/table_functions/others/license_info.rs index 651cd3c0b7d7e..2c4babc5369ba 100644 --- a/src/query/service/src/table_functions/others/license_info.rs +++ b/src/query/service/src/table_functions/others/license_info.rs @@ -216,7 +216,7 @@ impl AsyncSource for LicenseInfoSource { let settings = self.ctx.get_settings(); // sync global changes on distributed node cluster. - settings.load_global_changes().await?; + settings.load_changes().await?; let license = unsafe { settings .get_enterprise_license() diff --git a/src/query/settings/src/settings_global.rs b/src/query/settings/src/settings_global.rs index 430712467eded..46a33dbdc3330 100644 --- a/src/query/settings/src/settings_global.rs +++ b/src/query/settings/src/settings_global.rs @@ -14,6 +14,8 @@ use std::sync::Arc; +use databend_common_config::GlobalConfig; +use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_meta_app::principal::UserSetting; use databend_common_meta_app::principal::UserSettingValue; @@ -61,7 +63,30 @@ impl Settings { } #[async_backtrace::framed] - pub async fn load_global_changes(&self) -> Result<()> { + pub async fn load_changes(&self) -> Result<()> { + self.load_config_changes()?; + self.load_global_changes().await + } + + fn load_config_changes(&self) -> Result<()> { + let query_config = &GlobalConfig::instance().query; + if let Some(parquet_fast_read_bytes) = query_config.parquet_fast_read_bytes { + self.set_parquet_fast_read_bytes(parquet_fast_read_bytes)?; + } + + if let Some(max_storage_io_requests) = query_config.max_storage_io_requests { + self.set_max_storage_io_requests(max_storage_io_requests)?; + } + + if let Some(enterprise_license_key) = query_config.databend_enterprise_license.clone() { + unsafe { + self.set_enterprise_license(enterprise_license_key)?; + } + } + Ok(()) + } + + async fn load_global_changes(&self) -> Result<(), ErrorCode> { let default_settings = DefaultSettings::instance()?; let api = UserApiProvider::instance(); diff --git a/tests/suites/6_background/00_target_tables/00_0000_target_tables.result b/tests/suites/6_background/00_target_tables/00_0000_target_tables.result index 213a689c3d790..cdbb78662da43 100644 --- a/tests/suites/6_background/00_target_tables/00_0000_target_tables.result +++ b/tests/suites/6_background/00_target_tables/00_0000_target_tables.result @@ -4,5 +4,3 @@ call system$execute_background_job('test_tenant-compactor-job'); target1 COMPACTION 'root'@'%' target1 COMPACTION 'root'@'%' target2 COMPACTION 'root'@'%' -default target1 -default target2 diff --git a/tests/suites/6_background/00_target_tables/00_0000_target_tables.sh b/tests/suites/6_background/00_target_tables/00_0000_target_tables.sh index 09763c6636c87..10b1a3cd2a022 100755 --- a/tests/suites/6_background/00_target_tables/00_0000_target_tables.sh +++ b/tests/suites/6_background/00_target_tables/00_0000_target_tables.sh @@ -28,14 +28,6 @@ sleep 5 echo "select st.name bt,type, bt.trigger from system.background_tasks AS bt JOIN system.tables st ON bt.table_id = st.table_id where bt.trigger is not null and bt.created_on > TO_TIMESTAMP('$current_time') order by st.name;" | $BENDSQL_CLIENT_CONNECT echo "select * from system.processes where type != 'HTTPQuery';" | $BENDSQL_CLIENT_CONNECT -table_ids=$(curl -X GET -s http://localhost:8080/v1/background/test_tenant/background_tasks?timestamp=$encoded_time | jq '[.task_infos[] | .[1].compaction_task_stats.table_id]') - -# Convert the table_ids JSON array to a comma-separated list -table_ids_list=$(echo $table_ids | jq -r 'join(",")') - -sql="select database, name from system.tables where table_id in ($table_ids_list) order by name;" -echo "$sql" | $BENDSQL_CLIENT_CONNECT - ## Drop table echo "drop table if exists target1;" | $BENDSQL_CLIENT_CONNECT echo "drop table if exists target2;" | $BENDSQL_CLIENT_CONNECT