From 10663db2a591b88621a7f580b8390e0060a5fe50 Mon Sep 17 00:00:00 2001 From: Dimitris Christodoulou <36637689+DemetrisChr@users.noreply.github.com> Date: Tue, 9 Apr 2024 11:02:38 +0100 Subject: [PATCH 01/11] CXXCBC-489: Add version_7_2_0 eventing function language compatibility (#554) --- core/management/eventing_function.hxx | 1 + core/management/eventing_function_json.hxx | 3 +++ core/operations/management/eventing_upsert_function.cxx | 3 +++ 3 files changed, 7 insertions(+) diff --git a/core/management/eventing_function.hxx b/core/management/eventing_function.hxx index 45dcd7d19..86eb59c59 100644 --- a/core/management/eventing_function.hxx +++ b/core/management/eventing_function.hxx @@ -44,6 +44,7 @@ enum class function_language_compatibility { version_6_0_0, version_6_5_0, version_6_6_2, + version_7_2_0, }; enum class function_log_level { diff --git a/core/management/eventing_function_json.hxx b/core/management/eventing_function_json.hxx index 7c9ac1238..bb17e4361 100644 --- a/core/management/eventing_function_json.hxx +++ b/core/management/eventing_function_json.hxx @@ -175,6 +175,9 @@ struct traits { } else if (language_compatibility_string == "6.6.2") { result.settings.language_compatibility = couchbase::core::management::eventing::function_language_compatibility::version_6_6_2; + } else if (language_compatibility_string == "7.2.0") { + result.settings.language_compatibility = + couchbase::core::management::eventing::function_language_compatibility::version_7_2_0; } } diff --git a/core/operations/management/eventing_upsert_function.cxx b/core/operations/management/eventing_upsert_function.cxx index e13baa06e..8191a7008 100644 --- a/core/operations/management/eventing_upsert_function.cxx +++ b/core/operations/management/eventing_upsert_function.cxx @@ -214,6 +214,9 @@ eventing_upsert_function_request::encode_to(encoded_request_type& encoded, http_ case couchbase::core::management::eventing::function_language_compatibility::version_6_6_2: settings["language_compatibility"] = "6.6.2"; break; + case couchbase::core::management::eventing::function_language_compatibility::version_7_2_0: + settings["language_compatibility"] = "7.2.0"; + break; } } From c77700e8517633fcd78466c40f00c2a728ee1e52 Mon Sep 17 00:00:00 2001 From: Jared Casey Date: Tue, 9 Apr 2024 16:18:05 -0500 Subject: [PATCH 02/11] CXXCBC-503: Ignore configuration if it contains an empty vBucketMap (#556) Motivation ========== Server versions prior to 7.6.2 (MB-60405) can sometimes provide a configuration that has a vBucketMap, but it does not contain any vbuckets. This should not happend, but since it does the SDK should ignore these configurations. Changes ======= Ignore a configuration if it contains an empty vBucketMap. --- core/bucket.cxx | 14 +++++++++++++- core/io/mcbp_session.cxx | 4 ++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/core/bucket.cxx b/core/bucket.cxx index 290eae832..26fe9eb45 100644 --- a/core/bucket.cxx +++ b/core/bucket.cxx @@ -591,7 +591,19 @@ class bucket_impl bool sequence_changed = false; { std::scoped_lock lock(config_mutex_); - if (!config_) { + if (config.vbmap && config.vbmap->size() == 0) { + if (!config_) { + CB_LOG_DEBUG("{} will not initialize configuration rev={} because config has an empty partition map", + log_prefix_, + config.rev_str()); + } else { + CB_LOG_DEBUG("{} will not update the configuration old={} -> new={}, because new config has an empty partition map", + log_prefix_, + config_->rev_str(), + config.rev_str()); + } + return; + } else if (!config_) { CB_LOG_DEBUG("{} initialize configuration rev={}", log_prefix_, config.rev_str()); } else if (config.force) { CB_LOG_DEBUG("{} forced to accept configuration rev={}", log_prefix_, config.rev_str()); diff --git a/core/io/mcbp_session.cxx b/core/io/mcbp_session.cxx index 82061c01f..9fb9843bb 100644 --- a/core/io/mcbp_session.cxx +++ b/core/io/mcbp_session.cxx @@ -1256,6 +1256,10 @@ class mcbp_session_impl return; } std::scoped_lock lock(config_mutex_); + if (config.vbmap && config.vbmap->size() == 0) { + CB_LOG_DEBUG("{} received a configuration with an empty vbucket map, ignoring", log_prefix_); + return; + } if (config_) { if (config_->vbmap && config.vbmap && config_->vbmap->size() != config.vbmap->size()) { CB_LOG_DEBUG("{} received a configuration with a different number of vbuckets, ignoring", log_prefix_); From 0ce5b719bfbc2017df9124bef1a71ecc21afa500 Mon Sep 17 00:00:00 2001 From: Jared Casey Date: Thu, 11 Apr 2024 14:50:24 -0500 Subject: [PATCH 03/11] CXXCBC-503 - Additions for more protection against picking up a config with an empty vbucket map. (#558) --- core/bucket.cxx | 20 +++++++++++++------- core/io/mcbp_session.cxx | 13 +++++++++++++ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/core/bucket.cxx b/core/bucket.cxx index 26fe9eb45..0e0b66582 100644 --- a/core/bucket.cxx +++ b/core/bucket.cxx @@ -591,17 +591,23 @@ class bucket_impl bool sequence_changed = false; { std::scoped_lock lock(config_mutex_); + // MB-60405 fixes this for 7.6.2, but for earlier versions we need to protect against using a + // config that has an empty vbucket map. Ideally we only run into this condition on initial + // bootstrap and that is handled in the session's update_config(), but just in case, only accept + // a config w/ a non-empty vbucket map. if (config.vbmap && config.vbmap->size() == 0) { if (!config_) { - CB_LOG_DEBUG("{} will not initialize configuration rev={} because config has an empty partition map", - log_prefix_, - config.rev_str()); + CB_LOG_WARNING("{} will not initialize configuration rev={} because config has an empty partition map", + log_prefix_, + config.rev_str()); } else { - CB_LOG_DEBUG("{} will not update the configuration old={} -> new={}, because new config has an empty partition map", - log_prefix_, - config_->rev_str(), - config.rev_str()); + CB_LOG_WARNING("{} will not update the configuration old={} -> new={}, because new config has an empty partition map", + log_prefix_, + config_->rev_str(), + config.rev_str()); } + // this is to make sure we can get a correct config soon + poll_config(errc::network::configuration_not_available); return; } else if (!config_) { CB_LOG_DEBUG("{} initialize configuration rev={}", log_prefix_, config.rev_str()); diff --git a/core/io/mcbp_session.cxx b/core/io/mcbp_session.cxx index 9fb9843bb..975ddb493 100644 --- a/core/io/mcbp_session.cxx +++ b/core/io/mcbp_session.cxx @@ -479,6 +479,15 @@ class mcbp_session_impl } protocol::client_response resp(std::move(msg), info); if (resp.status() == key_value_status_code::success) { + // MB-60405 fixes this for 7.6.2, but for earlier versions we need to protect against using a + // config that has an empty vbucket map. Ideally we don't timeout if we retry here, but a timeout + // would be more acceptable than a crash and if we do timeout, we have a clear indication of the + // problem (i.e. it is a server bug and we cannot use a config w/ an empty vbucket map). + if (resp.body().config().vbmap && resp.body().config().vbmap->size() == 0) { + CB_LOG_WARNING("{} received a configuration with an empty vbucket map, retrying", + session_->log_prefix_); + return complete(errc::network::configuration_not_available); + } session_->update_configuration(resp.body().config()); complete({}); } else if (resp.status() == key_value_status_code::not_found) { @@ -1256,6 +1265,10 @@ class mcbp_session_impl return; } std::scoped_lock lock(config_mutex_); + // MB-60405 fixes this for 7.6.2, but for earlier versions we need to protect against using a + // config that has an empty vbucket map. We should be okay to ignore at this point b/c we should + // already have a config w/ a non-empty vbucket map (bootstrap will not complete successfully + // unless we have a config w/ a non-empty vbucket map). if (config.vbmap && config.vbmap->size() == 0) { CB_LOG_DEBUG("{} received a configuration with an empty vbucket map, ignoring", log_prefix_); return; From ed19c54b0e1d0d87c8a7f1727f073ac2d9c04a1b Mon Sep 17 00:00:00 2001 From: Mateusz Date: Mon, 15 Apr 2024 15:35:24 +0100 Subject: [PATCH 04/11] CXXCBC-30 Inconsistent behaviour when using subdoc opcodes incorrectly (#559) --- test/test_integration_subdoc.cxx | 51 +++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/test/test_integration_subdoc.cxx b/test/test_integration_subdoc.cxx index 1a9c8a2de..0a1280721 100644 --- a/test/test_integration_subdoc.cxx +++ b/test/test_integration_subdoc.cxx @@ -150,8 +150,9 @@ assert_single_lookup_all_replica_success(test::utils::integration_test_guard& in INFO(fmt::format("assert_single_lookup_all_replica_success(\"{}\", \"{}\")", id, req.specs[0].path_)); REQUIRE_SUCCESS(response.ctx.ec()); REQUIRE(response.entries.size() == integration.number_of_replicas() + 1); - auto responses_from_active = - std::count_if(response.entries.begin(), response.entries.end(), [](const auto& r) { return !r.is_replica; }); + auto responses_from_active = std::count_if(response.entries.begin(), response.entries.end(), [](const auto& r) { + return !r.is_replica; + }); REQUIRE(responses_from_active == 1); for (auto& resp : response.entries) { REQUIRE_FALSE(resp.cas.empty()); @@ -181,8 +182,9 @@ assert_single_lookup_all_replica_error(test::utils::integration_test_guard& inte INFO(fmt::format("assert_single_lookup_all_replica_error(\"{}\", \"{}\")", id, req.specs[0].path_)); REQUIRE_SUCCESS(response.ctx.ec()); REQUIRE(response.entries.size() == integration.number_of_replicas() + 1); - auto responses_from_active = - std::count_if(response.entries.begin(), response.entries.end(), [](const auto& r) { return !r.is_replica; }); + auto responses_from_active = std::count_if(response.entries.begin(), response.entries.end(), [](const auto& r) { + return !r.is_replica; + }); REQUIRE(responses_from_active == 1); for (auto& resp : response.entries) { REQUIRE_FALSE(resp.cas.empty()); @@ -784,18 +786,31 @@ TEST_CASE("integration: subdoc multi lookup", "[integration]") SECTION("mismatched type and opcode") { - couchbase::core::operations::lookup_in_request req{ id }; - req.specs = - couchbase::mutate_in_specs{ - couchbase::mutate_in_specs::remove("array[0]"), - couchbase::mutate_in_specs::remove("array[0]"), - } - .specs(); - auto resp = test::utils::execute(integration.cluster, req); - if (integration.cluster_version().is_mock()) { - REQUIRE(resp.ctx.ec() == couchbase::errc::common::unsupported_operation); - } else { - REQUIRE(resp.ctx.ec() == couchbase::errc::common::invalid_argument); + { + couchbase::core::operations::lookup_in_request req{ id }; + req.specs = + couchbase::mutate_in_specs{ + couchbase::mutate_in_specs::remove("array[0]"), + couchbase::mutate_in_specs::remove("array[0]"), + } + .specs(); + auto resp = test::utils::execute(integration.cluster, req); + if (integration.cluster_version().is_mock()) { + REQUIRE(resp.ctx.ec() == couchbase::errc::common::unsupported_operation); + } else { + REQUIRE(resp.ctx.ec() == couchbase::errc::common::invalid_argument); + } + } + { + couchbase::core::operations::mutate_in_request req{ id }; + req.specs = + couchbase::lookup_in_specs{ couchbase::lookup_in_specs::get("foo"), couchbase::lookup_in_specs::get("foo") }.specs(); + auto resp = test::utils::execute(integration.cluster, req); + if (integration.cluster_version().is_mock()) { + REQUIRE(resp.ctx.ec() == couchbase::errc::common::unsupported_operation); + } else { + REQUIRE(resp.ctx.ec() == couchbase::errc::common::invalid_argument); + } } } @@ -1310,7 +1325,9 @@ TEST_CASE("integration: subdoc all replica reads", "[integration]") auto [ctx, result] = collection.lookup_in_all_replicas(key, specs).get(); REQUIRE_SUCCESS(ctx.ec()); REQUIRE(result.size() == number_of_replicas + 1); - auto responses_from_active = std::count_if(result.begin(), result.end(), [](const auto& r) { return !r.is_replica(); }); + auto responses_from_active = std::count_if(result.begin(), result.end(), [](const auto& r) { + return !r.is_replica(); + }); REQUIRE(responses_from_active == 1); for (auto& res : result) { REQUIRE(!res.cas().empty()); From b85bfeb44f929cbf5e3355439fe944da63abaf31 Mon Sep 17 00:00:00 2001 From: Dimitris Christodoulou <36637689+DemetrisChr@users.noreply.github.com> Date: Mon, 15 Apr 2024 16:28:06 +0100 Subject: [PATCH 05/11] Add feature check for scoped analyze_document in tests (#555) Co-authored-by: Sergey Avseyev --- test/test_integration_management.cxx | 4 ++-- test/utils/server_version.hxx | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/test/test_integration_management.cxx b/test/test_integration_management.cxx index 003fada39..fe7f18d82 100644 --- a/test/test_integration_management.cxx +++ b/test/test_integration_management.cxx @@ -4869,8 +4869,8 @@ TEST_CASE("integration: scope search index management analyze document public AP { test::utils::integration_test_guard integration; - if (!integration.cluster_version().supports_scope_search()) { - SKIP("cluster does not support scope search"); + if (!integration.cluster_version().supports_scope_search_analyze()) { + SKIP("cluster does not support scoped analyze_document"); } if (integration.cluster_version().is_capella()) { diff --git a/test/utils/server_version.hxx b/test/utils/server_version.hxx index b1694c8c5..670a54f01 100644 --- a/test/utils/server_version.hxx +++ b/test/utils/server_version.hxx @@ -186,6 +186,12 @@ struct server_version { return (major == 7 && minor >= 6) || major > 7; } + [[nodiscard]] bool supports_scope_search_analyze() const + { + // Scoped endpoint for analyze_document added in 7.6.2 (MB-60643) + return (major == 7 && minor == 6 && micro >= 2) || (major == 7 && minor > 6) || major > 7; + } + [[nodiscard]] bool is_enterprise() const { return edition == server_edition::enterprise; From 1b506b0eb2d88725f943aa62527ff604610ba680 Mon Sep 17 00:00:00 2001 From: Dimitris Christodoulou <36637689+DemetrisChr@users.noreply.github.com> Date: Tue, 16 Apr 2024 18:57:25 +0100 Subject: [PATCH 06/11] Always attempt to extract common query code if error has not been set (#561) --- core/operations/document_query.cxx | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/core/operations/document_query.cxx b/core/operations/document_query.cxx index 6e0a34a9c..16c176f92 100644 --- a/core/operations/document_query.cxx +++ b/core/operations/document_query.cxx @@ -400,15 +400,16 @@ query_request::make_response(error_context::query&& ctx, const encoded_response_ response.ctx.ec = errc::query::index_failure; } else if (response.ctx.first_error_code >= 4000 && response.ctx.first_error_code < 5000) { response.ctx.ec = errc::query::planning_failure; - } else { - auto common_ec = - management::extract_common_query_error_code(response.ctx.first_error_code, response.ctx.first_error_message); - if (common_ec) { - response.ctx.ec = common_ec.value(); - } } break; } + if (!response.ctx.ec) { + auto common_ec = + management::extract_common_query_error_code(response.ctx.first_error_code, response.ctx.first_error_message); + if (common_ec) { + response.ctx.ec = common_ec.value(); + } + } } if (!response.ctx.ec) { CB_LOG_TRACE("Unexpected error returned by query engine: client_context_id=\"{}\", body={}", From 328520a17cbf86860a0191feb9610092d9e60529 Mon Sep 17 00:00:00 2001 From: Sergey Avseyev Date: Mon, 29 Apr 2024 10:13:10 -0700 Subject: [PATCH 07/11] Improve test stability (#563) --- .github/workflows/tests.yml | 3 +- test/data/travel_sample_index_params.json | 177 ++++ test/data/travel_sample_index_params_v6.json | 171 ++++ test/test_integration_crud.cxx | 30 +- test/test_integration_examples.cxx | 64 +- test/test_integration_management.cxx | 783 ++++++++++++------ test/test_integration_management_eventing.cxx | 96 ++- test/test_integration_query.cxx | 6 + test/test_integration_search.cxx | 39 +- test/test_transaction_context.cxx | 8 + test/test_transaction_public_async_api.cxx | 9 +- test/test_transaction_public_blocking_api.cxx | 27 +- test/test_transaction_simple.cxx | 93 ++- test/test_transaction_simple_async.cxx | 47 +- test/test_unit_utils.cxx | 1 + test/utils/integration_test_guard.cxx | 2 +- test/utils/server_version.hxx | 5 + test/utils/test_context.cxx | 13 + test/utils/test_context.hxx | 1 + test/utils/test_data.cxx | 42 +- test/utils/wait_until.cxx | 241 +++++- test/utils/wait_until.hxx | 33 +- 22 files changed, 1538 insertions(+), 353 deletions(-) create mode 100644 test/data/travel_sample_index_params.json create mode 100644 test/data/travel_sample_index_params_v6.json diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fd3e12f8c..539f436ed 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,7 +12,8 @@ jobs: fail-fast: false matrix: server: - - 7.2.3 + - 7.6.1 + - 7.2.4 - 7.1.6 - 7.0.5 suite: diff --git a/test/data/travel_sample_index_params.json b/test/data/travel_sample_index_params.json new file mode 100644 index 000000000..c78934572 --- /dev/null +++ b/test/data/travel_sample_index_params.json @@ -0,0 +1,177 @@ +{ + "doc_config": { + "docid_prefix_delim": "", + "docid_regexp": "", + "mode": "scope.collection.type_field", + "type_field": "type" + }, + "mapping": { + "analysis": {}, + "default_analyzer": "standard", + "default_datetime_parser": "dateTimeOptional", + "default_field": "_all", + "default_mapping": { + "dynamic": false, + "enabled": false + }, + "default_type": "_default", + "docvalues_dynamic": false, + "index_dynamic": false, + "store_dynamic": false, + "type_field": "_type", + "types": { + "inventory.airline": { + "dynamic": false, + "enabled": true, + "properties": { + "country": { + "dynamic": false, + "enabled": true, + "fields": [ + { + "analyzer": "keyword", + "docvalues": true, + "index": true, + "name": "country", + "store": true, + "type": "text" + } + ] + }, + "name": { + "dynamic": false, + "enabled": true, + "fields": [ + { + "analyzer": "en", + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "name", + "store": true, + "type": "text" + } + ] + } + } + }, + "inventory.hotel": { + "dynamic": false, + "enabled": true, + "properties": { + "city": { + "dynamic": false, + "enabled": true, + "fields": [ + { + "analyzer": "en", + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "city", + "store": true, + "type": "text" + } + ] + }, + "country": { + "dynamic": false, + "enabled": true, + "fields": [ + { + "analyzer": "keyword", + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "country", + "store": true, + "type": "text" + } + ] + }, + "description": { + "dynamic": false, + "enabled": true, + "fields": [ + { + "analyzer": "en", + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "description", + "store": true, + "type": "text" + } + ] + }, + "reviews": { + "dynamic": false, + "enabled": true, + "properties": { + "content": { + "dynamic": false, + "enabled": true, + "fields": [ + { + "analyzer": "en", + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "content", + "store": true, + "type": "text" + } + ] + }, + "ratings": { + "dynamic": false, + "enabled": true, + "properties": { + "Overall": { + "dynamic": false, + "enabled": true, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "index": true, + "name": "Overall", + "store": true, + "type": "number" + } + ] + } + } + } + } + }, + "title": { + "dynamic": false, + "enabled": true, + "fields": [ + { + "analyzer": "en", + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "title", + "store": true, + "type": "text" + } + ] + } + } + } + } + }, + "store": { + "indexType": "scorch", + "segmentVersion": 15 + } +} diff --git a/test/data/travel_sample_index_params_v6.json b/test/data/travel_sample_index_params_v6.json new file mode 100644 index 000000000..29e2f0306 --- /dev/null +++ b/test/data/travel_sample_index_params_v6.json @@ -0,0 +1,171 @@ +{ + "doc_config": { + "docid_prefix_delim": "", + "docid_regexp": "", + "mode": "type_field", + "type_field": "type" + }, + "mapping": { + "default_analyzer": "standard", + "default_datetime_parser": "dateTimeOptional", + "default_field": "_all", + "default_mapping": { + "dynamic": true, + "enabled": true + }, + "default_type": "_default", + "docvalues_dynamic": true, + "index_dynamic": true, + "store_dynamic": false, + "type_field": "_type", + "types": { + "airline": { + "dynamic": true, + "enabled": true, + "properties": { + "country": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "country", + "store": true, + "type": "text" + } + ] + }, + "name": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "name", + "store": true, + "type": "text" + } + ] + } + } + }, + "hotel": { + "dynamic": true, + "enabled": true, + "properties": { + "reviews": { + "dynamic": true, + "enabled": true, + "properties": { + "ratings": { + "dynamic": true, + "enabled": true, + "properties": { + "Overall": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "Overall", + "store": true, + "type": "number" + } + ] + } + } + }, + "content": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "content", + "store": true, + "type": "text" + } + ] + } + } + }, + "city": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "city", + "store": true, + "type": "text" + } + ] + }, + "country": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "country", + "store": true, + "type": "text" + } + ] + }, + "description": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "description", + "store": true, + "type": "text" + } + ] + }, + "title": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "title", + "store": true, + "type": "text" + } + ] + } + } + } + } + }, + "store": { + "indexType": "scorch" + } +} diff --git a/test/test_integration_crud.cxx b/test/test_integration_crud.cxx index 1ec5a4e99..9003483da 100644 --- a/test/test_integration_crud.cxx +++ b/test/test_integration_crud.cxx @@ -249,8 +249,7 @@ TEST_CASE("integration: pessimistic locking", "[integration]") { couchbase::core::operations::get_and_lock_request req{ id }; req.lock_time = lock_time; - if (integration.ctx.deployment == test::utils::deployment_type::capella || - integration.ctx.deployment == test::utils::deployment_type::elixir) { + if (integration.ctx.use_wan_development_profile) { req.timeout = std::chrono::seconds{ 2 }; } auto resp = test::utils::execute(integration.cluster, req); @@ -672,9 +671,13 @@ TEST_CASE("integration: multi-threaded open/close bucket", "[integration]") threads.reserve(number_of_threads); for (auto i = 0; i < number_of_threads; ++i) { - threads.emplace_back([&integration]() { test::utils::open_bucket(integration.cluster, integration.ctx.bucket); }); + threads.emplace_back([&integration]() { + test::utils::open_bucket(integration.cluster, integration.ctx.bucket); + }); } - std::for_each(threads.begin(), threads.end(), [](auto& thread) { thread.join(); }); + std::for_each(threads.begin(), threads.end(), [](auto& thread) { + thread.join(); + }); threads.clear(); @@ -691,16 +694,22 @@ TEST_CASE("integration: multi-threaded open/close bucket", "[integration]") }); } - std::for_each(threads.begin(), threads.end(), [](auto& thread) { thread.join(); }); + std::for_each(threads.begin(), threads.end(), [](auto& thread) { + thread.join(); + }); threads.clear(); for (auto i = 0; i < number_of_threads; ++i) { - auto close_bucket = [&integration]() { test::utils::close_bucket(integration.cluster, integration.ctx.bucket); }; + auto close_bucket = [&integration]() { + test::utils::close_bucket(integration.cluster, integration.ctx.bucket); + }; std::thread closer(std::move(close_bucket)); threads.emplace_back(std::move(closer)); } - std::for_each(threads.begin(), threads.end(), [](auto& thread) { thread.join(); }); + std::for_each(threads.begin(), threads.end(), [](auto& thread) { + thread.join(); + }); } TEST_CASE("integration: open bucket that does not exist", "[integration]") @@ -715,7 +724,9 @@ TEST_CASE("integration: open bucket that does not exist", "[integration]") auto barrier = std::make_shared>(); auto f = barrier->get_future(); - integration.cluster.open_bucket(bucket_name, [barrier](std::error_code ec) mutable { barrier->set_value(ec); }); + integration.cluster.open_bucket(bucket_name, [barrier](std::error_code ec) mutable { + barrier->set_value(ec); + }); auto rc = f.get(); REQUIRE(rc == couchbase::errc::common::bucket_not_found); } @@ -882,8 +893,7 @@ TEST_CASE("integration: pessimistic locking with public API", "[integration]") // it is not allowed to lock the same key twice { couchbase::get_and_lock_options options{}; - if (integration.ctx.deployment == test::utils::deployment_type::capella || - integration.ctx.deployment == test::utils::deployment_type::elixir) { + if (integration.ctx.use_wan_development_profile) { options.timeout(std::chrono::seconds{ 2 }); } auto [ctx, resp] = collection.get_and_lock(id, lock_time, options).get(); diff --git a/test/test_integration_examples.cxx b/test/test_integration_examples.cxx index df9cf9579..55fadaebe 100644 --- a/test/test_integration_examples.cxx +++ b/test/test_integration_examples.cxx @@ -15,6 +15,7 @@ * limitations under the License. */ +#include "couchbase/configuration_profiles_registry.hxx" #include "test_helper_integration.hxx" #include "core/operations/management/query_index_build.hxx" @@ -33,6 +34,8 @@ #include +#include + #ifndef _WIN32 #include #endif @@ -210,6 +213,17 @@ namespace example_search #include +#include + +class github_actions_configuration_profile : public couchbase::configuration_profile +{ + public: + void apply(couchbase::cluster_options& options) override + { + options.timeouts().search_timeout(std::chrono::minutes(5)).management_timeout(std::chrono::minutes(5)); + } +}; + int main(int argc, const char* argv[]) { @@ -231,7 +245,9 @@ main(int argc, const char* argv[]) auto options = couchbase::cluster_options(username, password); // customize through the 'options'. // For example, optimize timeouts for WAN - options.apply_profile("wan_development"); + couchbase::configuration_profiles_registry::register_profile("github_actions", + std::make_shared()); + options.apply_profile("github_actions"); auto [cluster, ec] = couchbase::cluster::connect(io, connection_string, options).get(); if (ec) { @@ -365,16 +381,28 @@ main(int argc, const char* argv[]) state.add(upsert_result); } + auto start = std::chrono::system_clock::now(); auto [ctx, result] = cluster .search_query("travel-sample-index", couchbase::query_string_query("bree"), couchbase::search_options{}.consistent_with(state)) .get(); + auto stop = std::chrono::system_clock::now(); if (ctx.ec()) { - fmt::print("unable to perform search query: {}, ({}, {})\n", ctx.ec().message(), ctx.status(), ctx.error()); + fmt::print("unable to perform search query: {}, time: {} or {}, status: {}, error: {}\n", + ctx.ec().message(), + std::chrono::duration_cast(stop - start), + std::chrono::duration_cast(stop - start), + ctx.status(), + ctx.error()); return 1; } - fmt::print("{} hits, total: {}\n", result.rows().size(), result.meta_data().metrics().total_rows()); + fmt::print("{} hits, total: {}, time: {} or {} (server reported {})\n", + result.rows().size(), + result.meta_data().metrics().total_rows(), + std::chrono::duration_cast(stop - start), + std::chrono::duration_cast(stop - start), + result.meta_data().metrics().took()); for (const auto& row : result.rows()) { fmt::print("id: {}, score: {}\n", row.id(), row.score()); } @@ -445,13 +473,21 @@ row: {"airline":{"callsign":"MILE-AIR","country":"United States","iata":"Q5","ic TEST_CASE("example: search", "[integration]") { - test::utils::integration_test_guard integration; + { + test::utils::integration_test_guard integration; - if (integration.cluster_version().is_capella()) { - SKIP("Capella does not allow to use REST API to load sample buckets"); - } - if (!integration.cluster_version().supports_collections()) { - SKIP("cluster does not support collections"); + if (integration.cluster_version().is_capella()) { + SKIP("Capella does not allow to use REST API to load sample buckets"); + } + if (!integration.cluster_version().supports_collections()) { + SKIP("cluster does not support collections"); + } + + test::utils::create_search_index(integration, + "travel-sample", + "travel-sample-index", + integration.cluster_version().is_mad_hatter() ? "travel_sample_index_params_v6.json" + : "travel_sample_index_params.json"); } const auto env = test::utils::test_context::load_from_environment(); @@ -463,6 +499,11 @@ TEST_CASE("example: search", "[integration]") }; REQUIRE(example_search::main(4, argv) == 0); + + { + test::utils::integration_test_guard integration; + test::utils::drop_search_index(integration, "travel-sample-index"); + } } namespace example_buckets @@ -523,9 +564,10 @@ main(int argc, const char* argv[]) fmt::print("--- bucket has been successfully created\n"); } } + fmt::print("--- wait for couple of seconds (in highly distributed deployment, bucket creation might take few moments)\n"); + std::this_thread::sleep_for(std::chrono::seconds{ 2 }); { - fmt::print("--- get bucket\n"); - auto [ctx, bucket] = manager.get_bucket(bucket_name).get(); + auto [ctx, bucket] = manager.get_bucket(test_bucket_name).get(); if (ctx.ec()) { fmt::print("unable to get the bucket: {}\n", ctx.ec().message()); return 1; diff --git a/test/test_integration_management.cxx b/test/test_integration_management.cxx index fe7f18d82..193c860e7 100644 --- a/test/test_integration_management.cxx +++ b/test/test_integration_management.cxx @@ -18,6 +18,7 @@ #include "core/logger/logger.hxx" #include "test_helper_integration.hxx" +#include #include #include "core/management/analytics_link.hxx" @@ -42,13 +43,22 @@ using Catch::Approx; -static couchbase::core::operations::management::bucket_get_response +static bool wait_for_bucket_created(test::utils::integration_test_guard& integration, const std::string& bucket_name) { - test::utils::wait_until_bucket_healthy(integration.cluster, bucket_name); - couchbase::core::operations::management::bucket_get_request req{ bucket_name }; - auto resp = test::utils::execute(integration.cluster, req); - return resp; + // TODO: merge with success rounds code in collecton awaiter + constexpr int maximum_rounds{ 4 }; + constexpr int expected_success_rounds{ 4 }; + int success_rounds{ 0 }; + for (int round{ 0 }; round < maximum_rounds && success_rounds < expected_success_rounds; ++round) { + test::utils::wait_until_bucket_healthy(integration.cluster, bucket_name); + couchbase::core::operations::management::bucket_get_request req{ bucket_name }; + auto resp = test::utils::execute(integration.cluster, req); + if (!resp.ctx.ec) { + ++success_rounds; + } + } + return success_rounds >= expected_success_rounds; } template @@ -103,8 +113,9 @@ TEST_CASE("integration: bucket management", "[integration]") } { - auto resp = wait_for_bucket_created(integration, bucket_name); - REQUIRE_SUCCESS(resp.ctx.ec); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); + couchbase::core::operations::management::bucket_get_request req{ bucket_name }; + auto resp = test::utils::execute(integration.cluster, req); REQUIRE(bucket_settings.bucket_type == resp.bucket.bucket_type); REQUIRE(bucket_settings.name == resp.bucket.name); REQUIRE(Approx(bucket_settings.ram_quota_mb).margin(5) == resp.bucket.ram_quota_mb); @@ -182,8 +193,9 @@ TEST_CASE("integration: bucket management", "[integration]") auto resp = test::utils::execute(integration.cluster, req); REQUIRE_SUCCESS(resp.ctx.ec); REQUIRE(!resp.buckets.empty()); - auto known_buckets = std::count_if( - resp.buckets.begin(), resp.buckets.end(), [&bucket_name](auto& entry) { return entry.name == bucket_name; }); + auto known_buckets = std::count_if(resp.buckets.begin(), resp.buckets.end(), [&bucket_name](auto& entry) { + return entry.name == bucket_name; + }); REQUIRE(known_buckets == 0); } } @@ -208,11 +220,7 @@ TEST_CASE("integration: bucket management", "[integration]") REQUIRE_SUCCESS(ctx.ec()); } { - auto bucket_exists = test::utils::wait_until([&bucket_name, &c]() { - auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); - return ctx.ec() != couchbase::errc::common::bucket_not_found; - }); - REQUIRE(bucket_exists); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); REQUIRE_SUCCESS(ctx.ec()); REQUIRE(bucket_settings.bucket_type == bucket.bucket_type); @@ -276,8 +284,9 @@ TEST_CASE("integration: bucket management", "[integration]") auto [ctx, buckets] = c.buckets().get_all_buckets({}).get(); REQUIRE_SUCCESS(ctx.ec()); REQUIRE(!buckets.empty()); - auto known_buckets = - std::count_if(buckets.begin(), buckets.end(), [&bucket_name](auto& entry) { return entry.name == bucket_name; }); + auto known_buckets = std::count_if(buckets.begin(), buckets.end(), [&bucket_name](auto& entry) { + return entry.name == bucket_name; + }); REQUIRE(known_buckets == 0); } } @@ -416,11 +425,7 @@ TEST_CASE("integration: bucket management", "[integration]") REQUIRE_SUCCESS(ctx.ec()); } - auto bucket_exists = test::utils::wait_until([&bucket_name, c]() { - auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); - return ctx.ec() != couchbase::errc::common::bucket_not_found; - }); - REQUIRE(bucket_exists); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); { auto ctx = c.buckets().flush_bucket(bucket_name, {}).get(); @@ -446,8 +451,9 @@ TEST_CASE("integration: bucket management", "[integration]") } { - auto resp = wait_for_bucket_created(integration, bucket_name); - REQUIRE_SUCCESS(resp.ctx.ec); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); + couchbase::core::operations::management::bucket_get_request req{ bucket_name }; + auto resp = test::utils::execute(integration.cluster, req); REQUIRE(resp.bucket.bucket_type == couchbase::core::management::cluster::bucket_type::memcached); } } @@ -464,11 +470,8 @@ TEST_CASE("integration: bucket management", "[integration]") } { - auto bucket_exists = test::utils::wait_until([&bucket_name, &c]() { - auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); - return ctx.ec() != couchbase::errc::common::bucket_not_found; - }); - REQUIRE(bucket_exists); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); + auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); REQUIRE_SUCCESS(ctx.ec()); REQUIRE(bucket.bucket_type == couchbase::management::cluster::bucket_type::memcached); @@ -494,7 +497,9 @@ TEST_CASE("integration: bucket management", "[integration]") } { - auto resp = wait_for_bucket_created(integration, bucket_name); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); + couchbase::core::operations::management::bucket_get_request req{ bucket_name }; + auto resp = test::utils::execute(integration.cluster, req); REQUIRE_SUCCESS(resp.ctx.ec); REQUIRE(resp.bucket.bucket_type == couchbase::core::management::cluster::bucket_type::ephemeral); REQUIRE(resp.bucket.eviction_policy == couchbase::core::management::cluster::bucket_eviction_policy::no_eviction); @@ -511,7 +516,9 @@ TEST_CASE("integration: bucket management", "[integration]") } { - auto resp = wait_for_bucket_created(integration, bucket_name); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); + couchbase::core::operations::management::bucket_get_request req{ bucket_name }; + auto resp = test::utils::execute(integration.cluster, req); REQUIRE_SUCCESS(resp.ctx.ec); REQUIRE(resp.bucket.bucket_type == couchbase::core::management::cluster::bucket_type::ephemeral); REQUIRE(resp.bucket.eviction_policy == couchbase::core::management::cluster::bucket_eviction_policy::not_recently_used); @@ -529,8 +536,9 @@ TEST_CASE("integration: bucket management", "[integration]") } { - auto resp = wait_for_bucket_created(integration, bucket_name); - REQUIRE_SUCCESS(resp.ctx.ec); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); + couchbase::core::operations::management::bucket_get_request req{ bucket_name }; + auto resp = test::utils::execute(integration.cluster, req); REQUIRE(resp.bucket.bucket_type == couchbase::core::management::cluster::bucket_type::ephemeral); REQUIRE(resp.bucket.storage_backend == couchbase::core::management::cluster::bucket_storage_backend::unknown); } @@ -552,11 +560,7 @@ TEST_CASE("integration: bucket management", "[integration]") } { - auto bucket_exists = test::utils::wait_until([&bucket_name, c]() { - auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); - return ctx.ec() != couchbase::errc::common::bucket_not_found; - }); - REQUIRE(bucket_exists); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); REQUIRE_SUCCESS(ctx.ec()); REQUIRE(bucket.bucket_type == couchbase::management::cluster::bucket_type::ephemeral); @@ -573,11 +577,7 @@ TEST_CASE("integration: bucket management", "[integration]") } { - auto bucket_exists = test::utils::wait_until([&bucket_name, c]() { - auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); - return ctx.ec() != couchbase::errc::common::bucket_not_found; - }); - REQUIRE(bucket_exists); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); REQUIRE_SUCCESS(ctx.ec()); REQUIRE(bucket.bucket_type == couchbase::management::cluster::bucket_type::ephemeral); @@ -594,11 +594,7 @@ TEST_CASE("integration: bucket management", "[integration]") } { - auto bucket_exists = test::utils::wait_until([&bucket_name, c]() { - auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); - return ctx.ec() != couchbase::errc::common::bucket_not_found; - }); - REQUIRE(bucket_exists); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); REQUIRE_SUCCESS(ctx.ec()); REQUIRE(bucket.bucket_type == couchbase::management::cluster::bucket_type::ephemeral); @@ -626,8 +622,9 @@ TEST_CASE("integration: bucket management", "[integration]") } { - auto resp = wait_for_bucket_created(integration, bucket_name); - REQUIRE_SUCCESS(resp.ctx.ec); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); + couchbase::core::operations::management::bucket_get_request req{ bucket_name }; + auto resp = test::utils::execute(integration.cluster, req); REQUIRE(resp.bucket.bucket_type == couchbase::core::management::cluster::bucket_type::couchbase); REQUIRE(resp.bucket.eviction_policy == couchbase::core::management::cluster::bucket_eviction_policy::value_only); } @@ -643,8 +640,9 @@ TEST_CASE("integration: bucket management", "[integration]") } { - auto resp = wait_for_bucket_created(integration, bucket_name); - REQUIRE_SUCCESS(resp.ctx.ec); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); + couchbase::core::operations::management::bucket_get_request req{ bucket_name }; + auto resp = test::utils::execute(integration.cluster, req); REQUIRE(resp.bucket.bucket_type == couchbase::core::management::cluster::bucket_type::couchbase); REQUIRE(resp.bucket.eviction_policy == couchbase::core::management::cluster::bucket_eviction_policy::full); } @@ -663,8 +661,9 @@ TEST_CASE("integration: bucket management", "[integration]") } { - auto resp = wait_for_bucket_created(integration, bucket_name); - REQUIRE_SUCCESS(resp.ctx.ec); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); + couchbase::core::operations::management::bucket_get_request req{ bucket_name }; + auto resp = test::utils::execute(integration.cluster, req); REQUIRE(resp.bucket.bucket_type == couchbase::core::management::cluster::bucket_type::couchbase); REQUIRE(resp.bucket.storage_backend == couchbase::core::management::cluster::bucket_storage_backend::couchstore); @@ -682,8 +681,9 @@ TEST_CASE("integration: bucket management", "[integration]") } { - auto resp = wait_for_bucket_created(integration, bucket_name); - REQUIRE_SUCCESS(resp.ctx.ec); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); + couchbase::core::operations::management::bucket_get_request req{ bucket_name }; + auto resp = test::utils::execute(integration.cluster, req); REQUIRE(resp.bucket.bucket_type == couchbase::core::management::cluster::bucket_type::couchbase); REQUIRE(resp.bucket.storage_backend == couchbase::core::management::cluster::bucket_storage_backend::magma); } @@ -706,11 +706,7 @@ TEST_CASE("integration: bucket management", "[integration]") } { - auto bucket_exists = test::utils::wait_until([&bucket_name, c]() { - auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); - return ctx.ec() != couchbase::errc::common::bucket_not_found; - }); - REQUIRE(bucket_exists); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); REQUIRE_SUCCESS(ctx.ec()); REQUIRE(bucket.bucket_type == couchbase::management::cluster::bucket_type::couchbase); @@ -727,11 +723,7 @@ TEST_CASE("integration: bucket management", "[integration]") } { - auto bucket_exists = test::utils::wait_until([&bucket_name, c]() { - auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); - return ctx.ec() != couchbase::errc::common::bucket_not_found; - }); - REQUIRE(bucket_exists); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); REQUIRE_SUCCESS(ctx.ec()); REQUIRE(bucket.bucket_type == couchbase::management::cluster::bucket_type::couchbase); @@ -751,11 +743,7 @@ TEST_CASE("integration: bucket management", "[integration]") } { - auto bucket_exists = test::utils::wait_until([&bucket_name, c]() { - auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); - return ctx.ec() != couchbase::errc::common::bucket_not_found; - }); - REQUIRE(bucket_exists); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); REQUIRE_SUCCESS(ctx.ec()); REQUIRE(bucket.bucket_type == couchbase::management::cluster::bucket_type::couchbase); @@ -773,11 +761,7 @@ TEST_CASE("integration: bucket management", "[integration]") } { - auto bucket_exists = test::utils::wait_until([&bucket_name, c]() { - auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); - return ctx.ec() != couchbase::errc::common::bucket_not_found; - }); - REQUIRE(bucket_exists); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); REQUIRE_SUCCESS(ctx.ec()); REQUIRE(bucket.bucket_type == couchbase::management::cluster::bucket_type::couchbase); @@ -827,7 +811,9 @@ TEST_CASE("integration: bucket management", "[integration]") } { - auto resp = wait_for_bucket_created(integration, bucket_name); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); + couchbase::core::operations::management::bucket_get_request req{ bucket_name }; + auto resp = test::utils::execute(integration.cluster, req); REQUIRE_SUCCESS(resp.ctx.ec); REQUIRE(resp.bucket.minimum_durability_level == couchbase::durability_level::none); } @@ -844,7 +830,9 @@ TEST_CASE("integration: bucket management", "[integration]") } { - auto resp = wait_for_bucket_created(integration, bucket_name); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); + couchbase::core::operations::management::bucket_get_request req{ bucket_name }; + auto resp = test::utils::execute(integration.cluster, req); REQUIRE_SUCCESS(resp.ctx.ec); REQUIRE(resp.bucket.minimum_durability_level == couchbase::durability_level::majority); } @@ -864,11 +852,7 @@ TEST_CASE("integration: bucket management", "[integration]") REQUIRE_SUCCESS(ctx.ec()); } { - auto bucket_exists = test::utils::wait_until([&bucket_name, c]() { - auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); - return ctx.ec() != couchbase::errc::common::bucket_not_found; - }); - REQUIRE(bucket_exists); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); REQUIRE_SUCCESS(ctx.ec()); REQUIRE(bucket.minimum_durability_level == couchbase::durability_level::none); @@ -884,11 +868,7 @@ TEST_CASE("integration: bucket management", "[integration]") } { - auto bucket_exists = test::utils::wait_until([&bucket_name, c]() { - auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); - return ctx.ec() != couchbase::errc::common::bucket_not_found; - }); - REQUIRE(bucket_exists); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); auto [ctx, bucket] = c.buckets().get_bucket(bucket_name, {}).get(); REQUIRE_SUCCESS(ctx.ec()); REQUIRE(bucket.minimum_durability_level == couchbase::durability_level::majority); @@ -943,7 +923,9 @@ TEST_CASE("integration: bucket management history", "[integration]") } { - auto resp = wait_for_bucket_created(integration, bucket_name); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); + couchbase::core::operations::management::bucket_get_request req{ bucket_name }; + auto resp = test::utils::execute(integration.cluster, req); REQUIRE_SUCCESS(resp.ctx.ec); REQUIRE(resp.bucket.storage_backend == couchbase::core::management::cluster::bucket_storage_backend::magma); REQUIRE(resp.bucket.history_retention_collection_default == true); @@ -959,25 +941,37 @@ TEST_CASE("integration: bucket management history", "[integration]") bucket_settings.name = update_bucket_name; bucket_settings.storage_backend = couchbase::core::management::cluster::bucket_storage_backend::magma; { - couchbase::core::operations::management::bucket_create_request req{ bucket_settings }; - auto resp = test::utils::execute(integration.cluster, req); - REQUIRE_SUCCESS(resp.ctx.ec); - auto get_resp = wait_for_bucket_created(integration, update_bucket_name); - REQUIRE_SUCCESS(get_resp.ctx.ec); + { + couchbase::core::operations::management::bucket_create_request req{ bucket_settings }; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec); + } + { + REQUIRE(wait_for_bucket_created(integration, update_bucket_name)); + couchbase::core::operations::management::bucket_get_request req{ update_bucket_name }; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec); + } } { - bucket_settings.history_retention_collection_default = true; - bucket_settings.history_retention_bytes = 2147483648; - bucket_settings.history_retention_duration = 13000; - couchbase::core::operations::management::bucket_update_request req{ bucket_settings }; - auto resp = test::utils::execute(integration.cluster, req); - REQUIRE_SUCCESS(resp.ctx.ec); - auto get_resp = wait_for_bucket_created(integration, update_bucket_name); - REQUIRE_SUCCESS(get_resp.ctx.ec); - REQUIRE(get_resp.bucket.storage_backend == couchbase::core::management::cluster::bucket_storage_backend::magma); - REQUIRE(get_resp.bucket.history_retention_collection_default == true); - REQUIRE(get_resp.bucket.history_retention_duration == 13000); - REQUIRE(get_resp.bucket.history_retention_bytes == 2147483648); + { + bucket_settings.history_retention_collection_default = true; + bucket_settings.history_retention_bytes = 2147483648; + bucket_settings.history_retention_duration = 13000; + couchbase::core::operations::management::bucket_update_request req{ bucket_settings }; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec); + } + { + REQUIRE(wait_for_bucket_created(integration, update_bucket_name)); + couchbase::core::operations::management::bucket_get_request req{ update_bucket_name }; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec); + REQUIRE(resp.bucket.storage_backend == couchbase::core::management::cluster::bucket_storage_backend::magma); + REQUIRE(resp.bucket.history_retention_collection_default == true); + REQUIRE(resp.bucket.history_retention_duration == 13000); + REQUIRE(resp.bucket.history_retention_bytes == 2147483648); + } } } @@ -1071,7 +1065,9 @@ TEST_CASE("integration: collection management", "[integration]") } { - auto created = test::utils::wait_until([&]() { return scope_exists(integration.cluster, integration.ctx.bucket, scope_name); }); + auto created = test::utils::wait_until([&]() { + return scope_exists(integration.cluster, integration.ctx.bucket, scope_name); + }); REQUIRE(created); } @@ -1081,30 +1077,27 @@ TEST_CASE("integration: collection management", "[integration]") REQUIRE(resp.ctx.ec == couchbase::errc::management::scope_exists); } - { - couchbase::core::operations::management::collection_create_request req{ integration.ctx.bucket, scope_name, collection_name }; - if (integration.cluster_version().is_enterprise()) { + if (integration.cluster_version().is_enterprise()) { + { + couchbase::core::operations::management::collection_create_request req{ integration.ctx.bucket, + scope_name, + collection_name }; req.max_expiry = max_expiry; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec); + auto created = + test::utils::wait_until_collection_manifest_propagated(integration.cluster, integration.ctx.bucket, resp.uid); + REQUIRE(created); } - auto resp = test::utils::execute(integration.cluster, req); - REQUIRE_SUCCESS(resp.ctx.ec); - auto created = test::utils::wait_until_collection_manifest_propagated(integration.cluster, integration.ctx.bucket, resp.uid); - REQUIRE(created); - } - { - couchbase::core::topology::collections_manifest::collection collection; - auto created = test::utils::wait_until([&]() { - auto coll = get_collection(integration.cluster, integration.ctx.bucket, scope_name, collection_name); - if (coll) { - collection = *coll; - return true; - } - return false; - }); - REQUIRE(created); - if (integration.cluster_version().is_enterprise()) { - REQUIRE(collection.max_expiry == max_expiry); + { + std::optional collection{}; + REQUIRE(test::utils::wait_until([&]() { + collection = get_collection(integration.cluster, integration.ctx.bucket, scope_name, collection_name); + return collection.has_value(); + })); + + REQUIRE(collection->max_expiry == max_expiry); } } @@ -1120,8 +1113,9 @@ TEST_CASE("integration: collection management", "[integration]") } { - auto dropped = test::utils::wait_until( - [&]() { return !get_collection(integration.cluster, integration.ctx.bucket, scope_name, collection_name).has_value(); }); + auto dropped = test::utils::wait_until([&]() { + return !get_collection(integration.cluster, integration.ctx.bucket, scope_name, collection_name).has_value(); + }); REQUIRE(dropped); } @@ -1138,8 +1132,9 @@ TEST_CASE("integration: collection management", "[integration]") } { - auto dropped = - test::utils::wait_until([&]() { return !scope_exists(integration.cluster, integration.ctx.bucket, scope_name); }); + auto dropped = test::utils::wait_until([&]() { + return !scope_exists(integration.cluster, integration.ctx.bucket, scope_name); + }); REQUIRE(dropped); } @@ -1277,9 +1272,12 @@ TEST_CASE("integration: collection management create collection with max expiry" REQUIRE_SUCCESS(ctx.ec()); } - auto coll = get_collection(integration.cluster, integration.ctx.bucket, scope_name, collection_name); - REQUIRE(coll); - REQUIRE(coll.value().max_expiry == 0); + std::optional collection{}; + REQUIRE(test::utils::wait_until([&]() { + collection = get_collection(integration.cluster, integration.ctx.bucket, scope_name, collection_name); + return collection.has_value(); + })); + REQUIRE(collection->max_expiry == 0); } SECTION("positive max expiry") @@ -1303,9 +1301,12 @@ TEST_CASE("integration: collection management create collection with max expiry" REQUIRE_SUCCESS(ctx.ec()); } - auto coll = get_collection(integration.cluster, integration.ctx.bucket, scope_name, collection_name); - REQUIRE(coll); - REQUIRE(coll.value().max_expiry == 3600); + std::optional collection{}; + REQUIRE(test::utils::wait_until([&]() { + collection = get_collection(integration.cluster, integration.ctx.bucket, scope_name, collection_name); + return collection.has_value(); + })); + REQUIRE(collection->max_expiry == 3600); } SECTION("setting max expiry to no-expiry") @@ -1330,9 +1331,12 @@ TEST_CASE("integration: collection management create collection with max expiry" REQUIRE_SUCCESS(ctx.ec()); } - auto coll = get_collection(integration.cluster, integration.ctx.bucket, scope_name, collection_name); - REQUIRE(coll); - REQUIRE(coll.value().max_expiry == -1); + std::optional collection{}; + REQUIRE(test::utils::wait_until([&]() { + collection = get_collection(integration.cluster, integration.ctx.bucket, scope_name, collection_name); + return collection.has_value(); + })); + REQUIRE(collection->max_expiry == -1); } else { SECTION("core API") { @@ -1426,9 +1430,12 @@ TEST_CASE("integration: collection management update collection with max expiry" REQUIRE_SUCCESS(ctx.ec()); } - auto coll = get_collection(integration.cluster, integration.ctx.bucket, scope_name, collection_name); - REQUIRE(coll); - REQUIRE(coll.value().max_expiry == 0); + std::optional collection{}; + REQUIRE(test::utils::wait_until([&]() { + collection = get_collection(integration.cluster, integration.ctx.bucket, scope_name, collection_name); + return collection.has_value(); + })); + REQUIRE(collection->max_expiry == 0); } SECTION("positive max expiry") @@ -1452,9 +1459,12 @@ TEST_CASE("integration: collection management update collection with max expiry" REQUIRE_SUCCESS(ctx.ec()); } - auto coll = get_collection(integration.cluster, integration.ctx.bucket, scope_name, collection_name); - REQUIRE(coll); - REQUIRE(coll.value().max_expiry == 3600); + std::optional collection{}; + REQUIRE(test::utils::wait_until([&]() { + collection = get_collection(integration.cluster, integration.ctx.bucket, scope_name, collection_name); + return collection.has_value(); + })); + REQUIRE(collection->max_expiry == 3600); } SECTION("setting max expiry to no-expiry") @@ -1479,9 +1489,12 @@ TEST_CASE("integration: collection management update collection with max expiry" REQUIRE_SUCCESS(ctx.ec()); } - auto coll = get_collection(integration.cluster, integration.ctx.bucket, scope_name, collection_name); - REQUIRE(coll); - REQUIRE(coll.value().max_expiry == -1); + std::optional collection{}; + REQUIRE(test::utils::wait_until([&]() { + collection = get_collection(integration.cluster, integration.ctx.bucket, scope_name, collection_name); + return collection.has_value(); + })); + REQUIRE(collection->max_expiry == -1); } else { SECTION("core API") { @@ -1642,7 +1655,9 @@ TEST_CASE("integration: collection management bucket dedup", "[integration]") REQUIRE_SUCCESS(resp.ctx.ec); } { - auto resp = wait_for_bucket_created(integration, bucket_name); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); + couchbase::core::operations::management::bucket_get_request req{ bucket_name }; + auto resp = test::utils::execute(integration.cluster, req); REQUIRE_SUCCESS(resp.ctx.ec); } { @@ -1654,7 +1669,9 @@ TEST_CASE("integration: collection management bucket dedup", "[integration]") } { - auto created = test::utils::wait_until([&]() { return scope_exists(integration.cluster, bucket_name, scope_name); }); + auto created = test::utils::wait_until([&]() { + return scope_exists(integration.cluster, bucket_name, scope_name); + }); REQUIRE(created); } @@ -1667,17 +1684,12 @@ TEST_CASE("integration: collection management bucket dedup", "[integration]") REQUIRE(created); } { - couchbase::core::topology::collections_manifest::collection collection; - auto created = test::utils::wait_until([&]() { - auto coll = get_collection(integration.cluster, bucket_name, scope_name, collection_name); - if (coll) { - collection = *coll; - return true; - } - return false; - }); - REQUIRE(created); - REQUIRE(collection.history.value()); + std::optional collection{}; + REQUIRE(test::utils::wait_until([&]() { + collection = get_collection(integration.cluster, bucket_name, scope_name, collection_name); + return collection.has_value(); + })); + REQUIRE(collection->history.value()); } { couchbase::core::operations::management::collection_update_request req{ bucket_name, scope_name, collection_name }; @@ -1686,18 +1698,12 @@ TEST_CASE("integration: collection management bucket dedup", "[integration]") REQUIRE_SUCCESS(resp.ctx.ec); } { - couchbase::core::topology::collections_manifest::collection collection; - auto no_history = test::utils::wait_until([&]() { - auto coll = get_collection(integration.cluster, bucket_name, scope_name, collection_name); - if (coll.has_value()) { - if (!coll.value().history.value()) { - return true; - } - return false; - } - return false; - }); - REQUIRE(no_history); + std::optional collection{}; + REQUIRE(test::utils::wait_until([&]() { + collection = get_collection(integration.cluster, bucket_name, scope_name, collection_name); + return collection.has_value(); + })); + REQUIRE_FALSE(collection->history.value_or(false)); } // Clean up the bucket that was created for this test @@ -1716,8 +1722,9 @@ assert_user_and_metadata(const couchbase::core::management::rbac::user_and_metad REQUIRE(user.groups == expected.groups); REQUIRE(user.roles.size() == expected.roles.size()); for (const auto& role : user.roles) { - auto expected_role = - std::find_if(expected.roles.begin(), expected.roles.end(), [&role](const auto& exp_role) { return role.name == exp_role.name; }); + auto expected_role = std::find_if(expected.roles.begin(), expected.roles.end(), [&role](const auto& exp_role) { + return role.name == exp_role.name; + }); REQUIRE(expected_role != expected.roles.end()); REQUIRE(role.name == expected_role->name); REQUIRE(role.bucket == expected_role->bucket); @@ -1738,9 +1745,10 @@ assert_user_and_metadata(const couchbase::core::management::rbac::user_and_metad REQUIRE(role.collection == expected_role->collection); REQUIRE(role.origins.size() == expected_role->origins.size()); for (const auto& origin : role.origins) { - auto expected_origin = std::find_if(expected_role->origins.begin(), - expected_role->origins.end(), - [&origin](const auto& exp_origin) { return origin.name == exp_origin.name; }); + auto expected_origin = + std::find_if(expected_role->origins.begin(), expected_role->origins.end(), [&origin](const auto& exp_origin) { + return origin.name == exp_origin.name; + }); REQUIRE(expected_origin != expected_role->origins.end()); REQUIRE(origin.name == expected_origin->name); REQUIRE(origin.type == expected_origin->type); @@ -1939,8 +1947,9 @@ TEST_CASE("integration: user groups management", "[integration]") auto resp = test::utils::execute(integration.cluster, req); REQUIRE_SUCCESS(resp.ctx.ec); REQUIRE_FALSE(resp.users.empty()); - auto upserted_user = - std::find_if(resp.users.begin(), resp.users.end(), [&user_name](const auto& u) { return u.username == user_name; }); + auto upserted_user = std::find_if(resp.users.begin(), resp.users.end(), [&user_name](const auto& u) { + return u.username == user_name; + }); REQUIRE(upserted_user != resp.users.end()); assert_user_and_metadata(*upserted_user, expected); } @@ -2012,7 +2021,9 @@ TEST_CASE("integration: user management", "[integration]") { asio::io_context io; auto guard = asio::make_work_guard(io); - std::thread io_thread([&io]() { io.run(); }); + std::thread io_thread([&io]() { + io.run(); + }); // Create new user and upsert couchbase::core::management::rbac::user new_user{ user_name }; @@ -2036,7 +2047,9 @@ TEST_CASE("integration: user management", "[integration]") // Connect with new credentials and change password asio::io_context io; auto guard = asio::make_work_guard(io); - std::thread io_thread([&io]() { io.run(); }); + std::thread io_thread([&io]() { + io.run(); + }); auto [cluster_new, ec_new] = couchbase::cluster::connect(io, integration.ctx.connection_string, options_outdated).get(); couchbase::core::operations::management::change_password_request changePasswordReq{}; changePasswordReq.newPassword = "newPassword"; @@ -2052,7 +2065,9 @@ TEST_CASE("integration: user management", "[integration]") // Connect with old credentials, should fail asio::io_context io; auto guard = asio::make_work_guard(io); - std::thread io_thread([&io]() { io.run(); }); + std::thread io_thread([&io]() { + io.run(); + }); auto [cluster_fail, ec_fail] = couchbase::cluster::connect(io, integration.ctx.connection_string, options_outdated).get(); REQUIRE(ec_fail == couchbase::errc::common::authentication_failure); @@ -2138,10 +2153,14 @@ TEST_CASE("integration: user management collections roles", "[integration]") REQUIRE_SUCCESS(resp.ctx.ec); } + // Increase chance that the change will be replicated to all nodes + std::this_thread::sleep_for(std::chrono::seconds{ 1 }); + { couchbase::core::operations::management::user_get_request req{ user_name }; auto resp = test::utils::execute(integration.cluster, req); REQUIRE_SUCCESS(resp.ctx.ec); + INFO(resp.ctx.http_body); REQUIRE(resp.user.roles.size() == 1); REQUIRE(resp.user.roles[0].name == "data_reader"); REQUIRE(resp.user.roles[0].bucket == integration.ctx.bucket); @@ -2177,7 +2196,8 @@ TEST_CASE("integration: query index management", "[integration]") auto resp = test::utils::execute(integration.cluster, req); } - REQUIRE(!wait_for_bucket_created(integration, bucket_name).ctx.ec); + REQUIRE(wait_for_bucket_created(integration, bucket_name)); + SECTION("core API") { { @@ -2202,11 +2222,6 @@ TEST_CASE("integration: query index management", "[integration]") REQUIRE(resp.indexes[0].name == "#primary"); REQUIRE(resp.indexes[0].is_primary); } - - { - couchbase::core::operations::management::bucket_drop_request req{ bucket_name }; - test::utils::execute(integration.cluster, req); - } } SECTION("public api") { @@ -2226,7 +2241,9 @@ TEST_CASE("integration: query index management", "[integration]") if (ctx.ec()) { return false; } - return std::any_of(res.begin(), res.end(), [](const auto& index) { return index.name == "#primary"; }); + return std::any_of(res.begin(), res.end(), [](const auto& index) { + return index.name == "#primary"; + }); }); { auto [ctx, indexes] = c.query_indexes().get_all_indexes(bucket_name, {}).get(); @@ -2245,6 +2262,11 @@ TEST_CASE("integration: query index management", "[integration]") REQUIRE_SUCCESS(ctx.ec()); } } + + { + couchbase::core::operations::management::bucket_drop_request req{ bucket_name }; + test::utils::execute(integration.cluster, req); + } } } @@ -2291,8 +2313,9 @@ TEST_CASE("integration: query index management", "[integration]") req.bucket_name = integration.ctx.bucket; auto resp = test::utils::execute(integration.cluster, req); REQUIRE_SUCCESS(resp.ctx.ec); - auto index = std::find_if( - resp.indexes.begin(), resp.indexes.end(), [&index_name](const auto& exp_index) { return exp_index.name == index_name; }); + auto index = std::find_if(resp.indexes.begin(), resp.indexes.end(), [&index_name](const auto& exp_index) { + return exp_index.name == index_name; + }); REQUIRE(index != resp.indexes.end()); REQUIRE(index->name == index_name); REQUIRE_FALSE(index->is_primary); @@ -2339,7 +2362,9 @@ TEST_CASE("integration: query index management", "[integration]") if (ctx.ec()) { return false; } - return std::any_of(res.begin(), res.end(), [&index_name](const auto& index) { return index.name == index_name; }); + return std::any_of(res.begin(), res.end(), [&index_name](const auto& index) { + return index.name == index_name; + }); }); { auto ctx = c.query_indexes().watch_indexes(integration.ctx.bucket, { index_name }, {}).get(); @@ -2362,8 +2387,9 @@ TEST_CASE("integration: query index management", "[integration]") { auto [ctx, indexes] = c.query_indexes().get_all_indexes(integration.ctx.bucket, {}).get(); - auto index = std::find_if( - indexes.begin(), indexes.end(), [&index_name](const auto& exp_index) { return exp_index.name == index_name; }); + auto index = std::find_if(indexes.begin(), indexes.end(), [&index_name](const auto& exp_index) { + return exp_index.name == index_name; + }); REQUIRE(index != indexes.end()); REQUIRE(index->name == index_name); REQUIRE_FALSE(index->is_primary); @@ -2383,6 +2409,7 @@ TEST_CASE("integration: query index management", "[integration]") { auto ctx = c.query_indexes().drop_index(integration.ctx.bucket, index_name, {}).get(); couchbase::core::operations::management::query_index_drop_request req{}; + INFO(ctx.content()); REQUIRE(ctx.ec() == couchbase::errc::common::index_not_found); } { @@ -2420,8 +2447,9 @@ TEST_CASE("integration: query index management", "[integration]") { auto [ctx, indexes] = c.query_indexes().get_all_indexes(integration.ctx.bucket, {}).get(); REQUIRE_SUCCESS(ctx.ec()); - auto index = std::find_if( - indexes.begin(), indexes.end(), [&index_name](const auto& exp_index) { return exp_index.name == index_name; }); + auto index = std::find_if(indexes.begin(), indexes.end(), [&index_name](const auto& exp_index) { + return exp_index.name == index_name; + }); REQUIRE(index != indexes.end()); REQUIRE(index->name == index_name); REQUIRE(index->state == "deferred"); @@ -2439,12 +2467,19 @@ TEST_CASE("integration: query index management", "[integration]") if (indexes.empty()) { return false; } - auto index = std::find_if( - indexes.begin(), indexes.end(), [&index_name](const auto& exp_index) { return exp_index.name == index_name; }); + auto index = std::find_if(indexes.begin(), indexes.end(), [&index_name](const auto& exp_index) { + return exp_index.name == index_name; + }); return index->state == "online"; }); REQUIRE(operation_completed); } + { + auto ctx = c.query_indexes().drop_index(integration.ctx.bucket, index_name, {}).get(); + couchbase::core::operations::management::query_index_drop_request req{}; + INFO(ctx.content()); + REQUIRE_SUCCESS(ctx.ec()); + } } SECTION("core API") @@ -2485,6 +2520,14 @@ TEST_CASE("integration: query index management", "[integration]") return index->state == "online"; }); } + + { + couchbase::core::operations::management::query_index_drop_request req{}; + req.bucket_name = integration.ctx.bucket; + req.index_name = index_name; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec); + } } } @@ -2728,7 +2771,9 @@ TEST_CASE("integration: collections query index management", "[integration]") if (ctx.ec()) { return false; } - return std::any_of(res.begin(), res.end(), [&index_name](const auto& index) { return index.name == index_name; }); + return std::any_of(res.begin(), res.end(), [&index_name](const auto& index) { + return index.name == index_name; + }); }); { auto [ctx, indexes] = manager.get_all_indexes({}).get(); @@ -2847,7 +2892,9 @@ TEST_CASE("integration: collections query index management", "[integration]") if (ctx.ec()) { return false; } - return std::any_of(res.begin(), res.end(), [&index_name](const auto& index) { return index.name == index_name; }); + return std::any_of(res.begin(), res.end(), [&index_name](const auto& index) { + return index.name == index_name; + }); }); { REQUIRE(manager.create_index(index_name, { "field" }, {}).get().ec() == couchbase::errc::common::index_exists); @@ -2861,7 +2908,9 @@ TEST_CASE("integration: collections query index management", "[integration]") if (ctx.ec()) { return false; } - return std::any_of(res.begin(), res.end(), [&index_name](const auto& index) { return index.name == index_name; }); + return std::any_of(res.begin(), res.end(), [&index_name](const auto& index) { + return index.name == index_name; + }); }); { REQUIRE_SUCCESS(manager.watch_indexes({ index_name }, {}).get().ec()); @@ -2893,7 +2942,6 @@ TEST_CASE("integration: collections query index management", "[integration]") SECTION("deferred index") { - SKIP("XXX"); SECTION("public API") { { @@ -2906,7 +2954,9 @@ TEST_CASE("integration: collections query index management", "[integration]") if (ctx.ec()) { return false; } - return std::any_of(res.begin(), res.end(), [&index_name](const auto& index) { return index.name == index_name; }); + return std::any_of(res.begin(), res.end(), [&index_name](const auto& index) { + return index.name == index_name; + }); }); { auto [ctx, indexes] = manager.get_all_indexes({}).get(); @@ -3258,8 +3308,9 @@ TEST_CASE("integration: analytics index management with core API", "[integration auto resp = test::utils::execute(integration.cluster, req); REQUIRE_SUCCESS(resp.ctx.ec); REQUIRE_FALSE(resp.indexes.empty()); - auto index = std::find_if( - resp.indexes.begin(), resp.indexes.end(), [&index_name](const auto& exp_index) { return exp_index.name == index_name; }); + auto index = std::find_if(resp.indexes.begin(), resp.indexes.end(), [&index_name](const auto& exp_index) { + return exp_index.name == index_name; + }); REQUIRE(index != resp.indexes.end()); REQUIRE(index->dataverse_name == dataverse_name); REQUIRE(index->dataset_name == dataset_name); @@ -3882,8 +3933,9 @@ TEST_CASE("integration: analytics index management with public API", "[integrati REQUIRE_SUCCESS(ctx.ec()); REQUIRE_FALSE(res.empty()); - auto index = std::find_if( - res.begin(), res.end(), [&index_name](const couchbase::management::analytics_index& idx) { return idx.name == index_name; }); + auto index = std::find_if(res.begin(), res.end(), [&index_name](const couchbase::management::analytics_index& idx) { + return idx.name == index_name; + }); REQUIRE(index != res.end()); REQUIRE(index->dataverse_name == dataverse_name); REQUIRE(index->dataset_name == dataset_name); @@ -3894,9 +3946,19 @@ TEST_CASE("integration: analytics index management with public API", "[integrati // Getting unexpected result in 6.6 auto [ctx, res] = mgr.get_pending_mutations({}).get(); REQUIRE_SUCCESS(ctx.ec()); - REQUIRE(res.count(dataverse_name) == 1); - REQUIRE(res[dataverse_name].count(dataset_name) == 1); - REQUIRE(res[dataverse_name][dataset_name] >= 0); + if (res.count(dataverse_name) == 0 && integration.cluster_version().major == 7 && integration.cluster_version().minor == 0) { + fmt::print("Cluster {}.{}.{}, dataverse_name: {}, content: {}. Allow pending mutation to be empty\n", + integration.cluster_version().major, + integration.cluster_version().minor, + integration.cluster_version().micro, + dataverse_name, + ctx.content()); + } else { + INFO(fmt::format("dataverse_name: {}\ncontent: {}", dataverse_name, ctx.content())); + REQUIRE(res.count(dataverse_name) == 1); + REQUIRE(res[dataverse_name].count(dataset_name) == 1); + REQUIRE(res[dataverse_name][dataset_name] >= 0); + } } { @@ -4339,7 +4401,8 @@ TEST_CASE("integration: search index management", "[integration]") SECTION("search indexes crud") { - auto index1_name = test::utils::uniq_id("index1"); + auto index1_base_name = test::utils::uniq_id("index1"); + auto index1_name = index1_base_name; auto index2_name = test::utils::uniq_id("index2"); auto alias_name = test::utils::uniq_id("alias"); @@ -4356,11 +4419,14 @@ TEST_CASE("integration: search index management", "[integration]") req.index = index; auto resp = test::utils::execute(integration.cluster, req); REQUIRE_SUCCESS(resp.ctx.ec); + if (resp.name != index1_name) { + index1_name = resp.name; + } } { couchbase::core::management::search::index index; - index.name = index1_name; + index.name = index1_base_name; index.type = "fulltext-index"; index.source_type = "couchbase"; index.source_name = integration.ctx.bucket; @@ -4400,8 +4466,15 @@ TEST_CASE("integration: search index management", "[integration]") index.name = alias_name; index.type = "fulltext-alias"; index.source_type = "nil"; - index.params_json = couchbase::core::utils::json::generate( - tao::json::value{ { "targets", { { index1_name, tao::json::empty_object }, { index2_name, tao::json::empty_object } } } }); + index.params_json = couchbase::core::utils::json::generate(tao::json::value{ + { + "targets", + { + { index1_name, tao::json::empty_object }, + { index2_name, tao::json::empty_object }, + }, + }, + }); if (integration.cluster_version().is_serverless_config_profile()) { index.plan_params_json = serverless_plan_params; } @@ -4409,6 +4482,9 @@ TEST_CASE("integration: search index management", "[integration]") req.index = index; auto resp = test::utils::execute(integration.cluster, req); REQUIRE_SUCCESS(resp.ctx.ec); + if (resp.name != alias_name) { + alias_name = resp.name; + } } { @@ -4451,12 +4527,15 @@ TEST_CASE("integration: search index management", "[integration]") REQUIRE_SUCCESS(resp.ctx.ec); REQUIRE_FALSE(resp.indexes.empty()); - REQUIRE(1 == std::count_if( - resp.indexes.begin(), resp.indexes.end(), [&index1_name](const auto& i) { return i.name == index1_name; })); - REQUIRE(1 == std::count_if( - resp.indexes.begin(), resp.indexes.end(), [&index2_name](const auto& i) { return i.name == index2_name; })); - REQUIRE(1 == - std::count_if(resp.indexes.begin(), resp.indexes.end(), [&alias_name](const auto& i) { return i.name == alias_name; })); + REQUIRE(1 == std::count_if(resp.indexes.begin(), resp.indexes.end(), [&index1_name](const auto& i) { + return i.name == index1_name; + })); + REQUIRE(1 == std::count_if(resp.indexes.begin(), resp.indexes.end(), [&index2_name](const auto& i) { + return i.name == index2_name; + })); + REQUIRE(1 == std::count_if(resp.indexes.begin(), resp.indexes.end(), [&alias_name](const auto& i) { + return i.name == alias_name; + })); } { @@ -4631,7 +4710,9 @@ TEST_CASE("integration: search index management public API", "[integration]") auto [ctx, indexes] = c.search_indexes().get_all_indexes().get(); REQUIRE_SUCCESS(ctx.ec()); REQUIRE_FALSE(indexes.empty()); - REQUIRE(1 == std::count_if(indexes.begin(), indexes.end(), [&index_name](const auto& i) { return i.name == index_name; })); + REQUIRE(1 == std::count_if(indexes.begin(), indexes.end(), [&index_name](const auto& i) { + return i.name == index_name; + })); } } SECTION("control") @@ -4707,18 +4788,22 @@ TEST_CASE("integration: search index management analyze document", "[integration req.index = index; auto resp = test::utils::execute(integration.cluster, req); REQUIRE_SUCCESS(resp.ctx.ec); + index_name = resp.name; } REQUIRE(test::utils::wait_for_search_pindexes_ready(integration.cluster, integration.ctx.bucket, index_name)); couchbase::core::operations::management::search_index_analyze_document_response resp; - bool operation_completed = test::utils::wait_until([&integration, &index_name, &resp]() { - couchbase::core::operations::management::search_index_analyze_document_request req{}; - req.index_name = index_name; - req.encoded_document = R"({ "name": "hello world" })"; - resp = test::utils::execute(integration.cluster, req); - return resp.ctx.ec != couchbase::errc::common::internal_server_failure; - }); + bool operation_completed = test::utils::wait_until( + [&integration, &index_name, &resp]() { + couchbase::core::operations::management::search_index_analyze_document_request req{}; + req.index_name = index_name; + req.encoded_document = R"({ "name": "hello world" })"; + resp = test::utils::execute(integration.cluster, req); + return resp.ctx.ec != couchbase::errc::common::internal_server_failure; + }, + std::chrono::minutes{ 5 }, + std::chrono::seconds{ 1 }); REQUIRE(operation_completed); REQUIRE_SUCCESS(resp.ctx.ec); REQUIRE_FALSE(resp.analysis.empty()); @@ -4816,7 +4901,9 @@ TEST_CASE("integration: scope search index management public API", "[integration auto [ctx, indexes] = manager.get_all_indexes().get(); REQUIRE_SUCCESS(ctx.ec()); REQUIRE_FALSE(indexes.empty()); - REQUIRE(1 == std::count_if(indexes.begin(), indexes.end(), [&index_name](const auto& i) { return i.name == index_name; })); + REQUIRE(1 == std::count_if(indexes.begin(), indexes.end(), [&index_name](const auto& i) { + return i.name == index_name; + })); } } SECTION("control") @@ -4901,6 +4988,7 @@ TEST_CASE("integration: scope search index management analyze document public AP return result.first.ec() != couchbase::errc::common::internal_server_failure; }); REQUIRE(operation_completed); + INFO(result.first.content()); REQUIRE_SUCCESS(result.first.ec()); REQUIRE_FALSE(result.second.empty()); @@ -5092,3 +5180,236 @@ TEST_CASE("integration: freeform HTTP request", "[integration]") REQUIRE(result.is_array()); } } + +static bool +wait_for_function_reach_status(test::utils::integration_test_guard& integration, + const std::string& function_name, + couchbase::core::management::eventing::function_status status) +{ + return test::utils::wait_until( + [&integration, function_name, status]() { + couchbase::core::operations::management::eventing_get_status_request req{}; + auto resp = test::utils::execute(integration.cluster, req); + if (resp.ctx.ec) { + return false; + } + auto function = std::find_if(resp.status.functions.begin(), resp.status.functions.end(), [function_name](const auto& fun) { + return function_name == fun.name; + }); + if (function == resp.status.functions.end()) { + return false; + } + return function->status == status; + }, + std::chrono::minutes(3)); +} + +TEST_CASE("integration: eventing functions management", "[integration]") +{ + test::utils::integration_test_guard integration; + + if (!integration.cluster_version().supports_eventing_functions()) { + SKIP("cluster does not support eventing service"); + } + if (!integration.has_eventing_service()) { + SKIP("cluster does not have eventing service"); + } + + if (!integration.cluster_version().supports_gcccp()) { + test::utils::open_bucket(integration.cluster, integration.ctx.bucket); + } + + SECTION("lifecycle") + { + auto function_name = test::utils::uniq_id("name"); + + { + couchbase::core::operations::management::eventing_drop_function_request req{ function_name }; + auto resp = test::utils::execute(integration.cluster, req); + if (integration.cluster_version().is_cheshire_cat()) { + REQUIRE(resp.ctx.ec == couchbase::errc::management::eventing_function_not_deployed); + } else { + REQUIRE(resp.ctx.ec == couchbase::errc::management::eventing_function_not_found); + } + } + + { + couchbase::core::operations::management::eventing_get_function_request req{ function_name }; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE(resp.ctx.ec == couchbase::errc::management::eventing_function_not_found); + } + + auto meta_bucket_name = test::utils::uniq_id("meta"); + { + + couchbase::core::management::cluster::bucket_settings bucket_settings; + bucket_settings.name = meta_bucket_name; + bucket_settings.ram_quota_mb = 256; + + { + couchbase::core::operations::management::bucket_create_request req; + req.bucket = bucket_settings; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec); + } + } + + { + REQUIRE(wait_for_bucket_created(integration, meta_bucket_name)); + } + + std::string source_code = R"( +function OnUpdate(doc, meta) { + log("Doc created/updated", meta.id); +} + +function OnDelete(meta, options) { + log("Doc deleted/expired", meta.id); +} +)"; + + { + couchbase::core::operations::management::eventing_upsert_function_request req{}; + req.function.source_keyspace.bucket = integration.ctx.bucket; + req.function.metadata_keyspace.bucket = meta_bucket_name; + req.function.name = function_name; + req.function.code = source_code; + req.function.settings.handler_headers = { "// generated by Couchbase C++ SDK" }; + req.function.constant_bindings.emplace_back(couchbase::core::management::eventing::function_constant_binding{ "PI", "3.14" }); + req.function.bucket_bindings.emplace_back(couchbase::core::management::eventing::function_bucket_binding{ + "data", { integration.ctx.bucket }, couchbase::core::management::eventing::function_bucket_access::read_write }); + req.function.url_bindings.emplace_back( + couchbase::core::management::eventing::function_url_binding{ "home", "https://couchbase.com" }); + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec); + } + + { + REQUIRE(test::utils::wait_for_function_created(integration.cluster, function_name)); + auto resp = test::utils::execute(integration.cluster, + couchbase::core::operations::management::eventing_get_function_request{ + function_name, + }); + REQUIRE_SUCCESS(resp.ctx.ec); + } + + { + couchbase::core::operations::management::eventing_get_all_functions_request req{}; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec); + auto function = std::find_if(resp.functions.begin(), resp.functions.end(), [&function_name](const auto& fun) { + return function_name == fun.name; + }); + REQUIRE(function != resp.functions.end()); + REQUIRE(function->code == source_code); + REQUIRE(function->source_keyspace.bucket == integration.ctx.bucket); + REQUIRE(function->metadata_keyspace.bucket == meta_bucket_name); + REQUIRE(function->settings.deployment_status == couchbase::core::management::eventing::function_deployment_status::undeployed); + REQUIRE(function->settings.processing_status == couchbase::core::management::eventing::function_processing_status::paused); + REQUIRE(!function->settings.handler_headers.empty()); + REQUIRE(function->settings.handler_headers[0] == "// generated by Couchbase C++ SDK"); + REQUIRE(!function->constant_bindings.empty()); + REQUIRE(function->constant_bindings[0].alias == "PI"); + REQUIRE(function->constant_bindings[0].literal == "3.14"); + REQUIRE(!function->bucket_bindings.empty()); + REQUIRE(function->bucket_bindings[0].alias == "data"); + REQUIRE(function->bucket_bindings[0].name.bucket == "default"); + REQUIRE(function->bucket_bindings[0].access == couchbase::core::management::eventing::function_bucket_access::read_write); + REQUIRE(!function->url_bindings.empty()); + REQUIRE(function->url_bindings[0].alias == "home"); + REQUIRE(function->url_bindings[0].hostname == "https://couchbase.com"); + REQUIRE(std::holds_alternative(function->url_bindings[0].auth)); + } + + { + couchbase::core::operations::management::eventing_get_status_request req{}; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec); + REQUIRE(resp.status.num_eventing_nodes > 0); + auto function = std::find_if(resp.status.functions.begin(), resp.status.functions.end(), [&function_name](const auto& fun) { + return function_name == fun.name; + }); + REQUIRE(function != resp.status.functions.end()); + REQUIRE(function->status == couchbase::core::management::eventing::function_status::undeployed); + REQUIRE(function->deployment_status == couchbase::core::management::eventing::function_deployment_status::undeployed); + REQUIRE(function->processing_status == couchbase::core::management::eventing::function_processing_status::paused); + } + + { + couchbase::core::operations::management::eventing_undeploy_function_request req{ function_name }; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE(resp.ctx.ec == couchbase::errc::management::eventing_function_not_deployed); + } + + { + couchbase::core::operations::management::eventing_deploy_function_request req{ function_name }; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec); + } + + REQUIRE( + wait_for_function_reach_status(integration, function_name, couchbase::core::management::eventing::function_status::deployed)); + + { + couchbase::core::operations::management::eventing_drop_function_request req{ function_name }; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE(resp.ctx.ec == couchbase::errc::management::eventing_function_deployed); + } + + { + couchbase::core::operations::management::eventing_resume_function_request req{ function_name }; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE(resp.ctx.ec == couchbase::errc::management::eventing_function_deployed); + } + + { + couchbase::core::operations::management::eventing_pause_function_request req{ function_name }; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec); + } + + REQUIRE(wait_for_function_reach_status(integration, function_name, couchbase::core::management::eventing::function_status::paused)); + + { + couchbase::core::operations::management::eventing_pause_function_request req{ function_name }; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE(resp.ctx.ec == couchbase::errc::management::eventing_function_paused); + } + + { + couchbase::core::operations::management::eventing_resume_function_request req{ function_name }; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec); + } + + REQUIRE( + wait_for_function_reach_status(integration, function_name, couchbase::core::management::eventing::function_status::deployed)); + + { + couchbase::core::operations::management::eventing_undeploy_function_request req{ function_name }; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec); + } + + REQUIRE( + wait_for_function_reach_status(integration, function_name, couchbase::core::management::eventing::function_status::undeployed)); + + { + couchbase::core::operations::management::eventing_drop_function_request req{ function_name }; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec); + } + + { + couchbase::core::operations::management::eventing_get_function_request req{ function_name }; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE(resp.ctx.ec == couchbase::errc::management::eventing_function_not_found); + } + + { + couchbase::core::operations::management::bucket_drop_request req{ meta_bucket_name }; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec); + } + } +} diff --git a/test/test_integration_management_eventing.cxx b/test/test_integration_management_eventing.cxx index 2ff4f1da8..51ccf6d8e 100644 --- a/test/test_integration_management_eventing.cxx +++ b/test/test_integration_management_eventing.cxx @@ -19,32 +19,6 @@ #include "core/operations/management/eventing.hxx" #include "test_helper_integration.hxx" -static couchbase::core::operations::management::eventing_get_function_response -wait_for_function_created(test::utils::integration_test_guard& integration, - const std::string& function_name, - const std::optional& bucket_name, - const std::optional& scope_name) -{ - couchbase::core::operations::management::eventing_get_function_response resp{}; - test::utils::wait_until([&integration, &resp, function_name, bucket_name, scope_name]() { - couchbase::core::operations::management::eventing_get_function_request req{ function_name, bucket_name, scope_name }; - resp = test::utils::execute(integration.cluster, req); - if (resp.ctx.ec) { - return false; - } - // The function scope sometimes takes longer to be set correctly (especially for the admin scope). - if (bucket_name.has_value() && scope_name.has_value()) { - return resp.function.internal.bucket_name.has_value() && resp.function.internal.scope_name.has_value() && - resp.function.internal.bucket_name.value() == bucket_name.value() && - resp.function.internal.scope_name.value() == scope_name.value(); - } - return (!resp.function.internal.bucket_name.has_value() && !resp.function.internal.scope_name.has_value()) || - (resp.function.internal.bucket_name.has_value() && resp.function.internal.scope_name.has_value() && - resp.function.internal.bucket_name.value() == "*" && resp.function.internal.scope_name.value() == "*"); - }); - return resp; -} - static couchbase::core::operations::management::bucket_get_response wait_for_bucket_created(test::utils::integration_test_guard& integration, const std::string& bucket_name) { @@ -132,6 +106,11 @@ function OnDelete(meta, options) { } )"; + INFO(fmt::format("function_name: {}\nbucket_name: {}\nscope_name: {}", + function_name, + bucket_name.value_or("(not specified)"), + scope_name.value_or("(not specified)"))); + { couchbase::core::operations::management::eventing_upsert_function_request req{}; req.bucket_name = bucket_name; @@ -151,7 +130,13 @@ function OnDelete(meta, options) { } { - auto resp = wait_for_function_created(integration, function_name, bucket_name, scope_name); + REQUIRE(test::utils::wait_for_function_created(integration.cluster, function_name, bucket_name, scope_name)); + auto resp = test::utils::execute(integration.cluster, + couchbase::core::operations::management::eventing_get_function_request{ + function_name, + bucket_name, + scope_name, + }); REQUIRE_SUCCESS(resp.ctx.ec); } @@ -159,8 +144,9 @@ function OnDelete(meta, options) { couchbase::core::operations::management::eventing_get_all_functions_request req{ bucket_name, scope_name }; auto resp = test::utils::execute(integration.cluster, req); REQUIRE_SUCCESS(resp.ctx.ec); - auto function = std::find_if( - resp.functions.begin(), resp.functions.end(), [&function_name](const auto& fun) { return function_name == fun.name; }); + auto function = std::find_if(resp.functions.begin(), resp.functions.end(), [&function_name](const auto& fun) { + return function_name == fun.name; + }); REQUIRE(function != resp.functions.end()); REQUIRE(function->code == source_code); REQUIRE(function->source_keyspace.bucket == integration.ctx.bucket); @@ -262,6 +248,19 @@ function OnDelete(meta, options) { REQUIRE_SUCCESS(resp.ctx.ec); } + { + auto function_not_found = test::utils::wait_until([&]() { + auto resp = test::utils::execute(integration.cluster, + couchbase::core::operations::management::eventing_get_function_request{ + function_name, + bucket_name, + scope_name, + }); + return resp.ctx.ec == couchbase::errc::management::eventing_function_not_found; + }); + REQUIRE(function_not_found); + } + { couchbase::core::operations::management::eventing_get_function_request req{ function_name, bucket_name, scope_name }; auto resp = test::utils::execute(integration.cluster, req); @@ -369,7 +368,11 @@ function OnDelete(meta, options) { } { - auto resp = wait_for_function_created(integration, admin_function_name, {}, {}); + REQUIRE(test::utils::wait_for_function_created(integration.cluster, admin_function_name)); + auto resp = test::utils::execute(integration.cluster, + couchbase::core::operations::management::eventing_get_function_request{ + admin_function_name, + }); REQUIRE_SUCCESS(resp.ctx.ec); } @@ -394,7 +397,10 @@ function OnDelete(meta, options) { } { - auto resp = wait_for_function_created(integration, scoped_function_name, integration.ctx.bucket, "_default"); + REQUIRE(test::utils::wait_for_function_created(integration.cluster, scoped_function_name, integration.ctx.bucket, "_default")); + auto resp = test::utils::execute(integration.cluster, + couchbase::core::operations::management::eventing_get_function_request{ + scoped_function_name, integration.ctx.bucket, "_default" }); REQUIRE_SUCCESS(resp.ctx.ec); } @@ -449,17 +455,19 @@ function OnDelete(meta, options) { // The scoped function should be in the results of a scoped get_status { - auto function = std::find_if(resp.status.functions.begin(), - resp.status.functions.end(), - [&scoped_function_name](const auto& fun) { return scoped_function_name == fun.name; }); + auto function = + std::find_if(resp.status.functions.begin(), resp.status.functions.end(), [&scoped_function_name](const auto& fun) { + return scoped_function_name == fun.name; + }); REQUIRE(function != resp.status.functions.end()); } // The admin function should not be in the results of a scoped get_status { - auto function = std::find_if(resp.status.functions.begin(), - resp.status.functions.end(), - [&admin_function_name](const auto& fun) { return admin_function_name == fun.name; }); + auto function = + std::find_if(resp.status.functions.begin(), resp.status.functions.end(), [&admin_function_name](const auto& fun) { + return admin_function_name == fun.name; + }); REQUIRE(function == resp.status.functions.end()); } } @@ -471,17 +479,19 @@ function OnDelete(meta, options) { // The scoped function should not be in the results of a non-scoped get_status { - auto function = std::find_if(resp.status.functions.begin(), - resp.status.functions.end(), - [&scoped_function_name](const auto& fun) { return scoped_function_name == fun.name; }); + auto function = + std::find_if(resp.status.functions.begin(), resp.status.functions.end(), [&scoped_function_name](const auto& fun) { + return scoped_function_name == fun.name; + }); REQUIRE(function == resp.status.functions.end()); } // The admin function should be in the results of a non-scoped get_status { - auto function = std::find_if(resp.status.functions.begin(), - resp.status.functions.end(), - [&admin_function_name](const auto& fun) { return admin_function_name == fun.name; }); + auto function = + std::find_if(resp.status.functions.begin(), resp.status.functions.end(), [&admin_function_name](const auto& fun) { + return admin_function_name == fun.name; + }); REQUIRE(function != resp.status.functions.end()); } } diff --git a/test/test_integration_query.cxx b/test/test_integration_query.cxx index f40a365a6..5a082a2b2 100644 --- a/test/test_integration_query.cxx +++ b/test/test_integration_query.cxx @@ -18,6 +18,7 @@ #include "test_helper_integration.hxx" #include "utils/move_only_context.hxx" +#include "utils/wait_until.hxx" #include "core/operations/document_analytics.hxx" #include "core/operations/document_append.hxx" @@ -217,6 +218,8 @@ TEST_CASE("integration: read only with no results", "[integration]") test::utils::open_bucket(integration.cluster, integration.ctx.bucket); } + REQUIRE(test::utils::create_primary_index(integration.cluster, integration.ctx.bucket)); + { couchbase::core::operations::query_request req{ fmt::format("SELECT * FROM `{}` LIMIT 0", integration.ctx.bucket) }; auto resp = test::utils::execute(integration.cluster, req); @@ -510,6 +513,9 @@ TEST_CASE("integration: prepared query", "[integration]") } test::utils::open_bucket(integration.cluster, integration.ctx.bucket); + + REQUIRE(test::utils::create_primary_index(integration.cluster, integration.ctx.bucket)); + auto key = test::utils::uniq_id("foo"); tao::json::value value = { { "a", 1.0 }, diff --git a/test/test_integration_search.cxx b/test/test_integration_search.cxx index ada9fc7bc..aed91a3c1 100644 --- a/test/test_integration_search.cxx +++ b/test/test_integration_search.cxx @@ -64,40 +64,20 @@ TEST_CASE("integration: search query") } } - std::string index_name = test::utils::uniq_id("beer-search-index"); - - { - auto params = test::utils::read_test_data("search_beers_index_params.json"); - - couchbase::core::management::search::index index{}; - index.name = index_name; - index.params_json = params; - index.type = "fulltext-index"; - index.source_name = integration.ctx.bucket; - index.source_type = "couchbase"; - if (integration.cluster_version().requires_search_replicas()) { - index.plan_params_json = couchbase::core::utils::json::generate({ - { "indexPartitions", 1 }, - { "numReplicas", 1 }, - }); - } - couchbase::core::operations::management::search_index_upsert_request req{}; - req.index = index; - - auto resp = test::utils::execute(integration.cluster, req); - REQUIRE((!resp.ctx.ec || resp.ctx.ec == couchbase::errc::common::index_exists)); - if (index_name != resp.name) { - CB_LOG_INFO("update index name \"{}\" -> \"{}\"", index_name, resp.name); - } - index_name = resp.name; - } + std::uint64_t beer_sample_doc_count = 5; + bool completed{}; + std::string index_name{}; + std::tie(completed, index_name) = test::utils::create_search_index(integration, + integration.ctx.bucket, + test::utils::uniq_id("beer-search-index"), + "search_beers_index_params.json", + beer_sample_doc_count); + REQUIRE(completed); couchbase::core::json_string simple_query(R"({"query": "description:belgian"})"); - std::uint64_t beer_sample_doc_count = 5; // Wait until expected documents are indexed { - REQUIRE(test::utils::wait_until_indexed(integration.cluster, index_name, beer_sample_doc_count)); auto ok = test::utils::wait_until( [&]() { couchbase::core::operations::search_request req{}; @@ -466,6 +446,7 @@ TEST_CASE("integration: search query consistency", "[integration]") req.index_name = index_name; req.query = query_json; req.mutation_state.emplace_back(token); + req.timeout = std::chrono::minutes{ 5 }; auto resp = test::utils::execute(integration.cluster, req); if (resp.ctx.ec == couchbase::errc::search::consistency_mismatch) { // FIXME(MB-55920): ignore "err: bleve: pindex_consistency mismatched partition" diff --git a/test/test_transaction_context.cxx b/test/test_transaction_context.cxx index fc755cf8f..1df27be06 100644 --- a/test/test_transaction_context.cxx +++ b/test/test_transaction_context.cxx @@ -427,6 +427,10 @@ TEST_CASE("transactions: can do query", "[transactions]") { test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto cluster = integration.cluster; auto txns = integration.transactions(); @@ -463,6 +467,10 @@ TEST_CASE("transactions: can see some query errors but no transactions failed", { test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto cluster = integration.cluster; auto txns = integration.transactions(); diff --git a/test/test_transaction_public_async_api.cxx b/test/test_transaction_public_async_api.cxx index e2ba2fda5..9ebd9b49a 100644 --- a/test/test_transaction_public_async_api.cxx +++ b/test/test_transaction_public_async_api.cxx @@ -337,9 +337,12 @@ TEST_CASE("transactions public async API: can set transaction options", "[transa TEST_CASE("transactions public async API: can do mutating query", "[transactions]") { - test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto id = test::utils::uniq_id("txn"); auto c = integration.public_cluster(); auto coll = c.bucket(integration.ctx.bucket).default_collection(); @@ -365,6 +368,10 @@ TEST_CASE("transactions public async API: some query errors rollback", "[transac { test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto id = test::utils::uniq_id("txn"); auto id2 = test::utils::uniq_id("txn"); auto c = integration.public_cluster(); diff --git a/test/test_transaction_public_blocking_api.cxx b/test/test_transaction_public_blocking_api.cxx index 7c685e1bd..f9cc53a11 100644 --- a/test/test_transaction_public_blocking_api.cxx +++ b/test/test_transaction_public_blocking_api.cxx @@ -169,7 +169,7 @@ TEST_CASE("transactions public blocking API: can insert", "[transactions]") REQUIRE(final_doc.content_as() == content); } -TEST_CASE("transactions public blocking API: insert has error as expected when doc already exists", "[transactions]") +TEST_CASE("transactions public blocking API: insert has error when doc already exists", "[transactions]") { test::utils::integration_test_guard integration; @@ -231,7 +231,6 @@ TEST_CASE("transactions public blocking API: can replace", "[transactions]") TEST_CASE("transactions public blocking API: replace fails as expected with bad cas", "[transactions]") { - test::utils::integration_test_guard integration; auto id = test::utils::uniq_id("txn"); @@ -394,6 +393,10 @@ TEST_CASE("transactions public blocking API: can do simple query", "[transaction { test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto id = test::utils::uniq_id("txn"); auto c = integration.public_cluster(); auto coll = c.bucket(integration.ctx.bucket).default_collection(); @@ -415,6 +418,10 @@ TEST_CASE("transactions public blocking API: can do simple mutating query", "[tr { test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto id = test::utils::uniq_id("txn"); auto c = integration.public_cluster(); auto coll = c.bucket(integration.ctx.bucket).default_collection(); @@ -438,6 +445,10 @@ TEST_CASE("transactions public blocking API: some query errors don't force rollb { test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto id = test::utils::uniq_id("txn"); auto c = integration.public_cluster(); auto coll = c.bucket(integration.ctx.bucket).default_collection(); @@ -463,6 +474,10 @@ TEST_CASE("transactions public blocking API: some query errors do rollback", "[t { test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto id = test::utils::uniq_id("txn"); auto id2 = test::utils::uniq_id("txn"); auto c = integration.public_cluster(); @@ -492,6 +507,10 @@ TEST_CASE("transactions public blocking API: some query errors are seen immediat { test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto c = integration.public_cluster(); auto coll = c.bucket(integration.ctx.bucket).default_collection(); @@ -513,6 +532,10 @@ TEST_CASE("transactions public blocking API: can query from a scope", "[transact const std::string new_coll_name("newcoll"); test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto id = test::utils::uniq_id("txn"); auto c = integration.public_cluster(); diff --git a/test/test_transaction_simple.cxx b/test/test_transaction_simple.cxx index e12b1ed83..d6337636e 100644 --- a/test/test_transaction_simple.cxx +++ b/test/test_transaction_simple.cxx @@ -153,7 +153,7 @@ TEST_CASE("transactions: can use custom metadata collections per transactions", REQUIRE_SUCCESS(resp.ctx.ec()); } couchbase::transactions::transaction_options cfg; - cfg.metadata_collection(couchbase::transactions::transaction_keyspace("secBucket")); + cfg.metadata_collection(couchbase::transactions::transaction_keyspace(integration.ctx.other_bucket)); txn->run(cfg, [id](attempt_context& ctx) { auto doc = ctx.get(id); auto new_content = doc.content(); @@ -178,7 +178,7 @@ TEST_CASE("transactions: can use custom metadata collections", "[transactions]") test::utils::integration_test_guard integration; auto cluster = integration.cluster; couchbase::core::document_id id{ integration.ctx.bucket, "_default", "_default", test::utils::uniq_id("txn") }; - auto cfg = get_conf().metadata_collection(couchbase::transactions::transaction_keyspace("secBucket")); + auto cfg = get_conf().metadata_collection(couchbase::transactions::transaction_keyspace(integration.ctx.other_bucket)); auto [ec, txn] = couchbase::core::transactions::transactions::create(cluster, cfg).get(); REQUIRE_SUCCESS(ec); @@ -265,10 +265,10 @@ TEST_CASE("transactions: non existent collection in custom metadata collections" { test::utils::integration_test_guard integration; auto cluster = integration.cluster; - auto cfg = - get_conf() - .metadata_collection(couchbase::transactions::transaction_keyspace{ "secBucket", couchbase::scope::default_name, "i_dont_exist" }) - .cleanup_config(couchbase::transactions::transactions_cleanup_config().cleanup_lost_attempts(true)); + auto cfg = get_conf() + .metadata_collection(couchbase::transactions::transaction_keyspace{ + integration.ctx.other_bucket, couchbase::scope::default_name, "i_dont_exist" }) + .cleanup_config(couchbase::transactions::transactions_cleanup_config().cleanup_lost_attempts(true)); cfg.timeout(std::chrono::seconds(2)); auto [ec, txn] = couchbase::core::transactions::transactions::create(cluster, cfg).get(); REQUIRE_SUCCESS(ec); @@ -350,6 +350,11 @@ TEST_CASE("transactions: quoted std::strings end up with 2 quotes (that's bad)", TEST_CASE("transactions: query error can be handled", "[transactions]") { test::utils::integration_test_guard integration; + + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto cluster = integration.cluster; auto txn = integration.transactions(); txn->run([](attempt_context& ctx) { @@ -363,6 +368,11 @@ TEST_CASE("transactions: query error can be handled", "[transactions]") TEST_CASE("transactions: unhandled query error fails transaction", "[transactions]") { test::utils::integration_test_guard integration; + + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto cluster = integration.cluster; auto txn = integration.transactions(); REQUIRE_THROWS_AS( @@ -378,6 +388,11 @@ TEST_CASE("transactions: unhandled query error fails transaction", "[transaction TEST_CASE("transactions: query mode get optional", "[transactions]") { test::utils::integration_test_guard integration; + + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto cluster = integration.cluster; auto txn = integration.transactions(); @@ -548,6 +563,11 @@ TEST_CASE("transactions: can rollback replace", "[transactions]") TEST_CASE("transactions: can have trivial query in transaction", "[transactions]") { test::utils::integration_test_guard integration; + + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto cluster = integration.cluster; auto txn = integration.transactions(); @@ -571,6 +591,11 @@ TEST_CASE("transactions: can have trivial query in transaction", "[transactions] TEST_CASE("transactions: can modify doc in query", "[transactions]") { test::utils::integration_test_guard integration; + + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto cluster = integration.cluster; auto txn = integration.transactions(); @@ -633,6 +658,11 @@ TEST_CASE("transactions: can rollback", "[transactions]") TEST_CASE("transactions: query updates insert", "[transactions]") { test::utils::integration_test_guard integration; + + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto cluster = integration.cluster; auto txn = integration.transactions(); @@ -655,6 +685,11 @@ TEST_CASE("transactions: query updates insert", "[transactions]") TEST_CASE("transactions: can KV get", "[transactions]") { test::utils::integration_test_guard integration; + + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto cluster = integration.cluster; auto txn = integration.transactions(); @@ -679,6 +714,11 @@ TEST_CASE("transactions: can KV get", "[transactions]") TEST_CASE("transactions: can KV insert", "[transactions]") { test::utils::integration_test_guard integration; + + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto cluster = integration.cluster; auto txn = integration.transactions(); @@ -701,6 +741,10 @@ TEST_CASE("transactions: can KV insert", "[transactions]") TEST_CASE("transactions: can rollback KV insert", "[transactions]") { test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto cluster = integration.cluster; auto txn = integration.transactions(); @@ -726,6 +770,11 @@ TEST_CASE("transactions: can rollback KV insert", "[transactions]") TEST_CASE("transactions: can KV replace", "[transactions]") { test::utils::integration_test_guard integration; + + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto cluster = integration.cluster; auto txn = integration.transactions(); @@ -759,6 +808,10 @@ TEST_CASE("transactions: can KV replace", "[transactions]") TEST_CASE("transactions: can rollback KV replace", "[transactions]") { test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto cluster = integration.cluster; auto txn = integration.transactions(); @@ -797,6 +850,11 @@ TEST_CASE("transactions: can rollback KV replace", "[transactions]") TEST_CASE("transactions: can KV remove", "[transactions]") { test::utils::integration_test_guard integration; + + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto cluster = integration.cluster; auto txn = integration.transactions(); @@ -825,6 +883,10 @@ TEST_CASE("transactions: can KV remove", "[transactions]") TEST_CASE("transactions: can rollback KV remove", "[transactions]") { test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto cluster = integration.cluster; auto txn = integration.transactions(); @@ -859,6 +921,10 @@ TEST_CASE("transactions: can rollback KV remove", "[transactions]") TEST_CASE("transactions: can rollback retry bad KV replace", "[transactions]") { test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto cluster = integration.cluster; auto txn = integration.transactions(); @@ -947,6 +1013,13 @@ TEST_CASE("transactions: get after query behaves same as before a query", "[tran TEST_CASE("transactions: get_optional after query behaves same as before a query", "[transactions]") { test::utils::integration_test_guard integration; + + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + + REQUIRE(test::utils::create_primary_index(integration.cluster, integration.ctx.bucket)); + auto cluster = integration.cluster; auto txn = integration.transactions(); couchbase::core::document_id id{ integration.ctx.bucket, "_default", "_default", test::utils::uniq_id("txn") }; @@ -955,9 +1028,17 @@ TEST_CASE("transactions: get_optional after query behaves same as before a query ctx.get_optional(id); })); } + TEST_CASE("transactions: sergey example", "[transactions]") { test::utils::integration_test_guard integration; + + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + + REQUIRE(test::utils::create_primary_index(integration.cluster, integration.ctx.bucket)); + auto cluster = integration.cluster; auto txn = integration.transactions(); couchbase::core::document_id id_to_remove{ integration.ctx.bucket, "_default", "_default", test::utils::uniq_id("txn") }; diff --git a/test/test_transaction_simple_async.cxx b/test/test_transaction_simple_async.cxx index 5b1c69551..a248cd825 100644 --- a/test/test_transaction_simple_async.cxx +++ b/test/test_transaction_simple_async.cxx @@ -91,7 +91,7 @@ TEST_CASE("transactions: can't get from unopened bucket", "[transactions]") test::utils::integration_test_guard integration; auto txn = integration.transactions(); - couchbase::core::document_id bad_id{ "secBucket", "_default", "default", test::utils::uniq_id("txns") }; + couchbase::core::document_id bad_id{ integration.ctx.other_bucket, "_default", "default", test::utils::uniq_id("txns") }; auto cb_called = std::make_shared>(false); auto barrier = std::make_shared>(); auto f = barrier->get_future(); @@ -441,6 +441,10 @@ TEST_CASE("transactions: async query", "[transactions]") { test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto txn = integration.transactions(); test::utils::open_bucket(integration.cluster, integration.ctx.bucket); @@ -482,6 +486,10 @@ TEST_CASE("transactions: multiple racing queries", "[transactions]") { test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto txn = integration.transactions(); test::utils::open_bucket(integration.cluster, integration.ctx.bucket); @@ -534,6 +542,10 @@ TEST_CASE("transactions: rollback async query", "[transactions]") { test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto txn = integration.transactions(); test::utils::open_bucket(integration.cluster, integration.ctx.bucket); @@ -576,9 +588,12 @@ TEST_CASE("transactions: rollback async query", "[transactions]") TEST_CASE("transactions: async KV get", "[transactions]") { - test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto txn = integration.transactions(); test::utils::open_bucket(integration.cluster, integration.ctx.bucket); @@ -626,6 +641,10 @@ TEST_CASE("transactions: rollback async KV get", "[transactions]") { test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto txn = integration.transactions(); test::utils::open_bucket(integration.cluster, integration.ctx.bucket); @@ -673,6 +692,10 @@ TEST_CASE("transactions: async KV insert", "[transactions]") { test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto txn = integration.transactions(); test::utils::open_bucket(integration.cluster, integration.ctx.bucket); @@ -710,6 +733,10 @@ TEST_CASE("transactions: rollback async KV insert", "[transactions]") { test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto txn = integration.transactions(); test::utils::open_bucket(integration.cluster, integration.ctx.bucket); @@ -748,6 +775,10 @@ TEST_CASE("transactions: async KV replace", "[transactions]") { test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto txn = integration.transactions(); test::utils::open_bucket(integration.cluster, integration.ctx.bucket); @@ -806,6 +837,10 @@ TEST_CASE("transactions: rollback async KV replace", "[transactions]") { test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto txn = integration.transactions(); test::utils::open_bucket(integration.cluster, integration.ctx.bucket); @@ -865,6 +900,10 @@ TEST_CASE("transactions: async KV remove", "[transactions]") test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto txn = integration.transactions(); test::utils::open_bucket(integration.cluster, integration.ctx.bucket); @@ -918,6 +957,10 @@ TEST_CASE("transactions: rollback async KV remove", "[transactions]") { test::utils::integration_test_guard integration; + if (!integration.cluster_version().supports_queries_in_transactions()) { + SKIP("the server does not support queries inside transactions"); + } + auto txn = integration.transactions(); test::utils::open_bucket(integration.cluster, integration.ctx.bucket); diff --git a/test/test_unit_utils.cxx b/test/test_unit_utils.cxx index 9b1ffc3cc..93e7925b8 100644 --- a/test/test_unit_utils.cxx +++ b/test/test_unit_utils.cxx @@ -28,6 +28,7 @@ #include "core/utils/movable_function.hxx" #include "core/utils/url_codec.hxx" +#include #include #include diff --git a/test/utils/integration_test_guard.cxx b/test/utils/integration_test_guard.cxx index 637b234f1..f9f96d849 100644 --- a/test/utils/integration_test_guard.cxx +++ b/test/utils/integration_test_guard.cxx @@ -66,7 +66,7 @@ build_origin(const test_context& ctx, ctx.dns_nameserver.value_or(couchbase::core::io::dns::dns_config::default_nameserver), ctx.dns_port.value_or(couchbase::core::io::dns::dns_config::default_port), }; - if (ctx.deployment == deployment_type::capella || ctx.deployment == test::utils::deployment_type::elixir) { + if (ctx.use_wan_development_profile) { origin.options().apply_profile("wan_development"); } return origin; diff --git a/test/utils/server_version.hxx b/test/utils/server_version.hxx index 670a54f01..8bfee387f 100644 --- a/test/utils/server_version.hxx +++ b/test/utils/server_version.hxx @@ -91,6 +91,11 @@ struct server_version { return is_cheshire_cat() || is_neo(); } + [[nodiscard]] bool supports_queries_in_transactions() const + { + return is_neo(); + } + [[nodiscard]] bool supports_collections() const { return (is_mad_hatter() && developer_preview) || is_cheshire_cat() || is_neo(); diff --git a/test/utils/test_context.cxx b/test/utils/test_context.cxx index df4c4ec58..dc70c32c3 100644 --- a/test/utils/test_context.cxx +++ b/test/utils/test_context.cxx @@ -100,6 +100,19 @@ test_context::load_from_environment() } } + if (auto var = spdlog::details::os::getenv("TEST_USE_WAN_DEVELOPMENT_PROFILE"); !var.empty()) { + if (var == "true" || var == "yes" || var == "1") { + ctx.use_wan_development_profile = true; + } else if (var == "false" || var == "no" || var == "0") { + ctx.use_wan_development_profile = false; + } + } + + // Always use WAN profile for Capella or Elixir setups + if (ctx.deployment == deployment_type::capella || ctx.deployment == test::utils::deployment_type::elixir) { + ctx.use_wan_development_profile = true; + } + return ctx; } diff --git a/test/utils/test_context.hxx b/test/utils/test_context.hxx index 0886ae6f6..b19c111c8 100644 --- a/test/utils/test_context.hxx +++ b/test/utils/test_context.hxx @@ -39,6 +39,7 @@ struct test_context { std::optional dns_port{}; std::size_t number_of_io_threads{ 1 }; std::string other_bucket{ "secBucket" }; + bool use_wan_development_profile{ false }; [[nodiscard]] couchbase::core::cluster_credentials build_auth() const; diff --git a/test/utils/test_data.cxx b/test/utils/test_data.cxx index a87edaaf9..f3a259f51 100644 --- a/test/utils/test_data.cxx +++ b/test/utils/test_data.cxx @@ -15,11 +15,13 @@ * limitations under the License. */ +#include "core/mcbp/big_endian.hxx" + +#include + #include -#include +#include #include -#include -#include #include namespace test::utils @@ -30,12 +32,38 @@ uniq_id(const std::string& prefix) return fmt::format("{}_{}", prefix, std::chrono::steady_clock::now().time_since_epoch().count()); } +namespace +{ +auto +read_all(const std::string& path) -> std::string +{ + const auto file_size = std::filesystem::file_size(path); + std::ifstream input_file(path); + std::string content; + content.resize(file_size); + input_file.read(content.data(), static_cast(file_size)); + return content; +} +} // namespace + std::string read_test_data(const std::string& file) { - auto ss = std::ostringstream{}; - std::ifstream input_file(fmt::format("../../test/data/{}", file)); - ss << input_file.rdbuf(); - return ss.str(); + std::vector candidates{ + file, + fmt::format("data/{}", file), + fmt::format("test/data/{}", file), + fmt::format("../test/data/{}", file), + fmt::format("../../test/data/{}", file), + fmt::format("../../../test/data/{}", file), + }; + for (const auto& path : candidates) { + if (std::error_code ec{}; std::filesystem::exists(path, ec) && !ec) { + return read_all(path); + } + } + throw std::runtime_error(fmt::format("unable to load test_data.\nCurrent directory: {}\ncandidates: {}", + std::filesystem::current_path().string(), + fmt::join(candidates, ",\n\t"))); } } // namespace test::utils diff --git a/test/utils/wait_until.cxx b/test/utils/wait_until.cxx index c67ffed51..d3afc48e6 100644 --- a/test/utils/wait_until.cxx +++ b/test/utils/wait_until.cxx @@ -17,11 +17,20 @@ #include "wait_until.hxx" +#include "integration_test_guard.hxx" +#include "test_data.hxx" + #include "core/logger/logger.hxx" +#include "core/management/eventing_function.hxx" #include "core/operations/management/bucket_get.hxx" #include "core/operations/management/collections_manifest_get.hxx" +#include "core/operations/management/eventing_get_function.hxx" +#include "core/operations/management/freeform.hxx" +#include "core/operations/management/query_index_create.hxx" #include "core/operations/management/search_get_stats.hxx" +#include "core/operations/management/search_index_drop.hxx" #include "core/operations/management/search_index_get_documents_count.hxx" +#include "core/operations/management/search_index_upsert.hxx" #include "core/topology/collections_manifest_fmt.hxx" #include "core/utils/json.hxx" @@ -78,6 +87,8 @@ wait_until_collection_manifest_propagated(const couchbase::core::cluster& cluste std::this_thread::sleep_for(std::chrono::seconds{ 1 }); return propagated; } + } else { + round = 0; } } return false; @@ -106,7 +117,9 @@ wait_until_cluster_connected(const std::string& username, const std::string& pas auto connected = test::utils::wait_until([cluster_options, connection_string]() { asio::io_context io; auto guard = asio::make_work_guard(io); - std::thread io_thread([&io]() { io.run(); }); + std::thread io_thread([&io]() { + io.run(); + }); auto [cluster, ec] = couchbase::cluster::connect(io, connection_string, cluster_options).get(); cluster.close(); guard.reset(); @@ -129,31 +142,106 @@ to_string(std::optional value) -> std::string return "(empty)"; } +static bool +refresh_config_on_search_service(const couchbase::core::cluster& cluster) +{ + const couchbase::core::operations::management::freeform_request req{ + couchbase::core::service_type::search, + "POST", + "/api/cfgRefresh", + { + { "content-type", "application/json" }, + }, + }; + auto resp = test::utils::execute(cluster, req); + return !resp.ctx.ec; +} + +/** + * Forces the node to replan resource assignments (by running the planner, if enabled) + * and to update its runtime state to reflect the latest plan (by running the janitor, + * if enabled). + */ +static bool +kick_manager_manager_on_search_service(const couchbase::core::cluster& cluster) +{ + const couchbase::core::operations::management::freeform_request req{ + couchbase::core::service_type::search, + "POST", + "/api/managerKick", + { + { "content-type", "application/json" }, + }, + }; + auto resp = test::utils::execute(cluster, req); + return !resp.ctx.ec; +} + +static bool +starts_with(const std::string& str, const std::string& prefix) +{ + if (str.length() < prefix.length()) { + return false; + } + return str.compare(0, prefix.length(), prefix) == 0; +} + +static bool +ends_with(const std::string& str, const std::string& suffix) +{ + if (str.length() < suffix.length()) { + return false; + } + return str.compare(str.length() - suffix.length(), suffix.length(), suffix) == 0; +} + bool wait_for_search_pindexes_ready(const couchbase::core::cluster& cluster, const std::string& bucket_name, const std::string& index_name) { return test::utils::wait_until( [&]() { + if (!refresh_config_on_search_service(cluster)) { + return false; + } + couchbase::core::operations::management::search_get_stats_request req{}; auto resp = test::utils::execute(cluster, req); if (resp.ctx.ec || resp.stats.empty()) { return false; } auto stats = couchbase::core::utils::json::parse(resp.stats); - auto num_pindexes_target = stats.optional(fmt::format("{}:{}:num_pindexes_target", bucket_name, index_name)); - auto num_pindexes_actual = stats.optional(fmt::format("{}:{}:num_pindexes_actual", bucket_name, index_name)); + + std::optional num_pindexes_target{}; + std::optional num_pindexes_actual{}; + + auto target_suffix = fmt::format("{}:num_pindexes_target", index_name); + auto actual_suffix = fmt::format("{}:num_pindexes_actual", index_name); + for (const auto& [key, value] : stats.get_object()) { + if (starts_with(key, bucket_name) && ends_with(key, target_suffix)) { + num_pindexes_target = value.as(); + } + if (starts_with(key, bucket_name) && ends_with(key, actual_suffix)) { + num_pindexes_actual = value.as(); + } + } CB_LOG_INFO("wait_for_search_pindexes_ready for \"{}\", target: {}, actual: {}", index_name, to_string(num_pindexes_target), to_string(num_pindexes_actual)); + if (!num_pindexes_actual || !num_pindexes_target) { + kick_manager_manager_on_search_service(cluster); + return false; + } - if (num_pindexes_actual && num_pindexes_target) { - return num_pindexes_actual.value() == num_pindexes_target.value(); + if (num_pindexes_target == 0) { + kick_manager_manager_on_search_service(cluster); + return false; } - return false; + return num_pindexes_actual.value() == num_pindexes_target.value(); }, - std::chrono::minutes(5)); + std::chrono::minutes(5), + std::chrono::seconds{ 1 }); } bool @@ -161,6 +249,10 @@ wait_until_indexed(const couchbase::core::cluster& cluster, const std::string& i { return test::utils::wait_until( [cluster = std::move(cluster), &index_name, &expected_count]() { + if (!refresh_config_on_search_service(cluster)) { + return false; + } + couchbase::core::operations::management::search_index_get_documents_count_request req{}; req.index_name = index_name; req.timeout = std::chrono::seconds{ 1 }; @@ -168,7 +260,140 @@ wait_until_indexed(const couchbase::core::cluster& cluster, const std::string& i CB_LOG_INFO("wait_until_indexed for \"{}\", expected: {}, actual: {}", index_name, expected_count, resp.count); return resp.count >= expected_count; }, - std::chrono::minutes(5)); + std::chrono::minutes(10), + std::chrono::seconds{ 5 }); +} + +bool +create_primary_index(const couchbase::core::cluster& cluster, const std::string& bucket_name) +{ + couchbase::core::operations::management::query_index_create_response resp; + bool operation_completed = wait_until([&cluster, &bucket_name, &resp]() { + couchbase::core::operations::management::query_index_create_request req{}; + req.bucket_name = bucket_name; + req.ignore_if_exists = true; + req.is_primary = true; + resp = execute(cluster, req); + if (resp.ctx.ec) { + CB_LOG_INFO("create_primary_index for \"{}\", rc: {}, body:\n{}", bucket_name, resp.ctx.ec.message(), resp.ctx.http_body); + } + return resp.ctx.ec != couchbase::errc::common::bucket_not_found && resp.ctx.ec != couchbase::errc::common::scope_not_found; + }); + if (resp.ctx.ec) { + CB_LOG_ERROR( + "failed to create primary index for \"{}\", rc: {}, body:\n{}", bucket_name, resp.ctx.ec.message(), resp.ctx.http_body); + return false; + } + return operation_completed; +} + +std::pair +create_search_index(integration_test_guard& integration, + const std::string& bucket_name, + const std::string& index_name, + const std::string& index_params_file_name, + std::size_t expected_number_of_documents_indexed) +{ + auto params = read_test_data(index_params_file_name); + + couchbase::core::operations::management::search_index_upsert_response resp{}; + + bool operation_completed = wait_until([&integration, bucket_name, index_name, params, &resp]() { + couchbase::core::management::search::index index{}; + index.name = index_name; + index.params_json = params; + index.type = "fulltext-index"; + index.source_name = bucket_name; + index.source_type = "couchbase"; + if (integration.cluster_version().requires_search_replicas()) { + index.plan_params_json = couchbase::core::utils::json::generate({ + { "indexPartitions", 1 }, + { "numReplicas", 1 }, + }); + } + couchbase::core::operations::management::search_index_upsert_request req{}; + req.index = index; + resp = execute(integration.cluster, req); + + if (resp.ctx.ec) { + CB_LOG_INFO("create_search_index bucket: \"{}\", index_name: \"{}\", rc: {}, body:\n{}", + bucket_name, + index_name, + resp.ctx.ec.message(), + resp.ctx.http_body); + } else { + if (index_name != resp.name) { + CB_LOG_INFO("update index name \"{}\" -> \"{}\"", index_name, resp.name); + } + } + return !resp.ctx.ec || resp.ctx.ec == couchbase::errc::common::index_exists; + }); + + CB_LOG_INFO("completed: {}, index_name \"{}\" -> \"{}\", ec: {}", operation_completed, index_name, resp.name, resp.ctx.ec.message()); + if (!operation_completed) { + return { false, "" }; + } + + std::string actual_index_name{ index_name }; + if (!resp.ctx.ec) { + actual_index_name = resp.name; + } + + operation_completed = wait_until_indexed(integration.cluster, index_name, expected_number_of_documents_indexed); + + return { operation_completed, actual_index_name }; +} + +bool +wait_for_function_created(const couchbase::core::cluster& cluster, + const std::string& function_name, + const std::optional& bucket_name, + const std::optional& scope_name, + std::size_t successful_rounds, + std::chrono::seconds total_timeout) +{ + std::size_t round = 0; + auto deadline = std::chrono::system_clock::now() + total_timeout; + + couchbase::core::operations::management::eventing_get_function_response resp{}; + while (std::chrono::system_clock::now() < deadline) { + auto exists = test::utils::wait_until([&cluster, &resp, function_name, bucket_name, scope_name]() { + couchbase::core::operations::management::eventing_get_function_request req{ function_name, bucket_name, scope_name }; + resp = test::utils::execute(cluster, req); + if (resp.ctx.ec) { + return false; + } + + // The function scope sometimes takes longer to be set correctly (especially for the admin scope). + if (bucket_name.has_value() && scope_name.has_value()) { + return resp.function.internal.bucket_name.has_value() && resp.function.internal.scope_name.has_value() && + resp.function.internal.bucket_name.value() == bucket_name.value() && + resp.function.internal.scope_name.value() == scope_name.value(); + } + return (!resp.function.internal.bucket_name.has_value() && !resp.function.internal.scope_name.has_value()) || + (resp.function.internal.bucket_name.has_value() && resp.function.internal.scope_name.has_value() && + resp.function.internal.bucket_name.value() == "*" && resp.function.internal.scope_name.value() == "*"); + }); + if (exists) { + round += 1; + if (round >= successful_rounds) { + std::this_thread::sleep_for(std::chrono::seconds{ 1 }); + return exists; + } + } else { + round = 0; + } + } + return false; +} + +bool +drop_search_index(integration_test_guard& integration, const std::string& index_name) +{ + couchbase::core::operations::management::search_index_drop_request req{}; + req.index_name = index_name; + auto resp = execute(integration.cluster, req); + return !resp.ctx.ec; } } // namespace test::utils diff --git a/test/utils/wait_until.hxx b/test/utils/wait_until.hxx index 1325d1083..874b8174f 100644 --- a/test/utils/wait_until.hxx +++ b/test/utils/wait_until.hxx @@ -64,7 +64,7 @@ bool wait_until_collection_manifest_propagated(const couchbase::core::cluster& cluster, const std::string& bucket_name, std::uint64_t current_manifest_uid, - std::size_t successful_rounds = 4, + std::size_t successful_rounds = 7, std::chrono::seconds total_timeout = std::chrono::minutes{ 5 }); bool wait_until_user_present(const couchbase::core::cluster& cluster, const std::string& username); @@ -75,6 +75,37 @@ wait_until_cluster_connected(const std::string& username, const std::string& pas bool wait_for_search_pindexes_ready(const couchbase::core::cluster& cluster, const std::string& bucket_name, const std::string& index_name); +bool +wait_for_function_created(const couchbase::core::cluster& cluster, + const std::string& function_name, + const std::optional& bucket_name = {}, + const std::optional& scope_name = {}, + std::size_t successful_rounds = 4, + std::chrono::seconds total_timeout = std::chrono::seconds{ 120 }); + bool wait_until_indexed(const couchbase::core::cluster& cluster, const std::string& index_name, std::uint64_t expected_count); + +bool +create_primary_index(const couchbase::core::cluster& cluster, const std::string& bucket_name); + +class integration_test_guard; +/** + * + * @param integration cluster object + * @param bucket_name name of the bucket + * @param index_name name of the search index + * @param index_params_file_name the filename with index parameters in JSON format + * @param expected_number_of_documents_indexed consider job done when this number of the document has been indexed + * @return pair of boolean value (success if true), and name of the index created (service might rename index) + */ +std::pair +create_search_index(integration_test_guard& integration, + const std::string& bucket_name, + const std::string& index_name, + const std::string& index_params_file_name, + std::size_t expected_number_of_documents_indexed = 800); + +bool +drop_search_index(integration_test_guard& integration, const std::string& index_name); } // namespace test::utils From 153f76bbfcf3f8b20763ec3dd1e79b52f3bb921d Mon Sep 17 00:00:00 2001 From: Jared Casey Date: Tue, 14 May 2024 13:13:41 -0500 Subject: [PATCH 08/11] CXXCBC-511: Prevent use of HTTP session if idle timer has expired (#565) --- core/io/http_session.hxx | 7 +++-- core/io/http_session_manager.hxx | 54 +++++++++++++++++++------------- 2 files changed, 37 insertions(+), 24 deletions(-) diff --git a/core/io/http_session.hxx b/core/io/http_session.hxx index b746c15f6..74fd42616 100644 --- a/core/io/http_session.hxx +++ b/core/io/http_session.hxx @@ -349,13 +349,16 @@ class http_session : public std::enable_shared_from_this if (ec == asio::error::operation_aborted) { return; } + CB_LOG_DEBUG("{} idle timeout expired, stopping session: \"{}:{}\"", self->info_.log_prefix(), self->hostname_, self->service_); self->stop(); }); } - void reset_idle() + bool reset_idle() { - idle_timer_.cancel(); + // Return true if cancel() is successful. Since the idle_timer_ has a single pending + // wait per session, we know the timer has already expired if cancel() returns 0. + return idle_timer_.cancel() != 0; } private: diff --git a/core/io/http_session_manager.hxx b/core/io/http_session_manager.hxx index 36296284f..289689c23 100644 --- a/core/io/http_session_manager.hxx +++ b/core/io/http_session_manager.hxx @@ -204,32 +204,42 @@ class http_session_manager std::scoped_lock lock(sessions_mutex_); idle_sessions_[type].remove_if([](const auto& s) { return !s; }); busy_sessions_[type].remove_if([](const auto& s) { return !s; }); - if (idle_sessions_[type].empty()) { + std::shared_ptr session{}; + while (!idle_sessions_[type].empty()) { + if (preferred_node.empty()) { + session = idle_sessions_[type].front(); + idle_sessions_[type].pop_front(); + if (session->reset_idle()) { + break; + } + } else { + auto ptr = std::find_if(idle_sessions_[type].begin(), idle_sessions_[type].end(), [preferred_node](const auto& s) { + return s->remote_address() == preferred_node; + }); + if (ptr != idle_sessions_[type].end()) { + session = *ptr; + idle_sessions_[type].erase(ptr); + if (session->reset_idle()) { + break; + } + } else { + auto [hostname, port] = split_host_port(preferred_node); + session = bootstrap_session(type, credentials, hostname, port); + break; + } + } + CB_LOG_TRACE("{} Idle timer has expired for \"{}:{}\". Attempting to select another session.", + session->log_prefix(), + session->hostname(), + session->port()); + session.reset(); + } + if (!session) { auto [hostname, port] = preferred_node.empty() ? next_node(type) : lookup_node(type, preferred_node); if (port == 0) { return { errc::common::service_not_available, nullptr }; } - auto session = bootstrap_session(type, credentials, hostname, port); - busy_sessions_[type].push_back(session); - return { {}, session }; - } - std::shared_ptr session{}; - if (preferred_node.empty()) { - session = idle_sessions_[type].front(); - idle_sessions_[type].pop_front(); - session->reset_idle(); - } else { - auto ptr = std::find_if(idle_sessions_[type].begin(), idle_sessions_[type].end(), [preferred_node](const auto& s) { - return s->remote_address() == preferred_node; - }); - if (ptr != idle_sessions_[type].end()) { - session = *ptr; - idle_sessions_[type].erase(ptr); - session->reset_idle(); - } else { - auto [hostname, port] = split_host_port(preferred_node); - session = bootstrap_session(type, credentials, hostname, port); - } + session = bootstrap_session(type, credentials, hostname, port); } busy_sessions_[type].push_back(session); return { {}, session }; From 0e80cacab6355bf70497874c613cfdf5e62489ee Mon Sep 17 00:00:00 2001 From: Sergey Avseyev Date: Mon, 20 May 2024 14:27:34 -0700 Subject: [PATCH 09/11] CXXCBC-509: Zone-Aware replica reads (#566) --- CMakeLists.txt | 1 + core/cluster.cxx | 48 +- core/cluster_options.hxx | 1 + core/impl/cluster.cxx | 2 + core/impl/collection.cxx | 567 ++++-------------- core/impl/get_any_replica.hxx | 1 - core/impl/replica_utils.cxx | 66 ++ core/impl/replica_utils.hxx | 48 ++ core/operations/document_get_all_replicas.hxx | 131 ++-- core/operations/document_get_any_replica.hxx | 119 ++-- .../document_lookup_in_all_replicas.hxx | 201 ++++--- .../document_lookup_in_any_replica.hxx | 268 +++++---- .../operations/management/bucket_describe.cxx | 60 +- .../operations/management/bucket_describe.hxx | 23 + core/topology/configuration.cxx | 6 +- core/topology/configuration.hxx | 10 +- core/topology/configuration_json.hxx | 3 + core/utils/connection_string.cxx | 6 +- couchbase/get_all_replicas_options.hxx | 29 +- couchbase/get_any_replica_options.hxx | 29 +- couchbase/lookup_in_all_replicas_options.hxx | 29 +- couchbase/lookup_in_any_replica_options.hxx | 29 +- couchbase/network_options.hxx | 25 + couchbase/read_preference.hxx | 58 ++ test/test_integration_read_replica.cxx | 565 ++++++++++++++++- test/utils/integration_test_guard.cxx | 52 ++ test/utils/integration_test_guard.hxx | 5 + 27 files changed, 1605 insertions(+), 777 deletions(-) create mode 100644 core/impl/replica_utils.cxx create mode 100644 core/impl/replica_utils.hxx create mode 100644 couchbase/read_preference.hxx diff --git a/CMakeLists.txt b/CMakeLists.txt index cdaf79217..c9ecad9d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -168,6 +168,7 @@ set(couchbase_cxx_client_FILES core/impl/query_index_manager.cxx core/impl/query_string_query.cxx core/impl/regexp_query.cxx + core/impl/replica_utils.cxx core/impl/retry_action.cxx core/impl/retry_reason.cxx core/impl/scope.cxx diff --git a/core/cluster.cxx b/core/cluster.cxx index 9eabe6485..01346d8b3 100644 --- a/core/cluster.cxx +++ b/core/cluster.cxx @@ -232,7 +232,9 @@ class cluster_impl : public std::enable_shared_from_this return self->dns_srv_tracker_->get_srv_nodes([self, hostname = std::move(hostname), handler = std::move(handler)]( origin::node_list nodes, std::error_code ec) mutable { if (ec) { - return self->close([ec, handler = std::move(handler)]() mutable { handler(ec); }); + return self->close([ec, handler = std::move(handler)]() mutable { + handler(ec); + }); } if (!nodes.empty()) { self->origin_.set_nodes(std::move(nodes)); @@ -510,7 +512,9 @@ class cluster_impl : public std::enable_shared_from_this tls_.load_verify_file(origin_.options().trust_certificate, ec); if (ec) { CB_LOG_ERROR("[{}]: unable to load verify file \"{}\": {}", id_, origin_.options().trust_certificate, ec.message()); - return close([ec, handler = std::move(handler)]() mutable { return handler(ec); }); + return close([ec, handler = std::move(handler)]() mutable { + return handler(ec); + }); } } } @@ -520,13 +524,17 @@ class cluster_impl : public std::enable_shared_from_this tls_.use_certificate_chain_file(origin_.certificate_path(), ec); if (ec) { CB_LOG_ERROR("[{}]: unable to load certificate chain \"{}\": {}", id_, origin_.certificate_path(), ec.message()); - return close([ec, handler = std::move(handler)]() mutable { return handler(ec); }); + return close([ec, handler = std::move(handler)]() mutable { + return handler(ec); + }); } CB_LOG_DEBUG(R"([{}]: use TLS private key: "{}")", id_, origin_.key_path()); tls_.use_private_key_file(origin_.key_path(), asio::ssl::context::file_format::pem, ec); if (ec) { CB_LOG_ERROR("[{}]: unable to load private key \"{}\": {}", id_, origin_.key_path(), ec.message()); - return close([ec, handler = std::move(handler)]() mutable { return handler(ec); }); + return close([ec, handler = std::move(handler)]() mutable { + return handler(ec); + }); } } session_ = io::mcbp_session(id_, ctx_, tls_, origin_, dns_srv_tracker_); @@ -559,7 +567,9 @@ class cluster_impl : public std::enable_shared_from_this }); } if (ec) { - return self->close([ec, handler = std::move(handler)]() mutable { handler(ec); }); + return self->close([ec, handler = std::move(handler)]() mutable { + handler(ec); + }); } handler(ec); }); @@ -616,7 +626,9 @@ class cluster_impl : public std::enable_shared_from_this if (cluster->session_) { cluster->session_->ping(collector->build_reporter(), timeout); } - cluster->for_each_bucket([&collector, &timeout](auto bucket) { bucket->ping(collector, timeout); }); + cluster->for_each_bucket([&collector, &timeout](auto bucket) { + bucket->ping(collector, timeout); + }); } cluster->session_manager_->ping(services, timeout, collector, cluster->origin_.credentials()); } @@ -636,7 +648,9 @@ class cluster_impl : public std::enable_shared_from_this if (self->session_) { res.services[service_type::key_value].emplace_back(self->session_->diag_info()); } - self->for_each_bucket([&res](const auto& bucket) { bucket->export_diag_info(res); }); + self->for_each_bucket([&res](const auto& bucket) { + bucket->export_diag_info(res); + }); self->session_manager_->export_diag_info(res); handler(std::move(res)); })); @@ -653,7 +667,9 @@ class cluster_impl : public std::enable_shared_from_this self->session_->stop(retry_reason::do_not_retry); self->session_.reset(); } - self->for_each_bucket([](auto bucket) { bucket->close(); }); + self->for_each_bucket([](auto bucket) { + bucket->close(); + }); self->session_manager_->close(); handler(); self->work_.reset(); @@ -853,7 +869,13 @@ void cluster::execute(operations::get_all_replicas_request request, utils::movable_function&& handler) const { - return request.execute(impl_, std::move(handler)); + auto bucket_name = request.id.bucket(); + return open_bucket(bucket_name, [impl = impl_, request = std::move(request), handler = std::move(handler)](auto ec) mutable { + if (ec) { + return handler(operations::get_all_replicas_response{ make_key_value_error_context(ec, request.id) }); + } + return request.execute(impl, std::move(handler)); + }); } void @@ -873,7 +895,13 @@ void cluster::execute(operations::get_any_replica_request request, utils::movable_function&& handler) const { - return request.execute(impl_, std::move(handler)); + auto bucket_name = request.id.bucket(); + return open_bucket(bucket_name, [impl = impl_, request = std::move(request), handler = std::move(handler)](auto ec) mutable { + if (ec) { + return handler(operations::get_any_replica_response{ make_key_value_error_context(ec, request.id) }); + } + return request.execute(impl, std::move(handler)); + }); } void diff --git a/core/cluster_options.hxx b/core/cluster_options.hxx index 9dc29986a..814cdd641 100644 --- a/core/cluster_options.hxx +++ b/core/cluster_options.hxx @@ -85,6 +85,7 @@ struct cluster_options { std::size_t max_http_connections{ 0 }; std::chrono::milliseconds idle_http_connection_timeout = timeout_defaults::idle_http_connection_timeout; std::string user_agent_extra{}; + std::string server_group{}; couchbase::transactions::transactions_config::built transactions{}; bool dump_configuration{ false }; diff --git a/core/impl/cluster.cxx b/core/impl/cluster.cxx index 962478b4c..a77e44397 100644 --- a/core/impl/cluster.cxx +++ b/core/impl/cluster.cxx @@ -99,6 +99,7 @@ options_to_origin(const std::string& connection_string, const couchbase::cluster user_options.user_agent_extra = opts.behavior.user_agent_extra; user_options.network = opts.behavior.network; + user_options.server_group = opts.network.server_group; user_options.enable_tcp_keep_alive = opts.network.enable_tcp_keep_alive; user_options.tcp_keep_alive_interval = opts.network.tcp_keep_alive_interval; user_options.config_poll_interval = opts.network.config_poll_interval; @@ -120,6 +121,7 @@ options_to_origin(const std::string& connection_string, const couchbase::cluster user_options.use_ip_protocol = core::io::ip_protocol::force_ipv6; break; } + user_options.server_group = opts.network.server_group; user_options.enable_compression = opts.compression.enabled; diff --git a/core/impl/collection.cxx b/core/impl/collection.cxx index 1d694c88f..20aa254ef 100644 --- a/core/impl/collection.cxx +++ b/core/impl/collection.cxx @@ -22,12 +22,16 @@ #include "core/operations/document_decrement.hxx" #include "core/operations/document_exists.hxx" #include "core/operations/document_get.hxx" +#include "core/operations/document_get_all_replicas.hxx" #include "core/operations/document_get_and_lock.hxx" #include "core/operations/document_get_and_touch.hxx" +#include "core/operations/document_get_any_replica.hxx" #include "core/operations/document_get_projected.hxx" #include "core/operations/document_increment.hxx" #include "core/operations/document_insert.hxx" #include "core/operations/document_lookup_in.hxx" +#include "core/operations/document_lookup_in_all_replicas.hxx" +#include "core/operations/document_lookup_in_any_replica.hxx" #include "core/operations/document_mutate_in.hxx" #include "core/operations/document_prepend.hxx" #include "core/operations/document_remove.hxx" @@ -149,91 +153,28 @@ class collection_impl : public std::enable_shared_from_this options.timeout, { options.retry_strategy }, }, - [handler = std::move(handler)](auto resp) mutable { return handler(std::move(resp.ctx), result{ resp.cas }); }); + [handler = std::move(handler)](auto resp) mutable { + return handler(std::move(resp.ctx), result{ resp.cas }); + }); } void get_any_replica(std::string document_key, const get_any_replica_options::built& options, core::impl::movable_get_any_replica_handler&& handler) const { - auto request = - std::make_shared(bucket_name_, scope_name_, name_, std::move(document_key), options.timeout); - core_.with_bucket_configuration( - bucket_name_, - [core = core_, r = std::move(request), h = std::move(handler)](std::error_code ec, - const core::topology::configuration& config) mutable { - if (ec) { - return h(make_key_value_error_context(ec, r->id()), get_replica_result{}); - } - struct replica_context { - replica_context(core::impl::movable_get_any_replica_handler&& handler, std::uint32_t expected_responses) - : handler_(std::move(handler)) - , expected_responses_(expected_responses) - { - } - - core::impl::movable_get_any_replica_handler handler_; - std::uint32_t expected_responses_; - bool done_{ false }; - std::mutex mutex_{}; - }; - auto ctx = std::make_shared(std::move(h), config.num_replicas.value_or(0U) + 1U); - - for (std::size_t idx = 1U; idx <= config.num_replicas.value_or(0U); ++idx) { - core::document_id replica_id{ r->id() }; - replica_id.node_index(idx); - core.execute(core::impl::get_replica_request{ std::move(replica_id), r->timeout() }, [ctx](auto&& resp) { - core::impl::movable_get_any_replica_handler local_handler; - { - const std::scoped_lock lock(ctx->mutex_); - if (ctx->done_) { - return; - } - --ctx->expected_responses_; - if (resp.ctx.ec()) { - if (ctx->expected_responses_ > 0) { - // just ignore the response - return; - } - // consider document irretrievable and give up - resp.ctx.override_ec(errc::key_value::document_irretrievable); - } - ctx->done_ = true; - std::swap(local_handler, ctx->handler_); - } - if (local_handler) { - return local_handler(std::move(resp.ctx), - get_replica_result{ resp.cas, true /* replica */, { std::move(resp.value), resp.flags } }); - } - }); - } - - core::operations::get_request active{ core::document_id{ r->id() } }; - active.timeout = r->timeout(); - core.execute(active, [ctx](auto resp) { - core::impl::movable_get_any_replica_handler local_handler{}; - { - const std::scoped_lock lock(ctx->mutex_); - if (ctx->done_) { - return; - } - --ctx->expected_responses_; - if (resp.ctx.ec()) { - if (ctx->expected_responses_ > 0) { - // just ignore the response - return; - } - // consider document irretrievable and give up - resp.ctx.override_ec(errc::key_value::document_irretrievable); - } - ctx->done_ = true; - std::swap(local_handler, ctx->handler_); - } - if (local_handler) { - return local_handler(std::move(resp.ctx), - get_replica_result{ resp.cas, false /* active */, { std::move(resp.value), resp.flags } }); - } - }); + return core_.execute( + core::operations::get_any_replica_request{ + core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) }, + options.timeout, + options.read_preference, + }, + [handler = std::move(handler)](auto resp) mutable { + return handler(std::move(resp.ctx), + get_replica_result{ + resp.cas, + resp.replica, + { std::move(resp.value), resp.flags }, + }); }); } @@ -241,95 +182,22 @@ class collection_impl : public std::enable_shared_from_this const get_all_replicas_options::built& options, core::impl::movable_get_all_replicas_handler&& handler) const { - auto request = std::make_shared( - bucket_name_, scope_name_, name_, std::move(document_key), options.timeout); - core_.with_bucket_configuration( - bucket_name_, - [core = core_, r = std::move(request), h = std::move(handler)](std::error_code ec, - const core::topology::configuration& config) mutable { - if (ec) { - return h(make_key_value_error_context(ec, r->id()), get_all_replicas_result{}); - } - struct replica_context { - replica_context(core::impl::movable_get_all_replicas_handler handler, std::uint32_t expected_responses) - : handler_(std::move(handler)) - , expected_responses_(expected_responses) - { - } - - core::impl::movable_get_all_replicas_handler handler_; - std::uint32_t expected_responses_; - bool done_{ false }; - std::mutex mutex_{}; - get_all_replicas_result result_{}; - }; - auto ctx = std::make_shared(std::move(h), config.num_replicas.value_or(0U) + 1U); - - for (std::size_t idx = 1U; idx <= config.num_replicas.value_or(0U); ++idx) { - core::document_id replica_id{ r->id() }; - replica_id.node_index(idx); - core.execute(core::impl::get_replica_request{ std::move(replica_id), r->timeout() }, [ctx](auto resp) { - core::impl::movable_get_all_replicas_handler local_handler{}; - { - const std::scoped_lock lock(ctx->mutex_); - if (ctx->done_) { - return; - } - --ctx->expected_responses_; - if (resp.ctx.ec()) { - if (ctx->expected_responses_ > 0) { - // just ignore the response - return; - } - } else { - ctx->result_.emplace_back( - get_replica_result{ resp.cas, true /* replica */, { std::move(resp.value), resp.flags } }); - } - if (ctx->expected_responses_ == 0) { - ctx->done_ = true; - std::swap(local_handler, ctx->handler_); - } - } - if (local_handler) { - if (!ctx->result_.empty()) { - resp.ctx.override_ec({}); - } - return local_handler(std::move(resp.ctx), std::move(ctx->result_)); - } + return core_.execute( + core::operations::get_all_replicas_request{ + core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) }, + options.timeout, + options.read_preference, + }, + [handler = std::move(handler)](auto resp) mutable { + get_all_replicas_result result{}; + for (auto& entry : resp.entries) { + result.emplace_back(get_replica_result{ + entry.cas, + entry.replica, + { std::move(entry.value), entry.flags }, }); } - - core::operations::get_request active{ core::document_id{ r->id() } }; - active.timeout = r->timeout(); - core.execute(active, [ctx](auto resp) { - core::impl::movable_get_all_replicas_handler local_handler{}; - { - const std::scoped_lock lock(ctx->mutex_); - if (ctx->done_) { - return; - } - --ctx->expected_responses_; - if (resp.ctx.ec()) { - if (ctx->expected_responses_ > 0) { - // just ignore the response - return; - } - } else { - ctx->result_.emplace_back( - get_replica_result{ resp.cas, false /* active */, { std::move(resp.value), resp.flags } }); - } - if (ctx->expected_responses_ == 0) { - ctx->done_ = true; - std::swap(local_handler, ctx->handler_); - } - } - if (local_handler) { - if (!ctx->result_.empty()) { - resp.ctx.override_ec({}); - } - return local_handler(std::move(resp.ctx), std::move(ctx->result_)); - } - }); + return handler(std::move(resp.ctx), std::move(result)); }); } @@ -415,7 +283,9 @@ class collection_impl : public std::enable_shared_from_this options.timeout, { options.retry_strategy }, }, - [handler = std::move(handler)](auto&& resp) mutable { return handler(std::move(resp.ctx)); }); + [handler = std::move(handler)](auto&& resp) mutable { + return handler(std::move(resp.ctx)); + }); } void exists(std::string document_key, exists_options::built options, exists_handler&& handler) const @@ -478,253 +348,67 @@ class collection_impl : public std::enable_shared_from_this const lookup_in_all_replicas_options::built& options, lookup_in_all_replicas_handler&& handler) const { - auto request = std::make_shared( - bucket_name_, scope_name_, name_, std::move(document_key), specs, options.timeout); - core_.open_bucket( - bucket_name_, - [core = core_, bucket_name = bucket_name_, r = std::move(request), h = std::move(handler)](std::error_code ec) mutable { - if (ec) { - h(core::make_subdocument_error_context(make_key_value_error_context(ec, r->id()), ec, {}, {}, false), - lookup_in_all_replicas_result{}); - return; + return core_.execute( + core::operations::lookup_in_all_replicas_request{ + core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) }, + specs, + options.timeout, + {}, + options.read_preference, + }, + [handler = std::move(handler)](auto resp) mutable { + lookup_in_all_replicas_result result{}; + for (auto& res : resp.entries) { + std::vector entries; + entries.reserve(res.fields.size()); + for (auto& field : res.fields) { + entries.emplace_back(lookup_in_result::entry{ + std::move(field.path), + std::move(field.value), + field.original_index, + field.exists, + field.ec, + }); + } + result.emplace_back(lookup_in_replica_result{ + res.cas, + std::move(entries), + res.deleted, + res.is_replica, + }); } - - return core.with_bucket_configuration( - bucket_name, - [core = core, r = std::move(r), h = std::move(h)](std::error_code ec, const core::topology::configuration& config) mutable { - if (!config.capabilities.supports_subdoc_read_replica()) { - ec = errc::common::feature_not_available; - } - - if (ec) { - return h(core::make_subdocument_error_context(make_key_value_error_context(ec, r->id()), ec, {}, {}, false), - lookup_in_all_replicas_result{}); - } - struct replica_context { - replica_context(core::impl::movable_lookup_in_all_replicas_handler handler, std::uint32_t expected_responses) - : handler_(std::move(handler)) - , expected_responses_(expected_responses) - { - } - - core::impl::movable_lookup_in_all_replicas_handler handler_; - std::uint32_t expected_responses_; - bool done_{ false }; - std::mutex mutex_{}; - lookup_in_all_replicas_result result_{}; - }; - auto ctx = std::make_shared(std::move(h), config.num_replicas.value_or(0U) + 1U); - - for (std::size_t idx = 1U; idx <= config.num_replicas.value_or(0U); ++idx) { - core::document_id replica_id{ r->id() }; - replica_id.node_index(idx); - core.execute(core::impl::lookup_in_replica_request{ std::move(replica_id), r->specs(), r->timeout() }, - [ctx](core::impl::lookup_in_replica_response&& resp) { - core::impl::movable_lookup_in_all_replicas_handler local_handler{}; - { - const std::scoped_lock lock(ctx->mutex_); - if (ctx->done_) { - return; - } - --ctx->expected_responses_; - if (resp.ctx.ec()) { - if (ctx->expected_responses_ > 0) { - // just ignore the response - return; - } - } else { - std::vector entries{}; - for (const auto& field : resp.fields) { - lookup_in_replica_result::entry lookup_in_entry{}; - lookup_in_entry.path = field.path; - lookup_in_entry.value = field.value; - lookup_in_entry.exists = field.exists; - lookup_in_entry.original_index = field.original_index; - lookup_in_entry.ec = field.ec; - entries.emplace_back(lookup_in_entry); - } - ctx->result_.emplace_back(resp.cas, entries, resp.deleted, true /* replica */); - } - if (ctx->expected_responses_ == 0) { - ctx->done_ = true; - std::swap(local_handler, ctx->handler_); - } - } - if (local_handler) { - if (!ctx->result_.empty()) { - resp.ctx.override_ec({}); - } - return local_handler(std::move(resp.ctx), std::move(ctx->result_)); - } - }); - } - - core::operations::lookup_in_request active{ core::document_id{ r->id() } }; - active.specs = r->specs(); - active.timeout = r->timeout(); - core.execute(active, [ctx](core::operations::lookup_in_response&& resp) { - core::impl::movable_lookup_in_all_replicas_handler local_handler{}; - { - const std::scoped_lock lock(ctx->mutex_); - if (ctx->done_) { - return; - } - --ctx->expected_responses_; - if (resp.ctx.ec()) { - if (ctx->expected_responses_ > 0) { - // just ignore the response - return; - } - } else { - std::vector entries{}; - for (const auto& field : resp.fields) { - lookup_in_replica_result::entry lookup_in_entry{}; - lookup_in_entry.path = field.path; - lookup_in_entry.value = field.value; - lookup_in_entry.exists = field.exists; - lookup_in_entry.original_index = field.original_index; - lookup_in_entry.ec = field.ec; - entries.emplace_back(lookup_in_entry); - } - ctx->result_.emplace_back(resp.cas, entries, resp.deleted, false /* active */); - } - if (ctx->expected_responses_ == 0) { - ctx->done_ = true; - std::swap(local_handler, ctx->handler_); - } - } - if (local_handler) { - if (!ctx->result_.empty()) { - resp.ctx.override_ec({}); - } - return local_handler(std::move(resp.ctx), std::move(ctx->result_)); - } - }); - }); + return handler(std::move(resp.ctx), result); }); - }; + } void lookup_in_any_replica(std::string document_key, const std::vector& specs, const lookup_in_any_replica_options::built& options, lookup_in_any_replica_handler&& handler) const { - auto request = std::make_shared( - bucket_name_, scope_name_, name_, std::move(document_key), specs, options.timeout); - core_.open_bucket( - bucket_name_, - [core = core_, bucket_name = bucket_name_, r = std::move(request), h = std::move(handler)](std::error_code ec) mutable { - if (ec) { - h(core::make_subdocument_error_context(make_key_value_error_context(ec, r->id()), ec, {}, {}, false), - lookup_in_replica_result{}); - return; + return core_.execute( + core::operations::lookup_in_any_replica_request{ + core::document_id{ bucket_name_, scope_name_, name_, std::move(document_key) }, + specs, + options.timeout, + {}, + options.read_preference, + }, + [handler = std::move(handler)](auto resp) mutable { + std::vector entries; + for (auto& field : resp.fields) { + entries.emplace_back(lookup_in_result::entry{ + std::move(field.path), + std::move(field.value), + field.original_index, + field.exists, + field.ec, + }); } - - return core.with_bucket_configuration( - bucket_name, - [core = core, r = std::move(r), h = std::move(h)](std::error_code ec, const core::topology::configuration& config) mutable { - if (!config.capabilities.supports_subdoc_read_replica()) { - ec = errc::common::feature_not_available; - } - if (ec) { - return h(core::make_subdocument_error_context(make_key_value_error_context(ec, r->id()), ec, {}, {}, false), - lookup_in_replica_result{}); - } - struct replica_context { - replica_context(core::impl::movable_lookup_in_any_replica_handler handler, std::uint32_t expected_responses) - : handler_(std::move(handler)) - , expected_responses_(expected_responses) - { - } - - core::impl::movable_lookup_in_any_replica_handler handler_; - std::uint32_t expected_responses_; - bool done_{ false }; - std::mutex mutex_{}; - }; - auto ctx = std::make_shared(std::move(h), config.num_replicas.value_or(0U) + 1U); - - for (std::size_t idx = 1U; idx <= config.num_replicas.value_or(0U); ++idx) { - core::document_id replica_id{ r->id() }; - replica_id.node_index(idx); - core.execute(core::impl::lookup_in_replica_request{ std::move(replica_id), r->specs(), r->timeout() }, - [ctx](core::impl::lookup_in_replica_response&& resp) { - core::impl::movable_lookup_in_any_replica_handler local_handler; - { - const std::scoped_lock lock(ctx->mutex_); - if (ctx->done_) { - return; - } - --ctx->expected_responses_; - if (resp.ctx.ec()) { - if (ctx->expected_responses_ > 0) { - // just ignore the response - return; - } - // consider document irretrievable and give up - resp.ctx.override_ec(errc::key_value::document_irretrievable); - } - ctx->done_ = true; - std::swap(local_handler, ctx->handler_); - } - if (local_handler) { - std::vector entries; - for (const auto& field : resp.fields) { - lookup_in_replica_result::entry entry{}; - entry.path = field.path; - entry.original_index = field.original_index; - entry.exists = field.exists; - entry.value = field.value; - entry.ec = field.ec; - entries.emplace_back(entry); - } - return local_handler( - std::move(resp.ctx), - lookup_in_replica_result{ resp.cas, entries, resp.deleted, true /* replica */ }); - } - }); - } - - core::operations::lookup_in_request active{ core::document_id{ r->id() } }; - active.specs = r->specs(); - active.timeout = r->timeout(); - core.execute(active, [ctx](core::operations::lookup_in_response&& resp) { - core::impl::movable_lookup_in_any_replica_handler local_handler{}; - { - const std::scoped_lock lock(ctx->mutex_); - if (ctx->done_) { - return; - } - --ctx->expected_responses_; - if (resp.ctx.ec()) { - if (ctx->expected_responses_ > 0) { - // just ignore the response - return; - } - // consider document irretrievable and give up - resp.ctx.override_ec(errc::key_value::document_irretrievable); - } - ctx->done_ = true; - std::swap(local_handler, ctx->handler_); - } - if (local_handler) { - std::vector entries; - for (const auto& field : resp.fields) { - lookup_in_replica_result::entry entry{}; - entry.path = field.path; - entry.original_index = field.original_index; - entry.exists = field.exists; - entry.value = field.value; - entry.ec = field.ec; - entries.emplace_back(entry); - } - return local_handler(std::move(resp.ctx), - lookup_in_replica_result{ resp.cas, entries, resp.deleted, false /* active */ }); - } - }); - }); + entries.reserve(resp.fields.size()); + return handler(std::move(resp.ctx), lookup_in_replica_result{ resp.cas, std::move(entries), resp.deleted, resp.is_replica }); }); - }; + } void mutate_in(std::string document_key, const std::vector& specs, @@ -1152,7 +836,9 @@ collection::get(std::string document_id, const get_options& options) const -> st { auto barrier = std::make_shared>>(); auto future = barrier->get_future(); - get(std::move(document_id), options, [barrier](auto ctx, auto result) { barrier->set_value({ std::move(ctx), std::move(result) }); }); + get(std::move(document_id), options, [barrier](auto ctx, auto result) { + barrier->set_value({ std::move(ctx), std::move(result) }); + }); return future; } @@ -1166,8 +852,9 @@ collection::get_and_touch(std::string document_id, } auto -collection::get_and_touch(std::string document_id, std::chrono::seconds duration, const get_and_touch_options& options) const - -> std::future> +collection::get_and_touch(std::string document_id, + std::chrono::seconds duration, + const get_and_touch_options& options) const -> std::future> { auto barrier = std::make_shared>>(); auto future = barrier->get_future(); @@ -1206,8 +893,9 @@ collection::touch(std::string document_id, std::chrono::seconds duration, const } auto -collection::touch(std::string document_id, std::chrono::seconds duration, const touch_options& options) const - -> std::future> +collection::touch(std::string document_id, + std::chrono::seconds duration, + const touch_options& options) const -> std::future> { auto barrier = std::make_shared>>(); auto future = barrier->get_future(); @@ -1227,8 +915,9 @@ collection::touch(std::string document_id, } auto -collection::touch(std::string document_id, std::chrono::system_clock::time_point time_point, const touch_options& options) const - -> std::future> +collection::touch(std::string document_id, + std::chrono::system_clock::time_point time_point, + const touch_options& options) const -> std::future> { auto barrier = std::make_shared>>(); auto future = barrier->get_future(); @@ -1281,8 +970,8 @@ collection::remove(std::string document_id, const remove_options& options, remov } auto -collection::remove(std::string document_id, const remove_options& options) const - -> std::future> +collection::remove(std::string document_id, + const remove_options& options) const -> std::future> { auto barrier = std::make_shared>>(); auto future = barrier->get_future(); @@ -1302,8 +991,9 @@ collection::mutate_in(std::string document_id, } auto -collection::mutate_in(std::string document_id, const mutate_in_specs& specs, const mutate_in_options& options) const - -> std::future> +collection::mutate_in(std::string document_id, + const mutate_in_specs& specs, + const mutate_in_options& options) const -> std::future> { auto barrier = std::make_shared>>(); auto future = barrier->get_future(); @@ -1323,8 +1013,9 @@ collection::lookup_in(std::string document_id, } auto -collection::lookup_in(std::string document_id, const lookup_in_specs& specs, const lookup_in_options& options) const - -> std::future> +collection::lookup_in(std::string document_id, + const lookup_in_specs& specs, + const lookup_in_options& options) const -> std::future> { auto barrier = std::make_shared>>(); auto future = barrier->get_future(); @@ -1344,10 +1035,8 @@ collection::lookup_in_all_replicas(std::string document_id, } auto -collection::lookup_in_all_replicas(std::string document_id, - const lookup_in_specs& specs, - const lookup_in_all_replicas_options& options) const - -> std::future> +collection::lookup_in_all_replicas(std::string document_id, const lookup_in_specs& specs, const lookup_in_all_replicas_options& options) + const -> std::future> { auto barrier = std::make_shared>>(); auto future = barrier->get_future(); @@ -1388,8 +1077,9 @@ collection::get_and_lock(std::string document_id, } auto -collection::get_and_lock(std::string document_id, std::chrono::seconds lock_duration, const get_and_lock_options& options) const - -> std::future> +collection::get_and_lock(std::string document_id, + std::chrono::seconds lock_duration, + const get_and_lock_options& options) const -> std::future> { auto barrier = std::make_shared>>(); auto future = barrier->get_future(); @@ -1410,7 +1100,9 @@ collection::unlock(std::string document_id, couchbase::cas cas, const unlock_opt { auto barrier = std::make_shared>(); auto future = barrier->get_future(); - unlock(std::move(document_id), cas, options, [barrier](auto ctx) { barrier->set_value({ std::move(ctx) }); }); + unlock(std::move(document_id), cas, options, [barrier](auto ctx) { + barrier->set_value({ std::move(ctx) }); + }); return future; } @@ -1421,8 +1113,8 @@ collection::exists(std::string document_id, const exists_options& options, exist } auto -collection::exists(std::string document_id, const exists_options& options) const - -> std::future> +collection::exists(std::string document_id, + const exists_options& options) const -> std::future> { auto barrier = std::make_shared>>(); auto future = barrier->get_future(); @@ -1439,8 +1131,9 @@ collection::upsert(std::string document_id, codec::encoded_value document, const } auto -collection::upsert(std::string document_id, codec::encoded_value document, const upsert_options& options) const - -> std::future> +collection::upsert(std::string document_id, + codec::encoded_value document, + const upsert_options& options) const -> std::future> { auto barrier = std::make_shared>>(); auto future = barrier->get_future(); @@ -1457,8 +1150,9 @@ collection::insert(std::string document_id, codec::encoded_value document, const } auto -collection::insert(std::string document_id, codec::encoded_value document, const insert_options& options) const - -> std::future> +collection::insert(std::string document_id, + codec::encoded_value document, + const insert_options& options) const -> std::future> { auto barrier = std::make_shared>>(); auto future = barrier->get_future(); @@ -1475,8 +1169,9 @@ collection::replace(std::string document_id, codec::encoded_value document, cons } auto -collection::replace(std::string document_id, codec::encoded_value document, const replace_options& options) const - -> std::future> +collection::replace(std::string document_id, + codec::encoded_value document, + const replace_options& options) const -> std::future> { auto barrier = std::make_shared>>(); auto future = barrier->get_future(); @@ -1493,12 +1188,14 @@ collection::scan(const couchbase::scan_type& scan_type, const couchbase::scan_op } auto -collection::scan(const couchbase::scan_type& scan_type, const couchbase::scan_options& options) const - -> std::future> +collection::scan(const couchbase::scan_type& scan_type, + const couchbase::scan_options& options) const -> std::future> { auto barrier = std::make_shared>>(); auto future = barrier->get_future(); - scan(scan_type, options, [barrier](auto ec, auto result) { barrier->set_value({ ec, std::move(result) }); }); + scan(scan_type, options, [barrier](auto ec, auto result) { + barrier->set_value({ ec, std::move(result) }); + }); return future; } } // namespace couchbase diff --git a/core/impl/get_any_replica.hxx b/core/impl/get_any_replica.hxx index b27430584..dc15cc1f6 100644 --- a/core/impl/get_any_replica.hxx +++ b/core/impl/get_any_replica.hxx @@ -20,7 +20,6 @@ #include #include "core/document_id.hxx" -#include "core/error_context/key_value.hxx" #include "core/utils/movable_function.hxx" namespace couchbase::core::impl diff --git a/core/impl/replica_utils.cxx b/core/impl/replica_utils.cxx new file mode 100644 index 000000000..b12f5a6a1 --- /dev/null +++ b/core/impl/replica_utils.cxx @@ -0,0 +1,66 @@ + +/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* + * Copyright 2020-Present Couchbase, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "replica_utils.hxx" + +#include "core/logger/logger.hxx" + +namespace couchbase::core::impl +{ + +auto +effective_nodes(const document_id& id, + const topology::configuration& config, + const read_preference& preference, + const std::string& preferred_server_group) -> std::vector +{ + if (preference != read_preference::no_preference && preferred_server_group.empty()) { + CB_LOG_WARNING("Preferred server group is required for zone-aware replica reads"); + return {}; + } + + std::vector available_nodes{}; + std::vector local_nodes{}; + + for (std::size_t idx = 0U; idx <= config.num_replicas.value_or(0U); ++idx) { + bool is_replica = idx != 0; + auto [vbid, server] = config.map_key(id.key(), idx); + if (server.has_value() && server.value() < config.nodes.size()) { + available_nodes.emplace_back(readable_node{ is_replica, idx }); + if (preferred_server_group == config.nodes[server.value()].server_group) { + local_nodes.emplace_back(readable_node{ is_replica, idx }); + } + } + } + + switch (preference) { + case read_preference::no_preference: + return available_nodes; + + case read_preference::selected_server_group: + return local_nodes; + + case read_preference::selected_server_group_or_all_available: + if (local_nodes.empty()) { + return available_nodes; + } + return local_nodes; + } + return available_nodes; +} +} // namespace couchbase::core::impl diff --git a/core/impl/replica_utils.hxx b/core/impl/replica_utils.hxx new file mode 100644 index 000000000..e41bf9791 --- /dev/null +++ b/core/impl/replica_utils.hxx @@ -0,0 +1,48 @@ + +/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* + * Copyright 2020-Present Couchbase, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "core/document_id.hxx" +#include "core/topology/configuration.hxx" + +#include "couchbase/read_preference.hxx" + +#include +#include + +namespace couchbase::core::impl +{ +struct readable_node { + bool is_replica; + std::size_t index; +}; + +/** + * Returns list of server indexes to send operations. The index values are + * in range [0, number_of_replicas). + * + * In other words, the result is the subset of the vbucket array, which is + * filtered by optional read affinity and preferred server group. + */ +auto +effective_nodes(const document_id& id, + const topology::configuration& config, + const read_preference& preference, + const std::string& preferred_server_group) -> std::vector; +} // namespace couchbase::core::impl diff --git a/core/operations/document_get_all_replicas.hxx b/core/operations/document_get_all_replicas.hxx index 0cab9291e..10b16382f 100644 --- a/core/operations/document_get_all_replicas.hxx +++ b/core/operations/document_get_all_replicas.hxx @@ -19,6 +19,8 @@ #include "core/error_context/key_value.hxx" #include "core/impl/get_replica.hxx" +#include "core/impl/replica_utils.hxx" +#include "core/logger/logger.hxx" #include "core/operations/document_get.hxx" #include "core/operations/operation_traits.hxx" #include "core/utils/movable_function.hxx" @@ -48,91 +50,108 @@ struct get_all_replicas_request { core::document_id id; std::optional timeout{}; + couchbase::read_preference read_preference{ couchbase::read_preference::no_preference }; template void execute(Core core, Handler handler) { core->with_bucket_configuration( id.bucket(), - [core, id = id, timeout = timeout, h = std::forward(handler)](std::error_code ec, - const topology::configuration& config) mutable { + [core, id = id, timeout = timeout, read_preference = read_preference, h = std::forward(handler)]( + std::error_code ec, const topology::configuration& config) mutable { if (ec) { return h(response_type{ make_key_value_error_context(ec, id) }); } + const auto [e, origin] = core->origin(); + if (e) { + return h(response_type{ make_key_value_error_context(e, id) }); + } + + auto nodes = impl::effective_nodes(id, config, read_preference, origin.options().server_group); + if (nodes.empty()) { + CB_LOG_DEBUG("Unable to retrieve replicas for \"{}\", server_group={}, number_of_replicas={}", + id, + origin.options().server_group, + config.num_replicas.value_or(0)); + return h(response_type{ make_key_value_error_context(errc::key_value::document_irretrievable, id) }); + } + using handler_type = utils::movable_function; struct replica_context { - replica_context(handler_type handler, std::uint32_t expected_responses) + replica_context(handler_type handler, std::size_t expected_responses) : handler_(std::move(handler)) , expected_responses_(expected_responses) { } handler_type handler_; - std::uint32_t expected_responses_; + std::size_t expected_responses_; bool done_{ false }; std::mutex mutex_{}; std::vector result_{}; }; - auto ctx = std::make_shared(std::move(h), config.num_replicas.value_or(0U) + 1U); + auto ctx = std::make_shared(std::move(h), nodes.size()); - for (std::size_t idx = 1U; idx <= config.num_replicas.value_or(0U); ++idx) { - document_id replica_id{ id }; - replica_id.node_index(idx); - core->execute(impl::get_replica_request{ std::move(replica_id), timeout }, [ctx](impl::get_replica_response&& resp) { - handler_type local_handler{}; - { - std::scoped_lock lock(ctx->mutex_); - if (ctx->done_) { - return; - } - --ctx->expected_responses_; - if (resp.ctx.ec()) { - if (ctx->expected_responses_ > 0) { - // just ignore the response + for (const auto& node : nodes) { + if (node.is_replica) { + document_id replica_id{ id }; + replica_id.node_index(node.index); + core->execute(impl::get_replica_request{ std::move(replica_id), timeout }, [ctx](auto&& resp) { + handler_type local_handler{}; + { + std::scoped_lock lock(ctx->mutex_); + if (ctx->done_) { return; } - } else { - ctx->result_.emplace_back( - get_all_replicas_response::entry{ std::move(resp.value), resp.cas, resp.flags, true /* replica */ }); + --ctx->expected_responses_; + if (resp.ctx.ec()) { + if (ctx->expected_responses_ > 0) { + // just ignore the response + return; + } + } else { + ctx->result_.emplace_back( + get_all_replicas_response::entry{ std::move(resp.value), resp.cas, resp.flags, true /* replica */ }); + } + if (ctx->expected_responses_ == 0) { + ctx->done_ = true; + std::swap(local_handler, ctx->handler_); + } } - if (ctx->expected_responses_ == 0) { - ctx->done_ = true; - std::swap(local_handler, ctx->handler_); + if (local_handler) { + return local_handler({ std::move(resp.ctx), std::move(ctx->result_) }); } - } - if (local_handler) { - return local_handler({ std::move(resp.ctx), std::move(ctx->result_) }); - } - }); - } - - core->execute(get_request{ document_id{ id }, {}, {}, timeout }, [ctx](get_response&& resp) { - handler_type local_handler{}; - { - std::scoped_lock lock(ctx->mutex_); - if (ctx->done_) { - return; - } - --ctx->expected_responses_; - if (resp.ctx.ec()) { - if (ctx->expected_responses_ > 0) { - // just ignore the response - return; + }); + } else { + core->execute(get_request{ document_id{ id }, {}, {}, timeout }, [ctx](auto&& resp) { + handler_type local_handler{}; + { + std::scoped_lock lock(ctx->mutex_); + if (ctx->done_) { + return; + } + --ctx->expected_responses_; + if (resp.ctx.ec()) { + if (ctx->expected_responses_ > 0) { + // just ignore the response + return; + } + } else { + ctx->result_.emplace_back( + get_all_replicas_response::entry{ std::move(resp.value), resp.cas, resp.flags, false /* active */ }); + } + if (ctx->expected_responses_ == 0) { + ctx->done_ = true; + std::swap(local_handler, ctx->handler_); + } } - } else { - ctx->result_.emplace_back( - get_all_replicas_response::entry{ std::move(resp.value), resp.cas, resp.flags, false /* replica */ }); - } - if (ctx->expected_responses_ == 0) { - ctx->done_ = true; - std::swap(local_handler, ctx->handler_); - } - } - if (local_handler) { - return local_handler({ std::move(resp.ctx), std::move(ctx->result_) }); + if (local_handler) { + return local_handler({ std::move(resp.ctx), std::move(ctx->result_) }); + } + }); } - }); + } }); } }; diff --git a/core/operations/document_get_any_replica.hxx b/core/operations/document_get_any_replica.hxx index 6c9b77d66..e5177d775 100644 --- a/core/operations/document_get_any_replica.hxx +++ b/core/operations/document_get_any_replica.hxx @@ -19,6 +19,7 @@ #include "core/error_context/key_value.hxx" #include "core/impl/get_replica.hxx" +#include "core/impl/replica_utils.hxx" #include "core/operations/document_get.hxx" #include "core/operations/operation_traits.hxx" #include "core/utils/movable_function.hxx" @@ -45,84 +46,102 @@ struct get_any_replica_request { core::document_id id; std::optional timeout{}; + couchbase::read_preference read_preference{ couchbase::read_preference::no_preference }; template void execute(Core core, Handler handler) { core->with_bucket_configuration( id.bucket(), - [core, id = id, timeout = timeout, h = std::forward(handler)](std::error_code ec, - const topology::configuration& config) mutable { + [core, id = id, timeout = timeout, read_preference = read_preference, h = std::forward(handler)]( + std::error_code ec, const topology::configuration& config) mutable { + const auto [e, origin] = core->origin(); + if (e && !ec) { + ec = e; + } + + auto nodes = impl::effective_nodes(id, config, read_preference, origin.options().server_group); + if (nodes.empty()) { + CB_LOG_DEBUG("Unable to retrieve replicas for \"{}\", server_group={}, number_of_replicas={}", + id, + origin.options().server_group, + config.num_replicas.value_or(0)); + ec = errc::key_value::document_irretrievable; + } + if (ec) { return h(response_type{ make_key_value_error_context(ec, id) }); } using handler_type = utils::movable_function; struct replica_context { - replica_context(handler_type&& handler, std::uint32_t expected_responses) + replica_context(handler_type&& handler, std::size_t expected_responses) : handler_(std::move(handler)) , expected_responses_(expected_responses) { } handler_type handler_; - std::uint32_t expected_responses_; + std::size_t expected_responses_; bool done_{ false }; std::mutex mutex_{}; }; - auto ctx = std::make_shared(std::move(h), config.num_replicas.value_or(0U) + 1U); + auto ctx = std::make_shared(std::move(h), nodes.size()); - for (std::size_t idx = 1U; idx <= config.num_replicas.value_or(0U); ++idx) { - document_id replica_id{ id }; - replica_id.node_index(idx); - core->execute(impl::get_replica_request{ std::move(replica_id), timeout }, [ctx](impl::get_replica_response&& resp) { - handler_type local_handler; - { - std::scoped_lock lock(ctx->mutex_); - if (ctx->done_) { - return; + for (const auto& node : nodes) { + if (node.is_replica) { + document_id replica_id{ id }; + replica_id.node_index(node.index); + core->execute(impl::get_replica_request{ std::move(replica_id), timeout }, [ctx](auto&& resp) { + handler_type local_handler; + { + std::scoped_lock lock(ctx->mutex_); + if (ctx->done_) { + return; + } + --ctx->expected_responses_; + if (resp.ctx.ec()) { + if (ctx->expected_responses_ > 0) { + // just ignore the response + return; + } + // consider document irretrievable and give up + resp.ctx.override_ec(errc::key_value::document_irretrievable); + } + ctx->done_ = true; + std::swap(local_handler, ctx->handler_); } - --ctx->expected_responses_; - if (resp.ctx.ec()) { - if (ctx->expected_responses_ > 0) { - // just ignore the response + if (local_handler) { + return local_handler(response_type{ std::move(resp.ctx), std::move(resp.value), resp.cas, resp.flags, true }); + } + }); + } else { + core->execute(get_request{ id, {}, {}, timeout }, [ctx](auto&& resp) { + handler_type local_handler{}; + { + std::scoped_lock lock(ctx->mutex_); + if (ctx->done_) { return; } - // consider document irretrievable and give up - resp.ctx.override_ec(errc::key_value::document_irretrievable); + --ctx->expected_responses_; + if (resp.ctx.ec()) { + if (ctx->expected_responses_ > 0) { + // just ignore the response + return; + } + // consider document irretrievable and give up + resp.ctx.override_ec(errc::key_value::document_irretrievable); + } + ctx->done_ = true; + std::swap(local_handler, ctx->handler_); } - ctx->done_ = true; - std::swap(local_handler, ctx->handler_); - } - if (local_handler) { - return local_handler(response_type{ std::move(resp.ctx), std::move(resp.value), resp.cas, resp.flags, true }); - } - }); - } - - core->execute(get_request{ id, {}, {}, timeout }, [ctx](get_response&& resp) { - handler_type local_handler{}; - { - std::scoped_lock lock(ctx->mutex_); - if (ctx->done_) { - return; - } - --ctx->expected_responses_; - if (resp.ctx.ec()) { - if (ctx->expected_responses_ > 0) { - // just ignore the response - return; + if (local_handler) { + return local_handler( + response_type{ std::move(resp.ctx), std::move(resp.value), resp.cas, resp.flags, false }); } - // consider document irretrievable and give up - resp.ctx.override_ec(errc::key_value::document_irretrievable); - } - ctx->done_ = true; - std::swap(local_handler, ctx->handler_); - } - if (local_handler) { - return local_handler(response_type{ std::move(resp.ctx), std::move(resp.value), resp.cas, resp.flags, false }); + }); } - }); + } }); } }; diff --git a/core/operations/document_lookup_in_all_replicas.hxx b/core/operations/document_lookup_in_all_replicas.hxx index 6bdcb90c9..fb883e785 100644 --- a/core/operations/document_lookup_in_all_replicas.hxx +++ b/core/operations/document_lookup_in_all_replicas.hxx @@ -19,10 +19,12 @@ #include "core/error_context/key_value.hxx" #include "core/impl/lookup_in_replica.hxx" +#include "core/impl/replica_utils.hxx" #include "core/impl/subdoc/command.hxx" #include "core/operations/document_lookup_in.hxx" #include "core/operations/operation_traits.hxx" #include "core/utils/movable_function.hxx" + #include "couchbase/codec/encoded_value.hxx" #include "couchbase/error_codes.hxx" @@ -61,14 +63,20 @@ struct lookup_in_all_replicas_request { std::vector specs{}; std::optional timeout{}; std::shared_ptr parent_span{ nullptr }; + couchbase::read_preference read_preference{ couchbase::read_preference::no_preference }; template void execute(Core core, Handler handler) { core->open_bucket( id.bucket(), - [core, id = id, timeout = timeout, specs = specs, parent_span = parent_span, h = std::forward(handler)]( - std::error_code ec) mutable { + [core, + id = id, + timeout = timeout, + specs = specs, + parent_span = parent_span, + read_preference = read_preference, + h = std::forward(handler)](std::error_code ec) mutable { if (ec) { std::optional first_error_path{}; std::optional first_error_index{}; @@ -77,121 +85,136 @@ struct lookup_in_all_replicas_request { } core->with_bucket_configuration( id.bucket(), - [core, id = id, timeout = timeout, specs = specs, parent_span = parent_span, h = std::forward(h)]( + [core, id, timeout, specs, parent_span, read_preference, h = std::forward(h)]( std::error_code ec, const topology::configuration& config) mutable { if (!config.capabilities.supports_subdoc_read_replica()) { ec = errc::common::feature_not_available; } + const auto [e, origin] = core->origin(); + if (e && !ec) { + ec = e; + } + + auto nodes = impl::effective_nodes(id, config, read_preference, origin.options().server_group); + if (nodes.empty()) { + CB_LOG_DEBUG("Unable to retrieve replicas for \"{}\", server_group={}, number_of_replicas={}", + id, + origin.options().server_group, + config.num_replicas.value_or(0)); + ec = errc::key_value::document_irretrievable; + } + if (ec) { - std::optional first_error_path{}; - std::optional first_error_index{}; - return h(response_type{ make_subdocument_error_context( - make_key_value_error_context(ec, id), ec, first_error_path, first_error_index, false) }); + return h(response_type{ make_subdocument_error_context(make_key_value_error_context(ec, id), ec, {}, {}, false) }); } + using handler_type = utils::movable_function; struct replica_context { - replica_context(handler_type handler, std::uint32_t expected_responses) + replica_context(handler_type handler, std::size_t expected_responses) : handler_(std::move(handler)) , expected_responses_(expected_responses) { } handler_type handler_; - std::uint32_t expected_responses_; + std::size_t expected_responses_; bool done_{ false }; std::mutex mutex_{}; std::vector result_{}; }; - auto ctx = std::make_shared(std::move(h), config.num_replicas.value_or(0U) + 1U); - - for (std::size_t idx = 1U; idx <= config.num_replicas.value_or(0U); ++idx) { - document_id replica_id{ id }; - replica_id.node_index(idx); - core->execute(impl::lookup_in_replica_request{ std::move(replica_id), specs, timeout, parent_span }, - [ctx](impl::lookup_in_replica_response&& resp) { - handler_type local_handler{}; - { - std::scoped_lock lock(ctx->mutex_); - if (ctx->done_) { - return; - } - --ctx->expected_responses_; - if (resp.ctx.ec()) { - if (ctx->expected_responses_ > 0) { - // just ignore the response + + auto ctx = std::make_shared(std::move(h), nodes.size()); + + for (const auto& node : nodes) { + if (node.is_replica) { + document_id replica_id{ id }; + replica_id.node_index(node.index); + core->execute(impl::lookup_in_replica_request{ std::move(replica_id), specs, timeout, parent_span }, + [ctx](auto&& resp) { + handler_type local_handler{}; + { + std::scoped_lock lock(ctx->mutex_); + if (ctx->done_) { return; } - } else { - lookup_in_all_replicas_response::entry top_entry{}; - top_entry.cas = resp.cas; - top_entry.deleted = resp.deleted; - top_entry.is_replica = true; - for (auto& field : resp.fields) { - lookup_in_all_replicas_response::entry::lookup_in_entry lookup_in_entry{}; - lookup_in_entry.path = field.path; - lookup_in_entry.value = field.value; - lookup_in_entry.status = field.status; - lookup_in_entry.ec = field.ec; - lookup_in_entry.exists = field.exists; - lookup_in_entry.original_index = field.original_index; - lookup_in_entry.opcode = field.opcode; - top_entry.fields.emplace_back(lookup_in_entry); + --ctx->expected_responses_; + if (resp.ctx.ec()) { + if (ctx->expected_responses_ > 0) { + // just ignore the response + return; + } + } else { + lookup_in_all_replicas_response::entry top_entry{}; + top_entry.cas = resp.cas; + top_entry.deleted = resp.deleted; + top_entry.is_replica = true; + for (auto& field : resp.fields) { + lookup_in_all_replicas_response::entry::lookup_in_entry lookup_in_entry{}; + lookup_in_entry.path = field.path; + lookup_in_entry.value = field.value; + lookup_in_entry.status = field.status; + lookup_in_entry.ec = field.ec; + lookup_in_entry.exists = field.exists; + lookup_in_entry.original_index = field.original_index; + lookup_in_entry.opcode = field.opcode; + top_entry.fields.emplace_back(lookup_in_entry); + } + ctx->result_.emplace_back(lookup_in_all_replicas_response::entry{ top_entry }); + } + if (ctx->expected_responses_ == 0) { + ctx->done_ = true; + std::swap(local_handler, ctx->handler_); } - ctx->result_.emplace_back(lookup_in_all_replicas_response::entry{ top_entry }); } - if (ctx->expected_responses_ == 0) { - ctx->done_ = true; - std::swap(local_handler, ctx->handler_); + if (local_handler) { + return local_handler({ std::move(resp.ctx), std::move(ctx->result_) }); } - } - if (local_handler) { - return local_handler({ std::move(resp.ctx), std::move(ctx->result_) }); - } - }); - } - - core->execute(lookup_in_request{ document_id{ id }, {}, {}, false, specs, timeout }, [ctx](lookup_in_response&& resp) { - handler_type local_handler{}; - { - std::scoped_lock lock(ctx->mutex_); - if (ctx->done_) { - return; - } - --ctx->expected_responses_; - if (resp.ctx.ec()) { - if (ctx->expected_responses_ > 0) { - // just ignore the response - return; + }); + } else { + core->execute(lookup_in_request{ document_id{ id }, {}, {}, false, specs, timeout }, [ctx](auto&& resp) { + handler_type local_handler{}; + { + std::scoped_lock lock(ctx->mutex_); + if (ctx->done_) { + return; + } + --ctx->expected_responses_; + if (resp.ctx.ec()) { + if (ctx->expected_responses_ > 0) { + // just ignore the response + return; + } + } else { + lookup_in_all_replicas_response::entry top_entry{}; + top_entry.cas = resp.cas; + top_entry.deleted = resp.deleted; + top_entry.is_replica = false; + for (auto& field : resp.fields) { + lookup_in_all_replicas_response::entry::lookup_in_entry lookup_in_entry{}; + lookup_in_entry.path = field.path; + lookup_in_entry.value = field.value; + lookup_in_entry.status = field.status; + lookup_in_entry.ec = field.ec; + lookup_in_entry.exists = field.exists; + lookup_in_entry.original_index = field.original_index; + lookup_in_entry.opcode = field.opcode; + top_entry.fields.emplace_back(lookup_in_entry); + } + ctx->result_.emplace_back(lookup_in_all_replicas_response::entry{ top_entry }); + } + if (ctx->expected_responses_ == 0) { + ctx->done_ = true; + std::swap(local_handler, ctx->handler_); + } } - } else { - lookup_in_all_replicas_response::entry top_entry{}; - top_entry.cas = resp.cas; - top_entry.deleted = resp.deleted; - top_entry.is_replica = false; - for (auto& field : resp.fields) { - lookup_in_all_replicas_response::entry::lookup_in_entry lookup_in_entry{}; - lookup_in_entry.path = field.path; - lookup_in_entry.value = field.value; - lookup_in_entry.status = field.status; - lookup_in_entry.ec = field.ec; - lookup_in_entry.exists = field.exists; - lookup_in_entry.original_index = field.original_index; - lookup_in_entry.opcode = field.opcode; - top_entry.fields.emplace_back(lookup_in_entry); + if (local_handler) { + return local_handler({ std::move(resp.ctx), std::move(ctx->result_) }); } - ctx->result_.emplace_back(lookup_in_all_replicas_response::entry{ top_entry }); - } - if (ctx->expected_responses_ == 0) { - ctx->done_ = true; - std::swap(local_handler, ctx->handler_); - } - } - if (local_handler) { - return local_handler({ std::move(resp.ctx), std::move(ctx->result_) }); + }); } - }); + } }); }); } diff --git a/core/operations/document_lookup_in_any_replica.hxx b/core/operations/document_lookup_in_any_replica.hxx index df8609f5d..5d2ebe9f6 100644 --- a/core/operations/document_lookup_in_any_replica.hxx +++ b/core/operations/document_lookup_in_any_replica.hxx @@ -19,12 +19,14 @@ #include "core/error_context/key_value.hxx" #include "core/impl/lookup_in_replica.hxx" +#include "core/impl/replica_utils.hxx" #include "core/impl/subdoc/command.hxx" #include "core/operations/document_lookup_in.hxx" #include "core/operations/operation_traits.hxx" #include "core/utils/movable_function.hxx" #include "couchbase/codec/encoded_value.hxx" #include "couchbase/error_codes.hxx" +#include "couchbase/read_preference.hxx" #include #include @@ -58,135 +60,163 @@ struct lookup_in_any_replica_request { std::vector specs{}; std::optional timeout{}; std::shared_ptr parent_span{ nullptr }; + couchbase::read_preference read_preference{ couchbase::read_preference::no_preference }; template void execute(Core core, Handler handler) { - core->open_bucket(id.bucket(), - [core, id = id, timeout = timeout, specs = specs, parent_span = parent_span, h = std::forward(handler)]( - std::error_code ec) mutable { - if (ec) { - std::optional first_error_path{}; - std::optional first_error_index{}; - h(response_type{ make_subdocument_error_context( - make_key_value_error_context(ec, id), ec, first_error_path, first_error_index, false) }); - return; - } - return core->with_bucket_configuration( - id.bucket(), - [core, id = id, timeout = timeout, specs = specs, parent_span = parent_span, h = std::forward(h)]( - std::error_code ec, const topology::configuration& config) mutable { - if (!config.capabilities.supports_subdoc_read_replica()) { - ec = errc::common::feature_not_available; - } + core->open_bucket( + id.bucket(), + [core, + id = id, + timeout = timeout, + specs = specs, + parent_span = parent_span, + read_preference = read_preference, + h = std::forward(handler)](std::error_code ec) mutable { + if (ec) { + std::optional first_error_path{}; + std::optional first_error_index{}; + h(response_type{ + make_subdocument_error_context(make_key_value_error_context(ec, id), ec, first_error_path, first_error_index, false) }); + return; + } + return core->with_bucket_configuration( + id.bucket(), + [core, id, timeout, specs, parent_span, read_preference, h = std::forward(h)]( + std::error_code ec, const topology::configuration& config) mutable { + if (!config.capabilities.supports_subdoc_read_replica()) { + ec = errc::common::feature_not_available; + } + const auto [e, origin] = core->origin(); + if (e && !ec) { + ec = e; + } - if (ec) { - std::optional first_error_path{}; - std::optional first_error_index{}; - return h(response_type{ make_subdocument_error_context( - make_key_value_error_context(ec, id), ec, first_error_path, first_error_index, false) }); - } - using handler_type = utils::movable_function; + auto nodes = impl::effective_nodes(id, config, read_preference, origin.options().server_group); + if (nodes.empty()) { + CB_LOG_DEBUG("Unable to retrieve replicas for \"{}\", server_group={}, number_of_replicas={}", + id, + origin.options().server_group, + config.num_replicas.value_or(0)); + ec = errc::key_value::document_irretrievable; + } - struct replica_context { - replica_context(handler_type&& handler, std::uint32_t expected_responses) - : handler_(std::move(handler)) - , expected_responses_(expected_responses) - { - } + if (ec) { + return h(response_type{ make_subdocument_error_context(make_key_value_error_context(ec, id), ec, {}, {}, false) }); + } + + if (ec) { + std::optional first_error_path{}; + std::optional first_error_index{}; + return h(response_type{ make_subdocument_error_context( + make_key_value_error_context(ec, id), ec, first_error_path, first_error_index, false) }); + } + + using handler_type = utils::movable_function; - handler_type handler_; - std::uint32_t expected_responses_; - bool done_{ false }; - std::mutex mutex_{}; - }; - auto ctx = std::make_shared(std::move(h), config.num_replicas.value_or(0U) + 1U); + struct replica_context { + replica_context(handler_type&& handler, std::size_t expected_responses) + : handler_(std::move(handler)) + , expected_responses_(expected_responses) + { + } - for (std::size_t idx = 1U; idx <= config.num_replicas.value_or(0U); ++idx) { - document_id replica_id{ id }; - replica_id.node_index(idx); - core->execute(impl::lookup_in_replica_request{ std::move(replica_id), specs, timeout, parent_span }, - [ctx](impl::lookup_in_replica_response&& resp) { - handler_type local_handler; - { - std::scoped_lock lock(ctx->mutex_); - if (ctx->done_) { - return; - } - --ctx->expected_responses_; - if (resp.ctx.ec()) { - if (ctx->expected_responses_ > 0) { - // just ignore the response - return; - } - // consider document irretrievable and give up - resp.ctx.override_ec(errc::key_value::document_irretrievable); - } - ctx->done_ = true; - std::swap(local_handler, ctx->handler_); - } - if (local_handler) { - response_type res{}; - res.ctx = resp.ctx; - res.cas = resp.cas; - res.deleted = resp.deleted; - res.is_replica = true; - for (auto& field : resp.fields) { - auto lookup_in_entry = lookup_in_any_replica_response::entry{}; - lookup_in_entry.path = field.path; - lookup_in_entry.value = field.value; - lookup_in_entry.status = field.status; - lookup_in_entry.ec = field.ec; - lookup_in_entry.exists = field.exists; - lookup_in_entry.original_index = field.original_index; - lookup_in_entry.opcode = field.opcode; - res.fields.emplace_back(lookup_in_entry); - } - return local_handler(res); - } - }); + handler_type handler_; + std::size_t expected_responses_; + bool done_{ false }; + std::mutex mutex_{}; + }; + auto ctx = std::make_shared(std::move(h), nodes.size()); + + for (const auto& node : nodes) { + if (node.is_replica) { + document_id replica_id{ id }; + replica_id.node_index(node.index); + core->execute(impl::lookup_in_replica_request{ std::move(replica_id), specs, timeout, parent_span }, + [ctx](auto&& resp) { + handler_type local_handler; + { + std::scoped_lock lock(ctx->mutex_); + if (ctx->done_) { + return; + } + --ctx->expected_responses_; + if (resp.ctx.ec()) { + if (ctx->expected_responses_ > 0) { + // just ignore the response + return; + } + // consider document irretrievable and give up + resp.ctx.override_ec(errc::key_value::document_irretrievable); + } + ctx->done_ = true; + std::swap(local_handler, ctx->handler_); + } + if (local_handler) { + response_type res{}; + res.ctx = resp.ctx; + res.cas = resp.cas; + res.deleted = resp.deleted; + res.is_replica = true; + for (auto& field : resp.fields) { + auto lookup_in_entry = lookup_in_any_replica_response::entry{}; + lookup_in_entry.path = field.path; + lookup_in_entry.value = field.value; + lookup_in_entry.status = field.status; + lookup_in_entry.ec = field.ec; + lookup_in_entry.exists = field.exists; + lookup_in_entry.original_index = field.original_index; + lookup_in_entry.opcode = field.opcode; + res.fields.emplace_back(lookup_in_entry); + } + return local_handler(res); + } + }); + } else { + core->execute(lookup_in_request{ id, {}, {}, false, specs, timeout }, [ctx](auto&& resp) { + handler_type local_handler{}; + { + std::scoped_lock lock(ctx->mutex_); + if (ctx->done_) { + return; } - core->execute(lookup_in_request{ id, {}, {}, false, specs, timeout }, [ctx](lookup_in_response&& resp) { - handler_type local_handler{}; - { - std::scoped_lock lock(ctx->mutex_); - if (ctx->done_) { - return; - } - --ctx->expected_responses_; - if (resp.ctx.ec()) { - if (ctx->expected_responses_ > 0) { - // just ignore the response - return; - } - // consider document irretrievable and give up - resp.ctx.override_ec(errc::key_value::document_irretrievable); - } - ctx->done_ = true; - std::swap(local_handler, ctx->handler_); - } - if (local_handler) { - auto res = response_type{}; - res.ctx = resp.ctx; - res.cas = resp.cas; - res.deleted = resp.deleted; - res.is_replica = false; - for (auto& field : resp.fields) { - auto lookup_in_entry = lookup_in_any_replica_response::entry{}; - lookup_in_entry.path = field.path; - lookup_in_entry.value = field.value; - lookup_in_entry.status = field.status; - lookup_in_entry.ec = field.ec; - lookup_in_entry.exists = field.exists; - lookup_in_entry.original_index = field.original_index; - lookup_in_entry.opcode = field.opcode; - res.fields.emplace_back(lookup_in_entry); - } - return local_handler(res); + --ctx->expected_responses_; + if (resp.ctx.ec()) { + if (ctx->expected_responses_ > 0) { + // just ignore the response + return; } - }); - }); - }); + // consider document irretrievable and give up + resp.ctx.override_ec(errc::key_value::document_irretrievable); + } + ctx->done_ = true; + std::swap(local_handler, ctx->handler_); + } + if (local_handler) { + auto res = response_type{}; + res.ctx = resp.ctx; + res.cas = resp.cas; + res.deleted = resp.deleted; + res.is_replica = false; + for (auto& field : resp.fields) { + auto lookup_in_entry = lookup_in_any_replica_response::entry{}; + lookup_in_entry.path = field.path; + lookup_in_entry.value = field.value; + lookup_in_entry.status = field.status; + lookup_in_entry.ec = field.ec; + lookup_in_entry.exists = field.exists; + lookup_in_entry.original_index = field.original_index; + lookup_in_entry.opcode = field.opcode; + res.fields.emplace_back(lookup_in_entry); + } + return local_handler(res); + } + }); + } + } + }); + }); } }; diff --git a/core/operations/management/bucket_describe.cxx b/core/operations/management/bucket_describe.cxx index 60132ecf5..9a088d083 100644 --- a/core/operations/management/bucket_describe.cxx +++ b/core/operations/management/bucket_describe.cxx @@ -20,6 +20,7 @@ #include "core/utils/json.hxx" #include "error_utils.hxx" +#include #include namespace couchbase::core::operations::management @@ -55,19 +56,76 @@ bucket_describe_request::make_response(error_context::http&& ctx, const encoded_ if (response.ctx.ec) { return response; } + response.info.config_json = encoded.body.data(); - auto payload = utils::json::parse(encoded.body.data()); + auto payload = utils::json::parse(response.info.config_json); response.info.name = payload.at("name").get_string(); response.info.uuid = payload.at("uuid").get_string(); if (const auto* nodes_ext = payload.find("nodesExt"); nodes_ext != nullptr && nodes_ext->is_array()) { response.info.number_of_nodes = nodes_ext->get_array().size(); } + std::vector> vbucket_map{}; if (const auto* vbs_map = payload.find("vBucketServerMap"); vbs_map != nullptr && vbs_map->is_object()) { if (const auto* num_replicas = vbs_map->find("numReplicas"); num_replicas != nullptr && num_replicas->is_number()) { response.info.number_of_replicas = num_replicas->get_unsigned(); } + if (const auto* map = vbs_map->find("vBucketMap"); map != nullptr && map->is_array()) { + for (const auto& vb : map->get_array()) { + std::vector vbucket; + for (const auto& v : vb.get_array()) { + vbucket.emplace_back(v.as()); + } + vbucket_map.emplace_back(vbucket); + } + } } + if (const auto* nodes = payload.find("nodesExt"); nodes != nullptr && nodes->is_array()) { + std::size_t server_index = 0; + for (const auto& node : nodes->get_array()) { + if (const auto* server_group_name = node.find("serverGroup"); server_group_name != nullptr && server_group_name->is_string()) { + std::string group_name = server_group_name->get_string(); + auto& group = response.info.server_groups[group_name]; + group.name = group_name; + server_node server; + server.server_index = server_index; + server.server_group_name = group_name; + if (const auto* hostname = node.find("hostname"); hostname != nullptr && hostname->is_string()) { + server.default_network.hostname = hostname->get_string(); + } + if (const auto* services = node.find("services"); services != nullptr && services->is_object()) { + server.default_network.kv_plain = services->template optional("kv").value_or(0); + server.default_network.kv_tls = services->template optional("kvSSL").value_or(0); + } + if (const auto* alt = node.find("alternateAddresses"); alt != nullptr && alt->is_object()) { + if (const auto* external = alt->find("external"); external != nullptr && external->is_object()) { + if (const auto* hostname = external->find("hostname"); hostname != nullptr && hostname->is_string()) { + server.external_network.hostname = hostname->get_string(); + if (const auto* services = external->find("ports"); services != nullptr && services->is_object()) { + server.external_network.kv_plain = services->template optional("kv").value_or(0); + server.external_network.kv_tls = services->template optional("kvSSL").value_or(0); + } + } + } + } + + for (std::size_t vbid = 0; vbid < vbucket_map.size(); ++vbid) { + for (std::size_t idx = 0; idx < vbucket_map[vbid].size(); ++idx) { + if (vbucket_map[vbid][idx] == static_cast(server_index)) { + if (idx == 0) { + server.active_vbuckets.insert(static_cast(vbid)); + } else { + server.replica_vbuckets.insert(static_cast(vbid)); + } + } + } + } + group.nodes.emplace_back(server); + ++server_index; + } + } + } + if (const auto* storage_backend = payload.find("storageBackend"); storage_backend != nullptr && storage_backend->is_string()) { if (const auto& str = storage_backend->get_string(); str == "couchstore") { response.info.storage_backend = couchbase::core::management::cluster::bucket_storage_backend::couchstore; diff --git a/core/operations/management/bucket_describe.hxx b/core/operations/management/bucket_describe.hxx index 9dd06440d..d7455953f 100644 --- a/core/operations/management/bucket_describe.hxx +++ b/core/operations/management/bucket_describe.hxx @@ -26,6 +26,26 @@ namespace couchbase::core::operations::management { +struct server_node_address { + std::string hostname{}; + std::uint16_t kv_plain{}; + std::uint16_t kv_tls{}; +}; + +struct server_node { + std::string server_group_name{}; + std::size_t server_index{}; + server_node_address default_network{}; + server_node_address external_network{}; + std::set active_vbuckets{}; + std::set replica_vbuckets{}; +}; + +struct server_group { + std::string name{}; + std::vector nodes{}; +}; + struct bucket_describe_response { struct bucket_info { std::string name{}; @@ -33,9 +53,12 @@ struct bucket_describe_response { std::size_t number_of_nodes{ 0 }; std::size_t number_of_replicas{ 0 }; std::vector bucket_capabilities{}; + std::map server_groups{}; + couchbase::core::management::cluster::bucket_storage_backend storage_backend{ couchbase::core::management::cluster::bucket_storage_backend::unknown }; + std::string config_json{}; [[nodiscard]] auto has_capability(const std::string& capability) const -> bool; }; diff --git a/core/topology/configuration.cxx b/core/topology/configuration.cxx index de72ccbf0..f49ac0273 100644 --- a/core/topology/configuration.cxx +++ b/core/topology/configuration.cxx @@ -220,7 +220,7 @@ configuration::index_for_this_node() const } std::optional -configuration::server_by_vbucket(std::uint16_t vbucket, std::size_t index) +configuration::server_by_vbucket(std::uint16_t vbucket, std::size_t index) const { if (!vbmap.has_value() || vbucket >= vbmap->size()) { return {}; @@ -232,7 +232,7 @@ configuration::server_by_vbucket(std::uint16_t vbucket, std::size_t index) } std::pair> -configuration::map_key(const std::string& key, std::size_t index) +configuration::map_key(const std::string& key, std::size_t index) const { if (!vbmap.has_value()) { return { 0, {} }; @@ -243,7 +243,7 @@ configuration::map_key(const std::string& key, std::size_t index) } std::pair> -configuration::map_key(const std::vector& key, std::size_t index) +configuration::map_key(const std::vector& key, std::size_t index) const { if (!vbmap.has_value()) { return { 0, {} }; diff --git a/core/topology/configuration.hxx b/core/topology/configuration.hxx index 21fb69e2f..61136e584 100644 --- a/core/topology/configuration.hxx +++ b/core/topology/configuration.hxx @@ -60,6 +60,7 @@ struct configuration { port_map services_plain{}; port_map services_tls{}; std::map alt{}; + std::string server_group{}; bool operator!=(const node& other) const { @@ -117,17 +118,14 @@ struct configuration { const std::string& hostname, const std::string& port) const; - std::pair> map_key(const std::string& key, std::size_t index); - std::pair> map_key(const std::vector& key, std::size_t index); + std::pair> map_key(const std::string& key, std::size_t index) const; + std::pair> map_key(const std::vector& key, std::size_t index) const; - std::optional server_by_vbucket(std::uint16_t vbucket, std::size_t index); + std::optional server_by_vbucket(std::uint16_t vbucket, std::size_t index) const; }; using endpoint = std::pair; -configuration -parse_configuration(std::string_view input, endpoint source); - configuration make_blank_configuration(const std::string& hostname, std::uint16_t plain_port, std::uint16_t tls_port); diff --git a/core/topology/configuration_json.hxx b/core/topology/configuration_json.hxx index b6c5ffb9a..e06528c3a 100644 --- a/core/topology/configuration_json.hxx +++ b/core/topology/configuration_json.hxx @@ -55,6 +55,9 @@ struct traits { if (const auto& hostname = o.find("hostname"); hostname != o.end()) { n.hostname = hostname->second.get_string(); } + if (const auto& group = o.find("serverGroup"); group != o.end()) { + n.server_group = group->second.get_string(); + } const auto& s = o.at("services"); n.services_plain.key_value = s.template optional("kv"); n.services_plain.management = s.template optional("mgmt"); diff --git a/core/utils/connection_string.cxx b/core/utils/connection_string.cxx index 418ba7edf..ac2774070 100644 --- a/core/utils/connection_string.cxx +++ b/core/utils/connection_string.cxx @@ -162,7 +162,9 @@ struct action { static void apply(const ActionInput& in, connection_string& /* cs */, connection_string::node& cur_node) { std::string mode = in.string(); - std::transform(mode.begin(), mode.end(), mode.begin(), [](unsigned char c) { return std::tolower(c); }); + std::transform(mode.begin(), mode.end(), mode.begin(), [](unsigned char c) { + return std::tolower(c); + }); if (mode == "mcd" || mode == "gcccp" || mode == "cccp") { cur_node.mode = connection_string::bootstrap_mode::gcccp; } else if (mode == "http") { @@ -430,6 +432,8 @@ extract_options(connection_string& connstr) * Whether to dump every new configuration on TRACE level */ parse_option(connstr.options.dump_configuration, name, value, connstr.warnings); + } else if (name == "server_group") { + parse_option(connstr.options.server_group, name, value, connstr.warnings); } else { connstr.warnings.push_back(fmt::format(R"(unknown parameter "{}" in connection string (value "{}"))", name, value)); } diff --git a/couchbase/get_all_replicas_options.hxx b/couchbase/get_all_replicas_options.hxx index 180b3111f..cc57ab0cc 100644 --- a/couchbase/get_all_replicas_options.hxx +++ b/couchbase/get_all_replicas_options.hxx @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -43,8 +44,26 @@ struct get_all_replicas_options : public common_options::built { + couchbase::read_preference read_preference; }; + /** + * Choose how the replica nodes will be selected. By default it has no + * preference and will select all replicas available. But it is possible to + * prioritize or restrict to only nodes in local server group. + * + * @param preference + * @return this options builder for chaining purposes. + * + * @since 1.0.0 + * @volatile + */ + auto read_preference(read_preference preference) -> get_all_replicas_options& + { + read_preference_ = preference; + return self(); + } + /** * Validates options and returns them as an immutable value. * @@ -57,8 +76,16 @@ struct get_all_replicas_options : public common_options built { - return { build_common_options() }; + return { + build_common_options(), + read_preference_, + }; } + + private: + enum read_preference read_preference_ { + read_preference::no_preference + }; }; /** diff --git a/couchbase/get_any_replica_options.hxx b/couchbase/get_any_replica_options.hxx index d48eab1d5..a572cd30f 100644 --- a/couchbase/get_any_replica_options.hxx +++ b/couchbase/get_any_replica_options.hxx @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -42,8 +43,26 @@ struct get_any_replica_options : public common_options * @internal */ struct built : public common_options::built { + couchbase::read_preference read_preference; }; + /** + * Choose how the replica nodes will be selected. By default it has no + * preference and will select any available replica, but it is possible to + * prioritize or restrict to only nodes in local server group. + * + * @param preference + * @return this options builder for chaining purposes. + * + * @since 1.0.0 + * @volatile + */ + auto read_preference(read_preference preference) -> get_any_replica_options& + { + read_preference_ = preference; + return self(); + } + /** * Validates options and returns them as an immutable value. * @@ -56,8 +75,16 @@ struct get_any_replica_options : public common_options */ [[nodiscard]] auto build() const -> built { - return { build_common_options() }; + return { + build_common_options(), + read_preference_, + }; } + + private: + enum read_preference read_preference_ { + read_preference::no_preference + }; }; /** diff --git a/couchbase/lookup_in_all_replicas_options.hxx b/couchbase/lookup_in_all_replicas_options.hxx index 02aca77ae..a894b5345 100644 --- a/couchbase/lookup_in_all_replicas_options.hxx +++ b/couchbase/lookup_in_all_replicas_options.hxx @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -49,8 +50,26 @@ struct lookup_in_all_replicas_options : common_options::built { + couchbase::read_preference read_preference; }; + /** + * Choose how the replica nodes will be selected. By default it has no + * preference and will select any available replica, but it is possible to + * prioritize or restrict to only nodes in local server group. + * + * @param preference + * @return this options builder for chaining purposes. + * + * @since 1.0.0 + * @volatile + */ + auto read_preference(read_preference preference) -> lookup_in_all_replicas_options& + { + read_preference_ = preference; + return self(); + } + /** * Validates options and returns them as an immutable value. * @@ -63,8 +82,16 @@ struct lookup_in_all_replicas_options : common_options built { - return { build_common_options() }; + return { + build_common_options(), + read_preference_, + }; } + + private: + enum read_preference read_preference_ { + read_preference::no_preference + }; }; /** diff --git a/couchbase/lookup_in_any_replica_options.hxx b/couchbase/lookup_in_any_replica_options.hxx index 96f67cc06..b6e3520df 100644 --- a/couchbase/lookup_in_any_replica_options.hxx +++ b/couchbase/lookup_in_any_replica_options.hxx @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -49,8 +50,26 @@ struct lookup_in_any_replica_options : common_options::built { + couchbase::read_preference read_preference; }; + /** + * Choose how the replica nodes will be selected. By default it has no + * preference and will select any available replica, but it is possible to + * prioritize or restrict to only nodes in local server group. + * + * @param preference + * @return this options builder for chaining purposes. + * + * @since 1.0.0 + * @volatile + */ + auto read_preference(read_preference preference) -> lookup_in_any_replica_options& + { + read_preference_ = preference; + return self(); + } + /** * Validates options and returns them as an immutable value. * @@ -63,8 +82,16 @@ struct lookup_in_any_replica_options : common_options built { - return { build_common_options() }; + return { + build_common_options(), + read_preference_, + }; } + + private: + enum read_preference read_preference_ { + read_preference::no_preference + }; }; /** diff --git a/couchbase/network_options.hxx b/couchbase/network_options.hxx index a4365536d..7f2661e37 100644 --- a/couchbase/network_options.hxx +++ b/couchbase/network_options.hxx @@ -78,8 +78,31 @@ class network_options return *this; } + /** + * Select server group to use for replica APIs. + * + * For some use-cases it might be necessary to restrict list of the nodes, + * that are used in replica read APIs to single server group to optimize + * network costs. + * + * @see read_preference + * + * @see collection::get_all_replicas + * @see collection::get_any_replica + * @see collection::lookup_in_all_replicas + * @see collection::lookup_in_any_replica + * + * @see https://docs.couchbase.com/server/current/manage/manage-groups/manage-groups.html + */ + auto preferred_server_group(std::string server_group) -> network_options& + { + server_group_ = std::move(server_group); + return *this; + } + struct built { std::string network; + std::string server_group; bool enable_tcp_keep_alive; couchbase::ip_protocol ip_protocol; std::chrono::milliseconds tcp_keep_alive_interval; @@ -92,6 +115,7 @@ class network_options { return { network_, + server_group_, enable_tcp_keep_alive_, ip_protocol_, tcp_keep_alive_interval_, @@ -103,6 +127,7 @@ class network_options private: std::string network_{ "auto" }; + std::string server_group_{}; bool enable_tcp_keep_alive_{ true }; ip_protocol ip_protocol_{ ip_protocol::any }; std::chrono::milliseconds tcp_keep_alive_interval_{ default_tcp_keep_alive_interval }; diff --git a/couchbase/read_preference.hxx b/couchbase/read_preference.hxx new file mode 100644 index 000000000..6e50621b5 --- /dev/null +++ b/couchbase/read_preference.hxx @@ -0,0 +1,58 @@ + +/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* + * Copyright 2020-Present Couchbase, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace couchbase +{ +/** + * Select read preference (or affinity) for the replica APIs such as: + * + * * collection::get_all_replicas + * * collection::get_any_replica + * * collection::lookup_in_all_replicas + * * collection::lookup_in_any_replica + * + * @note all strategies except read_preference::no_preference, reduce number of the nodes + * that the SDK will use for replica read operations. In other words, it will + * increase likelihood of getting `errc::key_value::document_irretrievable` + * error code if the filtered set of the nodes is empty, or do not have any + * documents available. + * + * @see https://docs.couchbase.com/server/current/manage/manage-groups/manage-groups.html + */ +enum class read_preference { + /** + * Do not enforce any filtering for replica set. + */ + no_preference, + + /** + * Exclude any nodes that do not belong to local group selected during + * cluster instantiation with network_options::preferred_server_group(). + */ + selected_server_group, + + /** + * The same as read_preference::selected_server_group, but if the filtered + * replica set is empty, expand it to all available nodes (fall back to + * read_preference::no_preference effectively). + */ + selected_server_group_or_all_available, +}; +} // namespace couchbase diff --git a/test/test_integration_read_replica.cxx b/test/test_integration_read_replica.cxx index 8982684a0..9f6b23580 100644 --- a/test/test_integration_read_replica.cxx +++ b/test/test_integration_read_replica.cxx @@ -15,7 +15,9 @@ * limitations under the License. */ +#include "couchbase/error_codes.hxx" #include "test_helper_integration.hxx" +#include "utils/integration_shortcuts.hxx" #include "utils/move_only_context.hxx" #include "core/operations/document_append.hxx" @@ -29,9 +31,17 @@ #include "core/operations/document_remove.hxx" #include "core/operations/document_replace.hxx" #include "core/operations/document_upsert.hxx" +#include "core/utils/connection_string.hxx" #include + +#include +#include #include +#include +#include +#include +#include static const tao::json::value basic_doc = { { "a", 1.0 }, @@ -146,7 +156,9 @@ TEST_CASE("integration: get all replicas", "[integration]") auto [ctx, result] = collection.get_all_replicas(key, {}).get(); REQUIRE_SUCCESS(ctx.ec()); REQUIRE(result.size() == number_of_replicas + 1); - auto responses_from_active = std::count_if(result.begin(), result.end(), [](const auto& r) { return !r.is_replica(); }); + auto responses_from_active = std::count_if(result.begin(), result.end(), [](const auto& r) { + return !r.is_replica(); + }); REQUIRE(responses_from_active == 1); } } @@ -275,7 +287,556 @@ TEST_CASE("integration: get all replicas low-level version", "[integration]") auto resp = test::utils::execute(integration.cluster, req); REQUIRE_SUCCESS(resp.ctx.ec()); REQUIRE(resp.entries.size() == number_of_replicas + 1); - auto responses_from_active = std::count_if(resp.entries.begin(), resp.entries.end(), [](const auto& r) { return !r.replica; }); + auto responses_from_active = std::count_if(resp.entries.begin(), resp.entries.end(), [](const auto& r) { + return !r.replica; + }); REQUIRE(responses_from_active == 1); } } + +TEST_CASE("integration: low-level zone-aware read replicas on balanced cluster", "[integration]") +{ + test::utils::integration_test_guard integration; + + if (integration.cluster_version().is_mock()) { + SKIP("GOCAVES does not support server groups"); + } + + const auto number_of_replicas = integration.number_of_replicas(); + if (number_of_replicas == 0) { + SKIP("bucket has zero replicas"); + } + if (integration.number_of_nodes() <= number_of_replicas) { + SKIP(fmt::format( + "number of nodes ({}) is less or equal to number of replicas ({})", integration.number_of_nodes(), number_of_replicas)); + } + + const auto server_groups = integration.server_groups(); + if (server_groups.size() != 2) { + SKIP(fmt::format("This test expects exactly 2 server groups and at least one replica, " + "but found {} server groups", + integration.server_groups().size())); + } + + asio::io_context io{}; + couchbase::core::cluster cluster(io); + auto io_thread = std::thread([&io]() { + io.run(); + }); + + auto connection_string = couchbase::core::utils::parse_connection_string(integration.ctx.connection_string); + connection_string.options.server_group = server_groups.front(); + + auto origin = couchbase::core::origin(integration.ctx.build_auth(), connection_string); + test::utils::open_cluster(cluster, origin); + test::utils::open_bucket(cluster, integration.ctx.bucket); + + couchbase::core::document_id id{ integration.ctx.bucket, "_default", "_default", test::utils::uniq_id("foo") }; + { + const tao::json::value value = { + { "a", 1.0 }, + { "b", 2.0 }, + }; + couchbase::core::operations::upsert_request req{ id, couchbase::core::utils::json::generate_binary(value) }; + req.durability_level = couchbase::durability_level::majority_and_persist_to_active; + auto resp = test::utils::execute(cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec()); + } + + { + couchbase::core::operations::get_all_replicas_request req{ id, {}, couchbase::read_preference::no_preference }; + auto resp = test::utils::execute(cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec()); + REQUIRE(resp.entries.size() == number_of_replicas + 1); + } + + { + couchbase::core::operations::get_all_replicas_request req{ id, {}, couchbase::read_preference::selected_server_group }; + auto resp = test::utils::execute(cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec()); + REQUIRE(resp.entries.size() <= number_of_replicas + 1); + REQUIRE(resp.entries.size() > 0); + } + + { + couchbase::core::operations::get_any_replica_request req{ id, {}, couchbase::read_preference::no_preference }; + auto resp = test::utils::execute(cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec()); + REQUIRE_FALSE(resp.value.empty()); + } + + { + couchbase::core::operations::get_any_replica_request req{ id, {}, couchbase::read_preference::selected_server_group }; + auto resp = test::utils::execute(cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec()); + REQUIRE_FALSE(resp.value.empty()); + } + + { + couchbase::core::operations::lookup_in_any_replica_request req{ + id, + couchbase::lookup_in_specs{ couchbase::lookup_in_specs::get("a") }.specs(), + {}, + {}, + couchbase::read_preference::no_preference, + }; + auto resp = test::utils::execute(cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec()); + REQUIRE_FALSE(resp.fields.empty()); + } + + { + couchbase::core::operations::lookup_in_any_replica_request req{ + id, + couchbase::lookup_in_specs{ couchbase::lookup_in_specs::get("a") }.specs(), + {}, + {}, + couchbase::read_preference::selected_server_group, + }; + auto resp = test::utils::execute(cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec()); + REQUIRE_FALSE(resp.fields.empty()); + } + + { + couchbase::core::operations::lookup_in_all_replicas_request req{ + id, + couchbase::lookup_in_specs{ couchbase::lookup_in_specs::get("a") }.specs(), + {}, + {}, + couchbase::read_preference::no_preference, + }; + auto resp = test::utils::execute(cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec()); + REQUIRE(resp.entries.size() <= number_of_replicas + 1); + REQUIRE(resp.entries.size() > 0); + } + + { + couchbase::core::operations::lookup_in_all_replicas_request req{ + id, + couchbase::lookup_in_specs{ couchbase::lookup_in_specs::get("a") }.specs(), + {}, + {}, + couchbase::read_preference::selected_server_group, + }; + auto resp = test::utils::execute(cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec()); + REQUIRE(resp.entries.size() <= number_of_replicas + 1); + REQUIRE(resp.entries.size() > 0); + } + + test::utils::close_cluster(cluster); + io_thread.join(); +} + +TEST_CASE("integration: low-level zone-aware read replicas on unbalanced cluster", "[integration]") +{ + test::utils::integration_test_guard integration; + + if (integration.cluster_version().is_mock()) { + SKIP("GOCAVES does not support server groups"); + } + + const auto number_of_replicas = integration.number_of_replicas(); + if (number_of_replicas == 0) { + SKIP("bucket has zero replicas"); + } + if (integration.number_of_nodes() <= number_of_replicas) { + SKIP(fmt::format( + "number of nodes ({}) is less or equal to number of replicas ({})", integration.number_of_nodes(), number_of_replicas)); + } + + const auto server_groups = integration.server_groups(); + if (server_groups.size() < 3 || number_of_replicas > 1) { + SKIP(fmt::format("{} server groups and {} replicas does not meet expected requirements of unbalanced cluster. " + "The number of replicas + 1 has to be less than number of the groups", + integration.server_groups().size(), + number_of_replicas)); + } + + // Now we need to craft key, for which both active and replica vbuckets + // are not bound to selected server group. + const auto& selected_server_group = server_groups.front(); + const auto selected_key = integration.generate_key_not_in_server_group(selected_server_group); + INFO(fmt::format("server group: \"{}\"\nkey: \"{}\"", selected_server_group, selected_key)); + + asio::io_context io{}; + couchbase::core::cluster cluster(io); + auto io_thread = std::thread([&io]() { + io.run(); + }); + + auto connection_string = couchbase::core::utils::parse_connection_string(integration.ctx.connection_string); + connection_string.options.server_group = selected_server_group; + + auto origin = couchbase::core::origin(integration.ctx.build_auth(), connection_string); + test::utils::open_cluster(cluster, origin); + test::utils::open_bucket(cluster, integration.ctx.bucket); + + couchbase::core::document_id id{ integration.ctx.bucket, "_default", "_default", selected_key }; + { + const tao::json::value value = { + { "a", 1.0 }, + { "b", 2.0 }, + }; + couchbase::core::operations::upsert_request req{ id, couchbase::core::utils::json::generate_binary(value) }; + req.durability_level = couchbase::durability_level::majority_and_persist_to_active; + auto resp = test::utils::execute(cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec()); + } + + { + couchbase::core::operations::get_all_replicas_request req{ id, {}, couchbase::read_preference::no_preference }; + auto resp = test::utils::execute(cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec()); + REQUIRE(resp.entries.size() == number_of_replicas + 1); + } + + { + couchbase::core::operations::get_all_replicas_request req{ id, {}, couchbase::read_preference::selected_server_group }; + auto resp = test::utils::execute(cluster, req); + REQUIRE(resp.ctx.ec() == couchbase::errc::key_value::document_irretrievable); + } + + { + couchbase::core::operations::get_any_replica_request req{ id, {}, couchbase::read_preference::no_preference }; + auto resp = test::utils::execute(cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec()); + REQUIRE_FALSE(resp.value.empty()); + } + + { + couchbase::core::operations::get_any_replica_request req{ id, {}, couchbase::read_preference::selected_server_group }; + auto resp = test::utils::execute(cluster, req); + REQUIRE(resp.ctx.ec() == couchbase::errc::key_value::document_irretrievable); + } + + { + couchbase::core::operations::lookup_in_any_replica_request req{ + id, + couchbase::lookup_in_specs{ couchbase::lookup_in_specs::get("a") }.specs(), + {}, + {}, + couchbase::read_preference::no_preference, + }; + auto resp = test::utils::execute(cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec()); + REQUIRE_FALSE(resp.fields.empty()); + } + + { + couchbase::core::operations::lookup_in_any_replica_request req{ + id, + couchbase::lookup_in_specs{ couchbase::lookup_in_specs::get("a") }.specs(), + {}, + {}, + couchbase::read_preference::selected_server_group, + }; + auto resp = test::utils::execute(cluster, req); + REQUIRE(resp.ctx.ec() == couchbase::errc::key_value::document_irretrievable); + } + + { + couchbase::core::operations::lookup_in_all_replicas_request req{ + id, + couchbase::lookup_in_specs{ couchbase::lookup_in_specs::get("a") }.specs(), + {}, + {}, + couchbase::read_preference::no_preference, + }; + auto resp = test::utils::execute(cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec()); + REQUIRE(resp.entries.size() == number_of_replicas + 1); + } + + { + couchbase::core::operations::lookup_in_all_replicas_request req{ + id, + couchbase::lookup_in_specs{ couchbase::lookup_in_specs::get("a") }.specs(), + {}, + {}, + couchbase::read_preference::selected_server_group, + }; + auto resp = test::utils::execute(cluster, req); + REQUIRE(resp.ctx.ec() == couchbase::errc::key_value::document_irretrievable); + } + + test::utils::close_cluster(cluster); + io_thread.join(); +} + +TEST_CASE("integration: zone-aware read replicas on balanced cluster", "[integration]") +{ + test::utils::integration_test_guard integration; + + if (integration.cluster_version().is_mock()) { + SKIP("GOCAVES does not support server groups"); + } + + const auto number_of_replicas = integration.number_of_replicas(); + if (number_of_replicas == 0) { + SKIP("bucket has zero replicas"); + } + if (integration.number_of_nodes() <= number_of_replicas) { + SKIP(fmt::format( + "number of nodes ({}) is less or equal to number of replicas ({})", integration.number_of_nodes(), number_of_replicas)); + } + + const auto server_groups = integration.server_groups(); + if (server_groups.size() != 2) { + SKIP(fmt::format("This test expects exactly 2 server groups and at least one replica, " + "but found {} server groups", + integration.server_groups().size())); + } + + couchbase::core::document_id id{ + integration.ctx.bucket, + couchbase::scope::default_name, + couchbase::collection::default_name, + test::utils::uniq_id("foo"), + }; + { + const tao::json::value value = { + { "a", 1.0 }, + { "b", 2.0 }, + }; + couchbase::core::operations::upsert_request req{ id, couchbase::core::utils::json::generate_binary(value) }; + req.durability_level = couchbase::durability_level::majority_and_persist_to_active; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec()); + } + + asio::io_context io{}; + auto guard = asio::make_work_guard(io); + auto io_thread = std::thread([&io]() { + io.run(); + }); + + auto connection_string = couchbase::core::utils::parse_connection_string(integration.ctx.connection_string); + connection_string.options.server_group = server_groups.front(); + + auto cluster_options = + integration.ctx.certificate_path.empty() + ? couchbase::cluster_options(couchbase::password_authenticator(integration.ctx.username, integration.ctx.password)) + : couchbase::cluster_options( + couchbase::certificate_authenticator(integration.ctx.certificate_path, integration.ctx.certificate_path)); + cluster_options.network().preferred_server_group(server_groups.front()); + auto [cluster, ec] = couchbase::cluster::connect(io, integration.ctx.connection_string, cluster_options).get(); + REQUIRE_SUCCESS(ec); + + auto collection = cluster.bucket(id.bucket()).scope(id.scope()).collection(id.collection()); + { + auto [ctx, result] = collection.get_any_replica(id.key(), {}).get(); + REQUIRE_SUCCESS(ctx.ec()); + } + { + auto [ctx, result] = + collection + .get_any_replica(id.key(), + couchbase::get_any_replica_options{}.read_preference(couchbase::read_preference::selected_server_group)) + .get(); + REQUIRE_SUCCESS(ctx.ec()); + } + { + auto [ctx, result] = collection.get_all_replicas(id.key(), {}).get(); + REQUIRE_SUCCESS(ctx.ec()); + REQUIRE(result.size() == number_of_replicas + 1); + } + { + auto [ctx, result] = + collection + .get_all_replicas(id.key(), + couchbase::get_all_replicas_options{}.read_preference(couchbase::read_preference::selected_server_group)) + .get(); + REQUIRE_SUCCESS(ctx.ec()); + REQUIRE(result.size() <= number_of_replicas + 1); + } + + { + auto [ctx, result] = collection + .lookup_in_any_replica(id.key(), + couchbase::lookup_in_specs{ + couchbase::lookup_in_specs::get("a"), + }, + {}) + .get(); + REQUIRE_SUCCESS(ctx.ec()); + } + { + auto [ctx, result] = collection + .lookup_in_any_replica(id.key(), + couchbase::lookup_in_specs{ + couchbase::lookup_in_specs::get("a"), + }, + couchbase::lookup_in_any_replica_options{}.read_preference( + couchbase::read_preference::selected_server_group)) + .get(); + REQUIRE_SUCCESS(ctx.ec()); + } + { + auto [ctx, result] = collection + .lookup_in_all_replicas(id.key(), + couchbase::lookup_in_specs{ + couchbase::lookup_in_specs::get("a"), + }) + .get(); + REQUIRE_SUCCESS(ctx.ec()); + } + { + auto [ctx, result] = collection + .lookup_in_all_replicas(id.key(), + couchbase::lookup_in_specs{ + couchbase::lookup_in_specs::get("a"), + }, + couchbase::lookup_in_all_replicas_options{}.read_preference( + couchbase::read_preference::selected_server_group)) + .get(); + REQUIRE_SUCCESS(ctx.ec()); + REQUIRE(result.size() <= number_of_replicas + 1); + } + + cluster.close(); + guard.reset(); + io_thread.join(); +} + +TEST_CASE("integration: zone-aware read replicas on unbalanced cluster", "[integration]") +{ + test::utils::integration_test_guard integration; + + if (integration.cluster_version().is_mock()) { + SKIP("GOCAVES does not support server groups"); + } + + const auto number_of_replicas = integration.number_of_replicas(); + if (number_of_replicas == 0) { + SKIP("bucket has zero replicas"); + } + if (integration.number_of_nodes() <= number_of_replicas) { + SKIP(fmt::format( + "number of nodes ({}) is less or equal to number of replicas ({})", integration.number_of_nodes(), number_of_replicas)); + } + + const auto server_groups = integration.server_groups(); + if (server_groups.size() < 3 || number_of_replicas > 1) { + SKIP(fmt::format("{} server groups and {} replicas does not meet expected requirements of unbalanced cluster. " + "The number of replicas + 1 has to be less than number of the groups", + integration.server_groups().size(), + number_of_replicas)); + } + + // Now we need to craft key, for which both active and replica vbuckets + // are not bound to selected server group. + const auto& selected_server_group = server_groups.front(); + const auto selected_key = integration.generate_key_not_in_server_group(selected_server_group); + INFO(fmt::format("server group: \"{}\"\nkey: \"{}\"", selected_server_group, selected_key)); + + couchbase::core::document_id id{ + integration.ctx.bucket, + couchbase::scope::default_name, + couchbase::collection::default_name, + selected_key, + }; + { + const tao::json::value value = { + { "a", 1.0 }, + { "b", 2.0 }, + }; + couchbase::core::operations::upsert_request req{ + id, + couchbase::core::utils::json::generate_binary(value), + }; + req.durability_level = couchbase::durability_level::majority_and_persist_to_active; + auto resp = test::utils::execute(integration.cluster, req); + REQUIRE_SUCCESS(resp.ctx.ec()); + } + + asio::io_context io{}; + auto guard = asio::make_work_guard(io); + auto io_thread = std::thread([&io]() { + io.run(); + }); + + auto cluster_options = + integration.ctx.certificate_path.empty() + ? couchbase::cluster_options(couchbase::password_authenticator(integration.ctx.username, integration.ctx.password)) + : couchbase::cluster_options( + couchbase::certificate_authenticator(integration.ctx.certificate_path, integration.ctx.certificate_path)); + cluster_options.network().preferred_server_group(selected_server_group); + auto [cluster, ec] = couchbase::cluster::connect(io, integration.ctx.connection_string, cluster_options).get(); + REQUIRE_SUCCESS(ec); + + auto collection = cluster.bucket(id.bucket()).scope(id.scope()).collection(id.collection()); + { + auto [ctx, result] = collection.get_any_replica(id.key(), {}).get(); + REQUIRE_SUCCESS(ctx.ec()); + } + { + auto [ctx, result] = + collection + .get_any_replica(id.key(), + couchbase::get_any_replica_options{}.read_preference(couchbase::read_preference::selected_server_group)) + .get(); + REQUIRE(ctx.ec() == couchbase::errc::key_value::document_irretrievable); + } + { + auto [ctx, result] = collection.get_all_replicas(id.key(), {}).get(); + REQUIRE_SUCCESS(ctx.ec()); + REQUIRE(result.size() == number_of_replicas + 1); + } + { + auto [ctx, result] = + collection + .get_all_replicas(id.key(), + couchbase::get_all_replicas_options{}.read_preference(couchbase::read_preference::selected_server_group)) + .get(); + REQUIRE(ctx.ec() == couchbase::errc::key_value::document_irretrievable); + } + + { + auto [ctx, result] = collection + .lookup_in_any_replica(id.key(), + couchbase::lookup_in_specs{ + couchbase::lookup_in_specs::get("a"), + }, + {}) + .get(); + REQUIRE_SUCCESS(ctx.ec()); + } + { + auto [ctx, result] = collection + .lookup_in_any_replica(id.key(), + couchbase::lookup_in_specs{ + couchbase::lookup_in_specs::get("a"), + }, + couchbase::lookup_in_any_replica_options{}.read_preference( + couchbase::read_preference::selected_server_group)) + .get(); + REQUIRE(ctx.ec() == couchbase::errc::key_value::document_irretrievable); + } + { + auto [ctx, result] = collection + .lookup_in_all_replicas(id.key(), + couchbase::lookup_in_specs{ + couchbase::lookup_in_specs::get("a"), + }) + .get(); + REQUIRE_SUCCESS(ctx.ec()); + } + { + auto [ctx, result] = collection + .lookup_in_all_replicas(id.key(), + couchbase::lookup_in_specs{ + couchbase::lookup_in_specs::get("a"), + }, + couchbase::lookup_in_all_replicas_options{}.read_preference( + couchbase::read_preference::selected_server_group)) + .get(); + REQUIRE(ctx.ec() == couchbase::errc::key_value::document_irretrievable); + } + + cluster.close(); + guard.reset(); + io_thread.join(); +} diff --git a/test/utils/integration_test_guard.cxx b/test/utils/integration_test_guard.cxx index f9f96d849..50e239b66 100644 --- a/test/utils/integration_test_guard.cxx +++ b/test/utils/integration_test_guard.cxx @@ -19,10 +19,13 @@ #include "core/logger/logger.hxx" #include "core/operations/management/freeform.hxx" +#include "core/protocol/cmd_get_cluster_config.hxx" #include "core/transactions.hxx" #include "core/utils/connection_string.hxx" +#include "core/utils/join_strings.hxx" #include "core/utils/json.hxx" #include "logger.hxx" +#include "test_data.hxx" namespace test::utils { @@ -234,4 +237,53 @@ integration_test_guard::cluster_version() return parsed_version; } +auto +integration_test_guard::server_groups() -> std::vector +{ + auto bucket_info = load_bucket_info(ctx.bucket); + std::vector groups; + for (const auto& [name, _] : bucket_info.server_groups) { + groups.emplace_back(name); + } + return groups; +} + +auto +integration_test_guard::generate_key_not_in_server_group(const std::string& group_name) -> std::string +{ + auto bucket_info = load_bucket_info(ctx.bucket); + + if (bucket_info.server_groups.count(group_name) == 0) { + auto message = fmt::format("group {} does not exist on the server", group_name); + throw std::runtime_error(message.c_str()); + } + + auto group = bucket_info.server_groups[group_name]; + + std::set local_vbuckets; + for (const auto& node : group.nodes) { + for (const auto& vbucket : node.active_vbuckets) { + local_vbuckets.insert(vbucket); + } + for (const auto& vbucket : node.replica_vbuckets) { + local_vbuckets.insert(vbucket); + } + } + const auto config = couchbase::core::protocol::parse_config(bucket_info.config_json, "127.0.0.1", 11210); + if (local_vbuckets.size() >= config.vbmap->size()) { + auto message = fmt::format("group {} covers all vbuckets, unable to generate key that is not in server group", group_name); + throw std::runtime_error(message.c_str()); + } + + for (;;) { + auto id = uniq_id(group_name); + for (std::size_t idx = 0; idx < config.num_replicas.value_or(0) + 1; ++idx) { + auto [vbid, server] = config.map_key(id, idx); + if (server && local_vbuckets.count(vbid) == 0) { + return id; + } + } + } +} + } // namespace test::utils diff --git a/test/utils/integration_test_guard.hxx b/test/utils/integration_test_guard.hxx index cef00bcf9..2aa699a25 100644 --- a/test/utils/integration_test_guard.hxx +++ b/test/utils/integration_test_guard.hxx @@ -28,6 +28,8 @@ #include #include +#include +#include namespace test::utils { @@ -57,6 +59,9 @@ class integration_test_guard return load_bucket_info(ctx.bucket).number_of_nodes; } + auto server_groups() -> std::vector; + auto generate_key_not_in_server_group(const std::string& group_name) -> std::string; + std::size_t number_of_nodes(const std::string& bucket_name) { return load_bucket_info(bucket_name).number_of_nodes; From 796033909ffd94ed7692cef69837173bda8921cf Mon Sep 17 00:00:00 2001 From: Sergey Avseyev Date: Tue, 21 May 2024 09:40:51 -0700 Subject: [PATCH 10/11] CXXCBC-445: return request_canceled on IO error in HTTP session (#568) --- core/io/http_command.hxx | 20 +++++---- core/io/http_session.hxx | 58 +++++++++++++------------- core/io/http_traits.hxx | 7 ++++ core/operations/document_analytics.hxx | 4 ++ core/operations/document_query.hxx | 4 ++ 5 files changed, 57 insertions(+), 36 deletions(-) diff --git a/core/io/http_command.hxx b/core/io/http_command.hxx index 0a5f0897d..178a017ed 100644 --- a/core/io/http_command.hxx +++ b/core/io/http_command.hxx @@ -95,16 +95,22 @@ struct http_command : public std::enable_shared_from_this> if (ec == asio::error::operation_aborted) { return; } - self->cancel(); + if constexpr (io::http_traits::supports_readonly_v) { + if (self->request.readonly) { + self->cancel(errc::common::unambiguous_timeout); + return; + } + } + self->cancel(errc::common::ambiguous_timeout); }); } - void cancel() + void cancel(std::error_code ec) { + invoke_handler(ec, {}); if (session_) { session_->stop(); } - invoke_handler(errc::common::unambiguous_timeout, {}); } void invoke_handler(std::error_code ec, io::http_response&& msg) @@ -113,10 +119,9 @@ struct http_command : public std::enable_shared_from_this> span_->end(); span_ = nullptr; } - if (handler_) { - handler_(ec, std::move(msg)); + if (auto handler = std::move(handler_); handler) { + handler(ec, std::move(msg)); } - handler_ = nullptr; retry_backoff.cancel(); deadline.cancel(); } @@ -166,10 +171,11 @@ struct http_command : public std::enable_shared_from_this> } self->deadline.cancel(); self->finish_dispatch(self->session_->remote_address(), self->session_->local_address()); - CB_LOG_TRACE(R"({} HTTP response: {}, client_context_id="{}", status={}, body={})", + CB_LOG_TRACE(R"({} HTTP response: {}, client_context_id="{}", ec={}, status={}, body={})", self->session_->log_prefix(), self->request.type, self->client_context_id_, + ec.message(), msg.status_code, msg.status_code == 200 ? "[hidden]" : msg.body.data()); if (auto parser_ec = msg.body.ec(); !ec && parser_ec) { diff --git a/core/io/http_session.hxx b/core/io/http_session.hxx index 74fd42616..3bf29e0fe 100644 --- a/core/io/http_session.hxx +++ b/core/io/http_session.hxx @@ -123,7 +123,7 @@ class http_session : public std::enable_shared_from_this , ctx_(ctx) , resolver_(ctx_) , stream_(std::make_unique(ctx_)) - , deadline_timer_(stream_->get_executor()) + , connect_deadline_timer_(stream_->get_executor()) , idle_timer_(stream_->get_executor()) , credentials_(credentials) , hostname_(hostname) @@ -148,7 +148,7 @@ class http_session : public std::enable_shared_from_this , ctx_(ctx) , resolver_(ctx_) , stream_(std::make_unique(ctx_, tls)) - , deadline_timer_(ctx_) + , connect_deadline_timer_(ctx_) , idle_timer_(ctx_) , credentials_(credentials) , hostname_(hostname) @@ -245,6 +245,14 @@ class http_session : public std::enable_shared_from_this on_stop_handler_ = std::move(handler); } + void cancel_current_response(std::error_code ec) + { + std::scoped_lock lock(current_response_mutex_); + if (auto ctx = std::move(current_response_); ctx.handler) { + ctx.handler(ec, std::move(ctx.parser.response)); + } + } + void stop() { if (stopped_) { @@ -252,17 +260,12 @@ class http_session : public std::enable_shared_from_this } stopped_ = true; state_ = diag::endpoint_state::disconnecting; - stream_->close([](std::error_code) {}); - deadline_timer_.cancel(); + stream_->close([](std::error_code) { + }); + connect_deadline_timer_.cancel(); idle_timer_.cancel(); - { - std::scoped_lock lock(current_response_mutex_); - auto ctx = std::move(current_response_); - if (ctx.handler) { - ctx.handler(errc::common::ambiguous_timeout, {}); - } - } + cancel_current_response(errc::common::request_canceled); if (auto handler = std::move(on_stop_handler_); handler) { handler(); @@ -306,7 +309,9 @@ class http_session : public std::enable_shared_from_this if (stopped_) { return; } - asio::post(asio::bind_executor(ctx_, [self = shared_from_this()]() { self->do_write(); })); + asio::post(asio::bind_executor(ctx_, [self = shared_from_this()]() { + self->do_write(); + })); } template @@ -380,7 +385,6 @@ class http_session : public std::enable_shared_from_this endpoints_ = endpoints; CB_LOG_TRACE("{} resolved \"{}:{}\" to {} endpoint(s)", info_.log_prefix(), hostname_, service_, endpoints_.size()); do_connect(endpoints_.begin()); - deadline_timer_.async_wait(std::bind(&http_session::check_deadline, shared_from_this(), std::placeholders::_1)); } void do_connect(asio::ip::tcp::resolver::results_type::iterator it) @@ -396,7 +400,15 @@ class http_session : public std::enable_shared_from_this hostname_, service_, http_ctx_.options.connect_timeout.count()); - deadline_timer_.expires_after(http_ctx_.options.connect_timeout); + connect_deadline_timer_.async_wait([self = shared_from_this()](std::error_code ec) { + if (ec == asio::error::operation_aborted || self->stopped_) { + return; + } + self->cancel_current_response(couchbase::errc::common::unambiguous_timeout); + self->stream_->close([](std::error_code) { + }); + }); + connect_deadline_timer_.expires_after(http_ctx_.options.connect_timeout); stream_->async_connect(it->endpoint(), std::bind(&http_session::on_connect, shared_from_this(), std::placeholders::_1, it)); } else { CB_LOG_ERROR("{} no more endpoints left to connect, \"{}:{}\" is not reachable", info_.log_prefix(), hostname_, service_); @@ -430,24 +442,11 @@ class http_session : public std::enable_shared_from_this std::scoped_lock lock(info_mutex_); info_ = http_session_info(client_id_, id_, stream_->local_endpoint(), it->endpoint()); } - deadline_timer_.cancel(); + connect_deadline_timer_.cancel(); flush(); } } - void check_deadline(std::error_code ec) - { - if (ec == asio::error::operation_aborted || stopped_) { - return; - } - if (deadline_timer_.expiry() <= asio::steady_timer::clock_type::now()) { - stream_->close([](std::error_code) {}); - deadline_timer_.cancel(); - return; - } - deadline_timer_.async_wait(std::bind(&http_session::check_deadline, shared_from_this(), std::placeholders::_1)); - } - void do_read() { if (stopped_ || reading_ || !stream_->is_open()) { @@ -484,6 +483,7 @@ class http_session : public std::enable_shared_from_this res = self->current_response_.parser.feed(reinterpret_cast(self->input_buffer_.data()), bytes_transferred); } if (res.failure) { + CB_LOG_ERROR("{} Parsing error while reading from the socket: {}", self->info_.log_prefix(), res.error); return self->stop(); } if (res.complete) { @@ -557,7 +557,7 @@ class http_session : public std::enable_shared_from_this asio::io_context& ctx_; asio::ip::tcp::resolver resolver_; std::unique_ptr stream_; - asio::steady_timer deadline_timer_; + asio::steady_timer connect_deadline_timer_; asio::steady_timer idle_timer_; cluster_credentials credentials_; diff --git a/core/io/http_traits.hxx b/core/io/http_traits.hxx index 8d0fef959..5cc8e6aa4 100644 --- a/core/io/http_traits.hxx +++ b/core/io/http_traits.hxx @@ -35,4 +35,11 @@ struct supports_parent_span : public std::false_type { template inline constexpr bool supports_parent_span_v = supports_parent_span::value; +template +struct supports_readonly : public std::false_type { +}; + +template +inline constexpr bool supports_readonly_v = supports_readonly::value; + } // namespace couchbase::core::io::http_traits diff --git a/core/operations/document_analytics.hxx b/core/operations/document_analytics.hxx index efdb4c0f4..f814fcc23 100644 --- a/core/operations/document_analytics.hxx +++ b/core/operations/document_analytics.hxx @@ -111,4 +111,8 @@ namespace couchbase::core::io::http_traits template<> struct supports_parent_span : public std::true_type { }; + +template<> +struct supports_readonly : public std::true_type { +}; } // namespace couchbase::core::io::http_traits diff --git a/core/operations/document_query.hxx b/core/operations/document_query.hxx index 25d45f397..d9d0170ea 100644 --- a/core/operations/document_query.hxx +++ b/core/operations/document_query.hxx @@ -132,4 +132,8 @@ struct supports_sticky_node : public template<> struct supports_parent_span : public std::true_type { }; + +template<> +struct supports_readonly : public std::true_type { +}; } // namespace couchbase::core::io::http_traits From dfc421b18877ddc43cd12c619dbcad9ce083b13a Mon Sep 17 00:00:00 2001 From: Sergey Avseyev Date: Tue, 21 May 2024 10:30:57 -0700 Subject: [PATCH 11/11] CXXCBC-407: allow to use 0 max expiry for new collections (#569) --- core/operations/management/collection_create.cxx | 10 +++++----- core/operations/management/collection_create.hxx | 2 +- couchbase/create_collection_options.hxx | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/operations/management/collection_create.cxx b/core/operations/management/collection_create.cxx index 8bf1f59aa..0b0f81cdc 100644 --- a/core/operations/management/collection_create.cxx +++ b/core/operations/management/collection_create.cxx @@ -34,12 +34,12 @@ collection_create_request::encode_to(encoded_request_type& encoded, http_context encoded.path = fmt::format("/pools/default/buckets/{}/scopes/{}/collections", bucket_name, scope_name); encoded.headers["content-type"] = "application/x-www-form-urlencoded"; encoded.body = fmt::format("name={}", utils::string_codec::form_encode(collection_name)); - if (max_expiry >= -1) { - if (max_expiry != 0) { - encoded.body.append(fmt::format("&maxTTL={}", max_expiry)); + if (max_expiry) { + if (max_expiry >= -1) { + encoded.body.append(fmt::format("&maxTTL={}", max_expiry.value())); + } else { + return couchbase::errc::common::invalid_argument; } - } else { - return couchbase::errc::common::invalid_argument; } if (history.has_value()) { encoded.body.append(fmt::format("&history={}", history.value())); diff --git a/core/operations/management/collection_create.hxx b/core/operations/management/collection_create.hxx index f16e71607..510bee634 100644 --- a/core/operations/management/collection_create.hxx +++ b/core/operations/management/collection_create.hxx @@ -41,7 +41,7 @@ struct collection_create_request { std::string bucket_name; std::string scope_name; std::string collection_name; - std::int32_t max_expiry{ 0 }; + std::optional max_expiry{}; std::optional history{}; std::optional client_context_id{}; diff --git a/couchbase/create_collection_options.hxx b/couchbase/create_collection_options.hxx index 61a11d292..dddc37093 100644 --- a/couchbase/create_collection_options.hxx +++ b/couchbase/create_collection_options.hxx @@ -61,7 +61,7 @@ struct create_collection_settings { * The maximum expiry, in seconds, for documents in this collection. Values greater than or equal to -1 are valid. * Value of 0 sets max_expiry to the bucket-level setting and value of -1 to set it as no-expiry. */ - std::int32_t max_expiry{ 0 }; + std::optional max_expiry{}; /** * Whether history retention should be enabled. If unset, the bucket-level setting is used.