[feature](Recycler) Retry object storage request when meeting 429 htt…

…p error code (#37680) When encountering 429 http code, the client sdk would not do retry by default. This pr makes the client retry `max_s3_client_retry` times.
apache · Jul 17, 2024 · 3778f3f · 3778f3f
1 parent 59edcaf
commit 3778f3f
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 4 deletions.
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
@@ -196,4 +196,7 @@ CONF_Validator(s3_client_http_scheme, [](const std::string& config) -> bool {
     return config == "http" || config == "https";
 });
 
+// Max retry times for object storage request
+CONF_mInt64(max_s3_client_retry, "10");
+
 } // namespace doris::cloud::config
diff --git a/cloud/src/recycler/s3_accessor.cpp b/cloud/src/recycler/s3_accessor.cpp
@@ -21,6 +21,7 @@
 #include <aws/core/auth/AWSCredentials.h>
 #include <aws/core/client/DefaultRetryStrategy.h>
 #include <aws/s3/S3Client.h>
+#include <bvar/reducer.h>
 #include <gen_cpp/cloud.pb.h>
 
 #include <algorithm>
@@ -41,6 +42,26 @@
 #include "recycler/s3_obj_client.h"
 #include "recycler/storage_vault_accessor.h"
 
+namespace {
+
+bvar::Adder<uint64_t> too_many_request_http_retry_times("too_many_request_http_retry_times");
+
+class CustomRetryStrategy final : public Aws::Client::DefaultRetryStrategy {
+public:
+    CustomRetryStrategy(int maxRetries) : DefaultRetryStrategy(maxRetries) {}
+
+    bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error,
+                     long attemptedRetries) const override {
+        if (attemptedRetries < m_maxRetries &&
+            error.GetResponseCode() == Aws::Http::HttpResponseCode::TOO_MANY_REQUESTS) {
+            too_many_request_http_retry_times << 1;
+            return true;
+        }
+        return Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries);
+    }
+};
+} // namespace
+
 namespace doris::cloud {
 
 struct AccessorRateLimiter {
@@ -231,12 +252,15 @@ int S3Accessor::create(S3Conf conf, std::shared_ptr<S3Accessor>* accessor) {
 int S3Accessor::init() {
     switch (conf_.provider) {
     case S3Conf::AZURE: {
+        Azure::Storage::Blobs::BlobClientOptions options;
+        options.Retry.StatusCodes.insert(Azure::Core::Http::HttpStatusCode::TooManyRequests);
+        options.Retry.MaxRetries = config::max_s3_client_retry;
         auto cred =
                 std::make_shared<Azure::Storage::StorageSharedKeyCredential>(conf_.ak, conf_.sk);
         uri_ = fmt::format("{}://{}.blob.core.windows.net/{}", config::s3_client_http_scheme,
                            conf_.ak, conf_.bucket);
-        auto container_client =
-                std::make_shared<Azure::Storage::Blobs::BlobContainerClient>(uri_, cred);
+        auto container_client = std::make_shared<Azure::Storage::Blobs::BlobContainerClient>(
+                uri_, cred, std::move(options));
         // uri format for debug: ${scheme}://${ak}.blob.core.windows.net/${bucket}/${prefix}
         uri_ = uri_ + '/' + conf_.prefix;
         obj_client_ = std::make_shared<AzureObjClient>(std::move(container_client));
@@ -255,8 +279,8 @@ int S3Accessor::init() {
         if (config::s3_client_http_scheme == "http") {
             aws_config.scheme = Aws::Http::Scheme::HTTP;
         }
-        aws_config.retryStrategy = std::make_shared<Aws::Client::DefaultRetryStrategy>(
-                /*maxRetries = 10, scaleFactor = 25*/);
+        aws_config.retryStrategy = std::make_shared<CustomRetryStrategy>(
+                config::max_s3_client_retry /*scaleFactor = 25*/);
         auto s3_client = std::make_shared<Aws::S3::S3Client>(
                 std::move(aws_cred), std::move(aws_config),
                 Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,