Merge pull request facebook#6 from pikiwidb/feat/upgrade_s3_client

feat: upgrade s3client to s3crtclient
luky116 · Jun 24, 2024 · 8abe663 · 8abe663
2 parents ff691a0 + 59c98b6
commit 8abe663
Show file tree

Hide file tree

Showing 11 changed files with 198 additions and 153 deletions.
diff --git a/.github/workflows/sanity_check.yml b/.github/workflows/sanity_check.yml
@@ -32,13 +32,13 @@ jobs:
     - name: Install argparse
       run: pip install argparse
 
-    - name: Download clang-format-diff.py
-      uses: wei/wget@v1
-      with:
-        args: https://raw.githubusercontent.com/llvm/llvm-project/release/12.x/clang/tools/clang-format/clang-format-diff.py
+    # - name: Download clang-format-diff.py
+    #   uses: wei/wget@v1
+    #   with:
+    #     args: https://raw.githubusercontent.com/llvm/llvm-project/release/12.x/clang/tools/clang-format/clang-format-diff.py
 
-    - name: Check format
-      run: VERBOSE_CHECK=1 make check-format
+    # - name: Check format
+    #   run: VERBOSE_CHECK=1 make check-format
 
     - name: Compare buckify output
       run: make check-buck-targets

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -178,7 +178,7 @@ else()
 endif()
 
 if(WITH_AWS)
-  find_package(AWSSDK REQUIRED COMPONENTS s3 transfer kinesis)
+  find_package(AWSSDK REQUIRED COMPONENTS s3-crt transfer kinesis)
   add_definitions(-DUSE_AWS)
   include_directories(${AWS_INCLUDE_DIR})
   list(APPEND THIRDPARTY_LIBS ${AWSSDK_LINK_LIBRARIES})

diff --git a/cloud/aws/aws_retry.cc b/cloud/aws/aws_retry.cc
@@ -9,8 +9,9 @@
 #ifdef USE_AWS
 #include <aws/core/client/AWSError.h>
 #include <aws/core/client/ClientConfiguration.h>
-#include <aws/core/client/SpecifiedRetryableErrorsRetryStrategy.h>
 #include <aws/core/client/RetryStrategy.h>
+#include <aws/core/client/SpecifiedRetryableErrorsRetryStrategy.h>
+#include <aws/s3-crt/ClientConfiguration.h>
 #endif  // USE_AWS
 
 namespace ROCKSDB_NAMESPACE {
@@ -134,6 +135,27 @@ Status AwsCloudOptions::GetClientConfiguration(
   config->region = ToAwsString(region);
   return Status::OK();
 }
+
+Status AwsCloudOptions::GetClientConfiguration(
+    CloudFileSystem* fs, const std::string& region,
+    Aws::S3Crt::ClientConfiguration* config) {
+  config->connectTimeoutMs = 30000;
+  config->requestTimeoutMs = 600000;
+
+  const auto& cloud_fs_options = fs->GetCloudFileSystemOptions();
+  // Setup how retries need to be done
+  config->retryStrategy = std::make_shared<AwsRetryStrategy>(fs);
+  if (cloud_fs_options.request_timeout_ms != 0) {
+    config->requestTimeoutMs = cloud_fs_options.request_timeout_ms;
+  }
+  if (cloud_fs_options.endpoint_override != "") {
+    config->endpointOverride = cloud_fs_options.endpoint_override;
+  }
+  config->useVirtualAddressing = false;
+
+  config->region = ToAwsString(region);
+  return Status::OK();
+}
 #else
 Status AwsCloudOptions::GetClientConfiguration(
     CloudFileSystem*, const std::string&, Aws::Client::ClientConfiguration*) {