GH-38663: [C++] Add support for service-specific endpoint for S3 usin…

…g `AWS_ENDPOINT_URL_S3` (#39160) ### Rationale for this change See #38663 ### What changes are included in this PR? set variable `endpoint_override` according the environment variable, prefer service-specific endpoint url over global endpoint url. ### Are these changes tested? unittest ### Are there any user-facing changes? No * Closes: #38663 Lead-authored-by: messense <messense@icloud.com> Co-authored-by: Antoine Pitrou <antoine@python.org> Signed-off-by: Antoine Pitrou <antoine@python.org>
apache · Jan 8, 2024 · 1362122 · 1362122
1 parent 48f704e
commit 1362122
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 3 deletions.
diff --git a/cpp/src/arrow/filesystem/s3fs.cc b/cpp/src/arrow/filesystem/s3fs.cc
@@ -156,6 +156,7 @@ using internal::ToURLEncodedAwsString;
 
 static const char kSep = '/';
 constexpr char kAwsEndpointUrlEnvVar[] = "AWS_ENDPOINT_URL";
+constexpr char kAwsEndpointUrlS3EnvVar[] = "AWS_ENDPOINT_URL_S3";
 
 // -----------------------------------------------------------------------
 // S3ProxyOptions implementation
@@ -366,9 +367,15 @@ Result<S3Options> S3Options::FromUri(const Uri& uri, std::string* out_path) {
   } else {
     options.ConfigureDefaultCredentials();
   }
-  auto endpoint_env = arrow::internal::GetEnvVar(kAwsEndpointUrlEnvVar);
-  if (endpoint_env.ok()) {
-    options.endpoint_override = *endpoint_env;
+  // Prefer AWS service-specific endpoint url
+  auto s3_endpoint_env = arrow::internal::GetEnvVar(kAwsEndpointUrlS3EnvVar);
+  if (s3_endpoint_env.ok()) {
+    options.endpoint_override = *s3_endpoint_env;
+  } else {
+    auto endpoint_env = arrow::internal::GetEnvVar(kAwsEndpointUrlEnvVar);
+    if (endpoint_env.ok()) {
+      options.endpoint_override = *endpoint_env;
+    }
   }
 
   bool region_set = false;

diff --git a/cpp/src/arrow/filesystem/s3fs_test.cc b/cpp/src/arrow/filesystem/s3fs_test.cc
@@ -303,6 +303,11 @@ TEST_F(S3OptionsTest, FromUri) {
   ASSERT_RAISES(Invalid, S3Options::FromUri("s3://mybucket/?xxx=zzz", &path));
 
   // Endpoint from environment variable
+  {
+    EnvVarGuard endpoint_guard("AWS_ENDPOINT_URL_S3", "http://127.0.0.1:9000");
+    ASSERT_OK_AND_ASSIGN(options, S3Options::FromUri("s3://mybucket/", &path));
+    ASSERT_EQ(options.endpoint_override, "http://127.0.0.1:9000");
+  }
   {
     EnvVarGuard endpoint_guard("AWS_ENDPOINT_URL", "http://127.0.0.1:9000");
     ASSERT_OK_AND_ASSIGN(options, S3Options::FromUri("s3://mybucket/", &path));

diff --git a/docs/source/cpp/env_vars.rst b/docs/source/cpp/env_vars.rst
@@ -163,6 +163,13 @@ that changing their value later will have an effect.
 .. envvar:: AWS_ENDPOINT_URL
 
    Endpoint URL used for S3-like storage, for example Minio or s3.scality.
+   Alternatively, one can set :envvar:`AWS_ENDPOINT_URL_S3`.
+
+.. envvar:: AWS_ENDPOINT_URL_S3
+
+   Endpoint URL used for S3-like storage, for example Minio or s3.scality.
+   This takes precedence over :envvar:`AWS_ENDPOINT_URL` if both variables
+   are set.
 
 .. envvar:: GANDIVA_CACHE_SIZE