diff --git a/include/aws/s3/private/s3_client_impl.h b/include/aws/s3/private/s3_client_impl.h index 18de331e6..fbe6d48ca 100644 --- a/include/aws/s3/private/s3_client_impl.h +++ b/include/aws/s3/private/s3_client_impl.h @@ -15,7 +15,9 @@ #include #include #include +#include #include +#include /* TODO automate this value in the future to prevent it from becoming out-of-sync. */ #define AWS_S3_CLIENT_VERSION "0.1.x" @@ -58,6 +60,40 @@ struct aws_s3_endpoint_options { /* HTTP port override. If zero, determine port based on TLS context */ uint16_t port; + + /** + * Optional. + * Proxy configuration for http connection. + */ + struct aws_http_proxy_config *proxy_config; + + /** + * Optional. + * Configuration for fetching proxy configuration from environment. + * By Default proxy_ev_settings.aws_http_proxy_env_var_type is set to AWS_HPEV_ENABLE which means read proxy + * configuration from environment. + * Only works when proxy_config is not set. If both are set, configuration from proxy_config is used. + */ + struct proxy_env_var_settings *proxy_ev_settings; + + /** + * Optional. + * If set to 0, default value is used. + */ + uint32_t connect_timeout_ms; + + /** + * Optional. + * Set keepalive to periodically transmit messages for detecting a disconnected peer. + */ + struct aws_s3_tcp_keep_alive_options *tcp_keep_alive_options; + + /** + * Optional. + * Configuration options for connection monitoring. + * If the transfer speed falls below the specified minimum_throughput_bytes_per_second, the operation is aborted. + */ + struct aws_http_connection_monitoring_options *monitoring_options; }; struct aws_s3_endpoint { @@ -179,6 +215,43 @@ struct aws_s3_client { /* Retry strategy used for scheduling request retries. */ struct aws_retry_strategy *retry_strategy; + /** + * Optional. + * Proxy configuration for http connection. + */ + struct aws_http_proxy_config *proxy_config; + + /** + * Optional. + * Configuration for fetching proxy configuration from environment. + * By Default proxy_ev_settings.aws_http_proxy_env_var_type is set to AWS_HPEV_ENABLE which means read proxy + * configuration from environment. + * Only works when proxy_config is not set. If both are set, configuration from proxy_config is used. + */ + struct proxy_env_var_settings *proxy_ev_settings; + + /** + * Optional. + * If set to 0, default value is used. + */ + uint32_t connect_timeout_ms; + + /** + * Optional. + * Set keepalive to periodically transmit messages for detecting a disconnected peer. + */ + struct aws_s3_tcp_keep_alive_options *tcp_keep_alive_options; + + /** + * Optional. + * Configuration options for connection monitoring. + * If the transfer speed falls below the specified minimum_throughput_bytes_per_second, the operation is aborted. + */ + struct aws_http_connection_monitoring_options *monitoring_options; + + /* tls options from proxy environment settings. */ + struct aws_tls_connection_options *proxy_ev_tls_options; + /* Shutdown callbacks to notify when the client is completely cleaned up. */ aws_s3_client_shutdown_complete_callback_fn *shutdown_callback; void *shutdown_callback_user_data; diff --git a/include/aws/s3/private/s3_meta_request_impl.h b/include/aws/s3/private/s3_meta_request_impl.h index d7940cdd9..e4031fe2f 100644 --- a/include/aws/s3/private/s3_meta_request_impl.h +++ b/include/aws/s3/private/s3_meta_request_impl.h @@ -51,7 +51,7 @@ struct aws_s3_prepare_request_payload { }; struct aws_s3_meta_request_vtable { - /* Update the meta request. out_request is required to be non-null. Returns true if there is there is any work in + /* Update the meta request. out_request is required to be non-null. Returns true if there is any work in * progress, false if there is not. */ bool (*update)(struct aws_s3_meta_request *meta_request, uint32_t flags, struct aws_s3_request **out_request); diff --git a/include/aws/s3/s3_client.h b/include/aws/s3/s3_client.h index e10854bdc..7253152c9 100644 --- a/include/aws/s3/s3_client.h +++ b/include/aws/s3/s3_client.h @@ -6,11 +6,10 @@ * SPDX-License-Identifier: Apache-2.0. */ +#include #include #include -#include - struct aws_allocator; struct aws_http_stream; @@ -150,6 +149,19 @@ enum aws_s3_checksum_algorithm { AWS_SCA_COUNT, }; +/* Keepalive properties are TCP only. + * If interval or timeout are zero, then default values are used. + */ +struct aws_s3_tcp_keep_alive_options { + + uint16_t keep_alive_interval_sec; + uint16_t keep_alive_timeout_sec; + + /* If set, sets the number of keep alive probes allowed to fail before the connection is considered + * lost. If zero OS defaults are used. On Windows, this option is meaningless until Windows 10 1703.*/ + uint16_t keep_alive_max_failed_probes; +}; + /* Options for a new client. */ struct aws_s3_client_config { @@ -200,6 +212,40 @@ struct aws_s3_client_config { /* Callback and associated user data for when the client has completed its shutdown process. */ aws_s3_client_shutdown_complete_callback_fn *shutdown_callback; void *shutdown_callback_user_data; + + /** + * Optional. + * Proxy configuration for http connection. + */ + struct aws_http_proxy_options *proxy_options; + + /** + * Optional. + * Configuration for fetching proxy configuration from environment. + * By Default proxy_ev_settings.aws_http_proxy_env_var_type is set to AWS_HPEV_ENABLE which means read proxy + * configuration from environment. + * Only works when proxy_options is not set. If both are set, configuration from proxy_options is used. + */ + struct proxy_env_var_settings *proxy_ev_settings; + + /** + * Optional. + * If set to 0, default value is used. + */ + uint32_t connect_timeout_ms; + + /** + * Optional. + * Set keepalive to periodically transmit messages for detecting a disconnected peer. + */ + struct aws_s3_tcp_keep_alive_options *tcp_keep_alive_options; + + /** + * Optional. + * Configuration options for connection monitoring. + * If the transfer speed falls below the specified minimum_throughput_bytes_per_second, the operation is aborted. + */ + struct aws_http_connection_monitoring_options *monitoring_options; }; /* Options for a new meta request, ie, file transfer that will be handled by the high performance client. */ diff --git a/source/s3_auto_ranged_get.c b/source/s3_auto_ranged_get.c index 338135333..b4782f635 100644 --- a/source/s3_auto_ranged_get.c +++ b/source/s3_auto_ranged_get.c @@ -135,7 +135,7 @@ static bool s_s3_auto_ranged_get_update( { aws_s3_meta_request_lock_synced_data(meta_request); - /* If nothing has set the the "finish result" then this meta request is still in progress and we can potentially + /* If nothing has set the "finish result" then this meta request is still in progress, and we can potentially * send additional requests. */ if (!aws_s3_meta_request_has_finish_result_synced(meta_request)) { diff --git a/source/s3_client.c b/source/s3_client.c index ee912ec12..e1cd20a37 100644 --- a/source/s3_client.c +++ b/source/s3_client.c @@ -304,6 +304,37 @@ struct aws_s3_client *aws_s3_client_new( *((size_t *)&client_config->max_part_size) = client_config->part_size; } + if (client_config->proxy_options) { + client->proxy_config = aws_http_proxy_config_new_from_proxy_options(allocator, client_config->proxy_options); + if (client->proxy_config == NULL) { + goto on_error; + } + } + client->connect_timeout_ms = client_config->connect_timeout_ms; + if (client_config->proxy_ev_settings) { + client->proxy_ev_settings = aws_mem_calloc(allocator, 1, sizeof(struct proxy_env_var_settings)); + *client->proxy_ev_settings = *client_config->proxy_ev_settings; + + if (client_config->proxy_ev_settings->tls_options) { + client->proxy_ev_tls_options = aws_mem_calloc(allocator, 1, sizeof(struct aws_tls_connection_options)); + if (aws_tls_connection_options_copy(client->proxy_ev_tls_options, client->proxy_ev_settings->tls_options)) { + goto on_error; + } + client->proxy_ev_settings->tls_options = client->proxy_ev_tls_options; + } + } + + if (client_config->tcp_keep_alive_options) { + client->tcp_keep_alive_options = aws_mem_calloc(allocator, 1, sizeof(struct aws_s3_tcp_keep_alive_options)); + *client->tcp_keep_alive_options = *client_config->tcp_keep_alive_options; + } + + if (client_config->monitoring_options) { + client->monitoring_options = + aws_mem_calloc(allocator, 1, sizeof(struct aws_http_connection_monitoring_options)); + *client->monitoring_options = *client_config->monitoring_options; + } + if (client_config->tls_mode == AWS_MR_TLS_ENABLED) { client->tls_connection_options = aws_mem_calloc(client->allocator, 1, sizeof(struct aws_tls_connection_options)); @@ -421,6 +452,18 @@ struct aws_s3_client *aws_s3_client_new( aws_mem_release(client->allocator, client->tls_connection_options); client->tls_connection_options = NULL; } + if (client->proxy_config) { + aws_http_proxy_config_destroy(client->proxy_config); + } + if (client->proxy_ev_tls_options) { + aws_tls_connection_options_clean_up(client->proxy_ev_tls_options); + aws_mem_release(client->allocator, client->proxy_ev_tls_options); + client->proxy_ev_settings->tls_options = NULL; + } + aws_mem_release(client->allocator, client->proxy_ev_settings); + aws_mem_release(client->allocator, client->monitoring_options); + aws_mem_release(client->allocator, client->tcp_keep_alive_options); + aws_event_loop_group_release(client->client_bootstrap->event_loop_group); aws_client_bootstrap_release(client->client_bootstrap); aws_mutex_clean_up(&client->synced_data.lock); @@ -495,6 +538,19 @@ static void s_s3_client_finish_destroy_default(struct aws_s3_client *client) { client->tls_connection_options = NULL; } + if (client->proxy_config) { + aws_http_proxy_config_destroy(client->proxy_config); + } + + if (client->proxy_ev_tls_options) { + aws_tls_connection_options_clean_up(client->proxy_ev_tls_options); + aws_mem_release(client->allocator, client->proxy_ev_tls_options); + client->proxy_ev_settings->tls_options = NULL; + } + aws_mem_release(client->allocator, client->proxy_ev_settings); + aws_mem_release(client->allocator, client->monitoring_options); + aws_mem_release(client->allocator, client->tcp_keep_alive_options); + aws_mutex_clean_up(&client->synced_data.lock); AWS_ASSERT(aws_linked_list_empty(&client->synced_data.pending_meta_request_work)); @@ -667,7 +723,6 @@ struct aws_s3_meta_request *aws_s3_client_make_meta_request( struct aws_hash_element *endpoint_hash_element = NULL; int was_created = 0; - if (aws_hash_table_create( &client->synced_data.endpoints, endpoint_host_name, &endpoint_hash_element, &was_created)) { error_occurred = true; @@ -684,7 +739,11 @@ struct aws_s3_meta_request *aws_s3_client_make_meta_request( .user_data = client, .max_connections = aws_s3_client_get_max_active_connections(client, NULL), .port = port, - }; + .proxy_config = client->proxy_config, + .proxy_ev_settings = client->proxy_ev_settings, + .connect_timeout_ms = client->connect_timeout_ms, + .tcp_keep_alive_options = client->tcp_keep_alive_options, + .monitoring_options = client->monitoring_options}; endpoint = aws_s3_endpoint_new(client->allocator, &endpoint_options); diff --git a/source/s3_endpoint.c b/source/s3_endpoint.c index c07c1a58c..8eb4a3f30 100644 --- a/source/s3_endpoint.c +++ b/source/s3_endpoint.c @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include @@ -51,7 +50,12 @@ static struct aws_http_connection_manager *s_s3_endpoint_create_http_connection_ struct aws_client_bootstrap *client_bootstrap, const struct aws_tls_connection_options *tls_connection_options, uint32_t max_connections, - uint16_t port); + uint16_t port, + const struct aws_http_proxy_config *proxy_config, + const struct proxy_env_var_settings *proxy_ev_settings, + uint32_t connect_timeout_ms, + const struct aws_s3_tcp_keep_alive_options *tcp_keep_alive_options, + const struct aws_http_connection_monitoring_options *monitoring_options); static void s_s3_endpoint_http_connection_manager_shutdown_callback(void *user_data); @@ -98,7 +102,12 @@ struct aws_s3_endpoint *aws_s3_endpoint_new( options->client_bootstrap, options->tls_connection_options, options->max_connections, - options->port); + options->port, + options->proxy_config, + options->proxy_ev_settings, + options->connect_timeout_ms, + options->tcp_keep_alive_options, + options->monitoring_options); if (endpoint->http_connection_manager == NULL) { goto error_cleanup; @@ -124,7 +133,13 @@ static struct aws_http_connection_manager *s_s3_endpoint_create_http_connection_ struct aws_client_bootstrap *client_bootstrap, const struct aws_tls_connection_options *tls_connection_options, uint32_t max_connections, - uint16_t port) { + uint16_t port, + const struct aws_http_proxy_config *proxy_config, + const struct proxy_env_var_settings *proxy_ev_settings, + uint32_t connect_timeout_ms, + const struct aws_s3_tcp_keep_alive_options *tcp_keep_alive_options, + const struct aws_http_connection_monitoring_options *monitoring_options) { + AWS_PRECONDITION(endpoint); AWS_PRECONDITION(client_bootstrap); AWS_PRECONDITION(host_name); @@ -136,11 +151,20 @@ static struct aws_http_connection_manager *s_s3_endpoint_create_http_connection_ AWS_ZERO_STRUCT(socket_options); socket_options.type = AWS_SOCKET_STREAM; socket_options.domain = AWS_SOCKET_IPV4; - socket_options.connect_timeout_ms = s_connection_timeout_ms; - struct proxy_env_var_settings proxy_ev_settings; - AWS_ZERO_STRUCT(proxy_ev_settings); + socket_options.connect_timeout_ms = connect_timeout_ms == 0 ? s_connection_timeout_ms : connect_timeout_ms; + if (tcp_keep_alive_options != NULL) { + socket_options.keepalive = true; + socket_options.keep_alive_interval_sec = tcp_keep_alive_options->keep_alive_interval_sec; + socket_options.keep_alive_timeout_sec = tcp_keep_alive_options->keep_alive_timeout_sec; + socket_options.keep_alive_max_failed_probes = tcp_keep_alive_options->keep_alive_max_failed_probes; + } + struct proxy_env_var_settings proxy_ev_settings_default; /* Turn on envrionment variable for proxy by default */ - proxy_ev_settings.env_var_type = AWS_HPEV_ENABLE; + if (proxy_ev_settings == NULL) { + AWS_ZERO_STRUCT(proxy_ev_settings_default); + proxy_ev_settings_default.env_var_type = AWS_HPEV_ENABLE; + proxy_ev_settings = &proxy_ev_settings_default; + } struct aws_http_connection_manager_options manager_options; AWS_ZERO_STRUCT(manager_options); @@ -151,7 +175,16 @@ static struct aws_http_connection_manager *s_s3_endpoint_create_http_connection_ manager_options.max_connections = max_connections; manager_options.shutdown_complete_callback = s_s3_endpoint_http_connection_manager_shutdown_callback; manager_options.shutdown_complete_user_data = endpoint; - manager_options.proxy_ev_settings = &proxy_ev_settings; + manager_options.proxy_ev_settings = proxy_ev_settings; + if (monitoring_options != NULL) { + manager_options.monitoring_options = monitoring_options; + } + + struct aws_http_proxy_options proxy_options; + if (proxy_config != NULL) { + aws_http_proxy_options_init_from_config(&proxy_options, proxy_config); + manager_options.proxy_options = &proxy_options; + } struct aws_tls_connection_options *manager_tls_options = NULL; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3d5bf04b4..5aafe491e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -3,12 +3,12 @@ enable_testing() option(BYO_CRYPTO "Don't build a tls implementation or link against a crypto interface. This feature is only for unix builds currently." OFF) -if(BYO_CRYPTO) +if (BYO_CRYPTO) set(ENABLE_NET_TESTS OFF) add_test_case(test_s3_client_byo_crypto_no_options) add_test_case(test_s3_client_byo_crypto_with_options) -endif() +endif () file(GLOB TEST_SRC "*.c") file(GLOB TEST_HDRS "*.h") @@ -24,6 +24,9 @@ add_test_case(test_s3_complete_multipart_message_new) add_test_case(test_s3_abort_multipart_upload_message_new) add_net_test_case(test_s3_client_create_destroy) +add_net_test_case(test_s3_client_monitoring_options_override) +add_net_test_case(test_s3_client_proxy_ev_settings_override) +add_net_test_case(test_s3_client_tcp_keep_alive_options_override) add_net_test_case(test_s3_client_max_active_connections_override) add_test_case(test_s3_client_get_max_active_connections) add_test_case(test_s3_request_create_destroy) diff --git a/tests/s3_data_plane_tests.c b/tests/s3_data_plane_tests.c index d0cf3131b..32cdb099b 100644 --- a/tests/s3_data_plane_tests.c +++ b/tests/s3_data_plane_tests.c @@ -51,6 +51,85 @@ static int s_test_s3_client_create_destroy(struct aws_allocator *allocator, void return 0; } +AWS_TEST_CASE(test_s3_client_monitoring_options_override, s_test_s3_client_monitoring_options_override) +static int s_test_s3_client_monitoring_options_override(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + + struct aws_s3_tester tester; + AWS_ZERO_STRUCT(tester); + ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester)); + + struct aws_http_connection_monitoring_options monitoring_options = {.minimum_throughput_bytes_per_second = 3000}; + + struct aws_s3_client_config client_config = {.monitoring_options = &monitoring_options}; + + ASSERT_SUCCESS(aws_s3_tester_bind_client(&tester, &client_config, 0)); + + struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config); + + ASSERT_TRUE( + client->monitoring_options->minimum_throughput_bytes_per_second == + client_config.monitoring_options->minimum_throughput_bytes_per_second); + + aws_s3_client_release(client); + aws_s3_tester_clean_up(&tester); + + return 0; +} + +AWS_TEST_CASE(test_s3_client_proxy_ev_settings_override, s_test_s3_client_proxy_ev_settings_override) +static int s_test_s3_client_proxy_ev_settings_override(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + + struct aws_s3_tester tester; + AWS_ZERO_STRUCT(tester); + ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester)); + struct aws_tls_connection_options tls_conn_options; + AWS_ZERO_STRUCT(tls_conn_options); + + struct proxy_env_var_settings proxy_ev_settings = {.env_var_type = AWS_HPEV_ENABLE, + .tls_options = &tls_conn_options}; + + struct aws_s3_client_config client_config = {.proxy_ev_settings = &proxy_ev_settings}; + + ASSERT_SUCCESS(aws_s3_tester_bind_client(&tester, &client_config, 0)); + + struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config); + + ASSERT_TRUE(client->proxy_ev_settings->env_var_type == client_config.proxy_ev_settings->env_var_type); + + aws_s3_client_release(client); + aws_s3_tester_clean_up(&tester); + + return 0; +} + +AWS_TEST_CASE(test_s3_client_tcp_keep_alive_options_override, s_test_s3_client_tcp_keep_alive_options_override) +static int s_test_s3_client_tcp_keep_alive_options_override(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + + struct aws_s3_tester tester; + AWS_ZERO_STRUCT(tester); + ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester)); + + struct aws_s3_tcp_keep_alive_options keep_alive_options = {.keep_alive_interval_sec = 20}; + + struct aws_s3_client_config client_config = {.tcp_keep_alive_options = &keep_alive_options}; + + ASSERT_SUCCESS(aws_s3_tester_bind_client(&tester, &client_config, 0)); + + struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config); + + ASSERT_TRUE( + client->tcp_keep_alive_options->keep_alive_interval_sec == + client_config.tcp_keep_alive_options->keep_alive_interval_sec); + + aws_s3_client_release(client); + aws_s3_tester_clean_up(&tester); + + return 0; +} + AWS_TEST_CASE(test_s3_client_max_active_connections_override, s_test_s3_client_max_active_connections_override) static int s_test_s3_client_max_active_connections_override(struct aws_allocator *allocator, void *ctx) { (void)ctx;