Fix Host header not including port (#249)

* update endpoint/host setting logic in meta request creation

include/aws/s3/s3_client.h

@@ -394,9 +394,16 @@ struct aws_s3_meta_request_options {
 
     /**
      * Optional.
-     * The endpoint for the meta request to connect to. If not set, then tls settings from client will
-     * determine the port, and host header from `message` will determine the host for the connection.
-     * Note: must match the host header from `message`
+     * Endpoint override for request. Can be used to override scheme and port of
+     * the endpoint.
+     * There is some overlap between Host header and Endpoint and corner cases
+     * are handled as follows:
+     * - Only Host header is set - Host is used to construct endpoint. https is
+     *   default with corresponding port
+     * - Only endpoint is set - Host header is created from endpoint. Port and
+     *   Scheme from endpoint is used.
+     * - Both Host and Endpoint is set - Host header must match Authority of
+     *   Endpoint uri. Port and Scheme from endpoint is used.
      */
     struct aws_uri *endpoint;
 

source/s3_client.c

@@ -636,6 +636,60 @@ struct aws_s3_request *aws_s3_client_dequeue_request_threaded(struct aws_s3_clie
     return request;
 }
 
+/*
+ * There is currently some overlap between user provided Host header and endpoint
+ * override. This function handles the corner cases for when either or both are provided.
+ */
+int s_apply_endpoint_override(
+    const struct aws_s3_client *client,
+    struct aws_http_headers *message_headers,
+    const struct aws_uri *endpoint) {
+    AWS_PRECONDITION(message_headers);
+
+    const struct aws_byte_cursor *endpoint_authority = endpoint == NULL ? NULL : aws_uri_authority(endpoint);
+
+    if (!aws_http_headers_has(message_headers, g_host_header_name)) {
+        if (endpoint_authority == NULL) {
+            AWS_LOGF_ERROR(
+                AWS_LS_S3_CLIENT,
+                "id=%p Cannot create meta s3 request; message provided in options does not have either 'Host' header "
+                "set or endpoint override.",
+                (void *)client);
+            return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
+        }
+
+        if (aws_http_headers_set(message_headers, g_host_header_name, *endpoint_authority)) {
+            AWS_LOGF_ERROR(
+                AWS_LS_S3_CLIENT,
+                "id=%p Cannot create meta s3 request; failed to set 'Host' header based on endpoint override.",
+                (void *)client);
+            return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
+        }
+    }
+
+    struct aws_byte_cursor host_value;
+    if (aws_http_headers_get(message_headers, g_host_header_name, &host_value)) {
+        AWS_LOGF_ERROR(
+            AWS_LS_S3_CLIENT,
+            "id=%p Cannot create meta s3 request; message provided in options does not have a 'Host' header.",
+            (void *)client);
+        return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
+    }
+
+    if (endpoint_authority != NULL && !aws_byte_cursor_eq(&host_value, endpoint_authority)) {
+        AWS_LOGF_ERROR(
+            AWS_LS_S3_CLIENT,
+            "id=%p Cannot create meta s3 request; host header value " PRInSTR
+            " does not match endpoint override " PRInSTR,
+            (void *)client,
+            AWS_BYTE_CURSOR_PRI(host_value),
+            AWS_BYTE_CURSOR_PRI(*endpoint_authority));
+        return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
+    }
+
+    return AWS_OP_SUCCESS;
+}
+
 /* Public facing make-meta-request function. */
 struct aws_s3_meta_request *aws_s3_client_make_meta_request(
     struct aws_s3_client *client,
@@ -724,8 +778,11 @@ struct aws_s3_meta_request *aws_s3_client_make_meta_request(
         }
     }
 
-    struct aws_byte_cursor host_header_value;
+    if (s_apply_endpoint_override(client, message_headers, options->endpoint)) {
+        return NULL;
+    }
 
+    struct aws_byte_cursor host_header_value;
     if (aws_http_headers_get(message_headers, g_host_header_name, &host_header_value)) {
         AWS_LOGF_ERROR(
             AWS_LS_S3_CLIENT,
@@ -739,19 +796,23 @@ struct aws_s3_meta_request *aws_s3_client_make_meta_request(
     uint16_t port = 0;
 
     if (options->endpoint != NULL) {
-        const struct aws_byte_cursor *host_name_cursor = aws_uri_host_name(options->endpoint);
-        if (host_name_cursor->len) {
-            if (!aws_byte_cursor_eq(host_name_cursor, &host_header_value)) {
-                AWS_LOGF_ERROR(
-                    AWS_LS_S3_CLIENT,
-                    "id=%p Cannot create meta s3 request; 'Host' header does not match URI 'hostname'.",
-                    (void *)client);
-                aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
-                return NULL;
-            }
-        }
         struct aws_byte_cursor https_scheme = aws_byte_cursor_from_c_str("https");
-        is_https = aws_byte_cursor_eq_ignore_case(aws_uri_scheme(options->endpoint), &https_scheme);
+        struct aws_byte_cursor http_scheme = aws_byte_cursor_from_c_str("http");
+
+        const struct aws_byte_cursor *scheme = aws_uri_scheme(options->endpoint);
+
+        is_https = aws_byte_cursor_eq_ignore_case(scheme, &https_scheme);
+
+        if (!is_https && !aws_byte_cursor_eq_ignore_case(scheme, &http_scheme)) {
+            AWS_LOGF_ERROR(
+                AWS_LS_S3_CLIENT,
+                "id=%p Cannot create meta s3 request; unexpected scheme '" PRInSTR "' in endpoint override.",
+                (void *)client,
+                AWS_BYTE_CURSOR_PRI(*scheme));
+            aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
+            return NULL;
+        }
+
         port = aws_uri_port(options->endpoint);
     }
 
@@ -768,14 +829,28 @@ struct aws_s3_meta_request *aws_s3_client_make_meta_request(
     {
         aws_s3_client_lock_synced_data(client);
 
-        struct aws_string *endpoint_host_name = aws_string_new_from_cursor(client->allocator, &host_header_value);
+        struct aws_string *endpoint_host_name = NULL;
+
+        if (options->endpoint != NULL) {
+            endpoint_host_name = aws_string_new_from_cursor(client->allocator, aws_uri_host_name(options->endpoint));
+        } else {
+            struct aws_uri host_uri;
+            if (aws_uri_init_parse(&host_uri, client->allocator, &host_header_value)) {
+                error_occurred = true;
+                goto unlock;
+            }
+
+            endpoint_host_name = aws_string_new_from_cursor(client->allocator, aws_uri_host_name(&host_uri));
+            aws_uri_clean_up(&host_uri);
+        }
 
         struct aws_s3_endpoint *endpoint = NULL;
         struct aws_hash_element *endpoint_hash_element = NULL;
 
         int was_created = 0;
         if (aws_hash_table_create(
                 &client->synced_data.endpoints, endpoint_host_name, &endpoint_hash_element, &was_created)) {
+            aws_string_destroy(endpoint_host_name);
             error_occurred = true;
             goto unlock;
         }
@@ -799,6 +874,7 @@ struct aws_s3_meta_request *aws_s3_client_make_meta_request(
 
             if (endpoint == NULL) {
                 aws_hash_table_remove(&client->synced_data.endpoints, endpoint_host_name, NULL, NULL);
+                aws_string_destroy(endpoint_host_name);
                 error_occurred = true;
                 goto unlock;
             }

tests/s3_mock_server_tests.c

@@ -158,7 +158,7 @@ TEST_CASE(get_object_modified_mock_server) {
     struct aws_s3_client *client = NULL;
     ASSERT_SUCCESS(aws_s3_tester_client_new(&tester, &client_options, &client));
 
-    /* Check the mock server READEME/GetObject Response for the response that will be received. */
+    /* Check the mock server README/GetObject Response for the response that will be received. */
     struct aws_byte_cursor object_path = aws_byte_cursor_from_c_str("/get_object_modified");
 
     struct aws_s3_tester_meta_request_options get_options = {

tests/s3_tester.c

@@ -1233,7 +1233,7 @@ int aws_s3_tester_send_meta_request_with_options(
 
         struct aws_string *host_name = NULL;
         if (options->mock_server) {
-            const struct aws_byte_cursor *host_cursor = aws_uri_host_name(&mock_server);
+            const struct aws_byte_cursor *host_cursor = aws_uri_authority(&mock_server);
             host_name = aws_string_new_from_cursor(allocator, host_cursor);
         } else if (options->mrap_test) {
             host_name = aws_string_new_from_cursor(allocator, &g_test_mrap_endpoint);