From 099294cc7e2ff2434260a70b5f0417b9a38ccfb0 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 9 Aug 2023 20:09:04 +0600 Subject: [PATCH 01/33] draft support for 100-continue through stdlib http.client --- b2sdk/b2http.py | 2 +- b2sdk/requests/_continue.py | 253 +++++++++++++++++++++++++++++++ test/integration/test_raw_api.py | 62 +++++++- 3 files changed, 312 insertions(+), 5 deletions(-) create mode 100644 b2sdk/requests/_continue.py diff --git a/b2sdk/b2http.py b/b2sdk/b2http.py index e497c7195..e02f0195f 100644 --- a/b2sdk/b2http.py +++ b/b2sdk/b2http.py @@ -22,8 +22,8 @@ from typing import Any import requests -from requests.adapters import HTTPAdapter +from .requests._continue import HTTPAdapter from .api_config import DEFAULT_HTTP_API_CONFIG, B2HttpApiConfig from .exception import ( B2ConnectionError, diff --git a/b2sdk/requests/_continue.py b/b2sdk/requests/_continue.py new file mode 100644 index 000000000..926f3c2a9 --- /dev/null +++ b/b2sdk/requests/_continue.py @@ -0,0 +1,253 @@ +# Code taken and modified from: https://github.com/boto/botocore/blob/1.31.20/botocore/awsrequest.py + +# Copyright (c) 2012-2013 Mitch Garnaat http://garnaat.org/ +# Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import functools +import logging +from http.client import HTTPResponse + +import requests +import urllib3 +from requests import adapters +from urllib3.connection import HTTPConnection, VerifiedHTTPSConnection +from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool + + +logger = logging.getLogger(__name__) + + +class AWSHTTPResponse(HTTPResponse): + # The *args, **kwargs is used because the args are slightly + # different in py2.6 than in py2.7/py3. + def __init__(self, *args, **kwargs): + self._status_tuple = kwargs.pop('status_tuple') + HTTPResponse.__init__(self, *args, **kwargs) + + def _read_status(self): + if self._status_tuple is not None: + status_tuple = self._status_tuple + self._status_tuple = None + return status_tuple + else: + return HTTPResponse._read_status(self) + + +class AWSConnection: + """Mixin for HTTPConnection that supports Expect 100-continue. + + This when mixed with a subclass of httplib.HTTPConnection (though + technically we subclass from urllib3, which subclasses + httplib.HTTPConnection) and we only override this class to support Expect + 100-continue, which we need for S3. As far as I can tell, this is + general purpose enough to not be specific to S3, but I'm being + tentative and keeping it in botocore because I've only tested + this against AWS services. + + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._original_response_cls = self.response_class + # This variable is set when we receive an early response from the + # server. If this value is set to True, any calls to send() are noops. + # This value is reset to false every time _send_request is called. + # This is to workaround changes in urllib3 2.0 which uses separate + # send() calls in request() instead of delegating to endheaders(), + # which is where the body is sent in CPython's HTTPConnection. + self._response_received = False + self._expect_header_set = False + self._send_called = False + + def close(self): + super().close() + # Reset all of our instance state we were tracking. + self._response_received = False + self._expect_header_set = False + self._send_called = False + self.response_class = self._original_response_cls + + def request(self, method, url, body=None, headers=None, *args, **kwargs): + if headers is None: + headers = {} + self._response_received = False + if headers.get('Expect', b'') == b'100-continue': + self._expect_header_set = True + else: + self._expect_header_set = False + self.response_class = self._original_response_cls + rval = super().request(method, url, body, headers, *args, **kwargs) + self._expect_header_set = False + return rval + + def _convert_to_bytes(self, mixed_buffer): + # Take a list of mixed str/bytes and convert it + # all into a single bytestring. + # Any str will be encoded as utf-8. + bytes_buffer = [] + for chunk in mixed_buffer: + if isinstance(chunk, str): + bytes_buffer.append(chunk.encode('utf-8')) + else: + bytes_buffer.append(chunk) + msg = b"\r\n".join(bytes_buffer) + return msg + + def _send_output(self, message_body=None, *args, **kwargs): + self._buffer.extend((b"", b"")) + msg = self._convert_to_bytes(self._buffer) + del self._buffer[:] + # If msg and message_body are sent in a single send() call, + # it will avoid performance problems caused by the interaction + # between delayed ack and the Nagle algorithm. + if isinstance(message_body, bytes): + msg += message_body + message_body = None + self.send(msg) + if self._expect_header_set: + # This is our custom behavior. If the Expect header was + # set, it will trigger this custom behavior. + logger.debug("Waiting for 100 Continue response.") + # Wait for 1 second for the server to send a response. + if urllib3.util.wait_for_read(self.sock, 1): + self._handle_expect_response(message_body) + return + else: + # From the RFC: + # Because of the presence of older implementations, the + # protocol allows ambiguous situations in which a client may + # send "Expect: 100-continue" without receiving either a 417 + # (Expectation Failed) status or a 100 (Continue) status. + # Therefore, when a client sends this header field to an origin + # server (possibly via a proxy) from which it has never seen a + # 100 (Continue) status, the client SHOULD NOT wait for an + # indefinite period before sending the request body. + logger.debug( + "No response seen from server, continuing to " + "send the response body." + ) + if message_body is not None: + # message_body was not a string (i.e. it is a file), and + # we must run the risk of Nagle. + self.send(message_body) + + def _consume_headers(self, fp): + # Most servers (including S3) will just return + # the CLRF after the 100 continue response. However, + # some servers (I've specifically seen this for squid when + # used as a straight HTTP proxy) will also inject a + # Connection: keep-alive header. To account for this + # we'll read until we read '\r\n', and ignore any headers + # that come immediately after the 100 continue response. + current = None + while current != b'\r\n': + current = fp.readline() + + def _handle_expect_response(self, message_body): + # This is called when we sent the request headers containing + # an Expect: 100-continue header and received a response. + # We now need to figure out what to do. + fp = self.sock.makefile('rb', 0) + try: + maybe_status_line = fp.readline() + parts = maybe_status_line.split(None, 2) + if self._is_100_continue_status(maybe_status_line): + self._consume_headers(fp) + logger.debug( + "100 Continue response seen, now sending request body." + ) + self._send_message_body(message_body) + elif len(parts) == 3 and parts[0].startswith(b'HTTP/'): + # From the RFC: + # Requirements for HTTP/1.1 origin servers: + # + # - Upon receiving a request which includes an Expect + # request-header field with the "100-continue" + # expectation, an origin server MUST either respond with + # 100 (Continue) status and continue to read from the + # input stream, or respond with a final status code. + # + # So if we don't get a 100 Continue response, then + # whatever the server has sent back is the final response + # and don't send the message_body. + logger.debug( + "Received a non 100 Continue response " + "from the server, NOT sending request body." + ) + status_tuple = ( + parts[0].decode('ascii'), + int(parts[1]), + parts[2].decode('ascii'), + ) + response_class = functools.partial( + AWSHTTPResponse, status_tuple=status_tuple + ) + self.response_class = response_class + self._response_received = True + finally: + fp.close() + + def _send_message_body(self, message_body): + if message_body is not None: + self.send(message_body) + + def send(self, str): + if self._response_received: + if not self._send_called: + # urllib3 2.0 chunks and calls send potentially + # thousands of times inside `request` unlike the + # standard library. Only log this once for sanity. + logger.debug( + "send() called, but response already received. " + "Not sending data." + ) + self._send_called = True + return + return super().send(str) + + def _is_100_continue_status(self, maybe_status_line): + parts = maybe_status_line.split(None, 2) + # Check for HTTP/ 100 Continue\r\n + return ( + len(parts) >= 3 + and parts[0].startswith(b'HTTP/') + and parts[1] == b'100' + ) + + +class AWSHTTPConnection(AWSConnection, HTTPConnection): + """An HTTPConnection that supports 100 Continue behavior.""" + + +class AWSHTTPSConnection(AWSConnection, VerifiedHTTPSConnection): + """An HTTPSConnection that supports 100 Continue behavior.""" + + +class AWSHTTPConnectionPool(HTTPConnectionPool): + ConnectionCls = AWSHTTPConnection + + +class AWSHTTPSConnectionPool(HTTPSConnectionPool): + ConnectionCls = AWSHTTPSConnection + + +pool_classes_by_scheme = {"http": AWSHTTPConnectionPool, "https": AWSHTTPSConnectionPool} + + +class HTTPAdapter(adapters.HTTPAdapter): + def init_poolmanager( + self, connections, maxsize, block=adapters.DEFAULT_POOLBLOCK, **pool_kwargs + ): + super().init_poolmanager(connections, maxsize, block, **pool_kwargs) + self.poolmanager.pool_classes_by_scheme = pool_classes_by_scheme diff --git a/test/integration/test_raw_api.py b/test/integration/test_raw_api.py index 89d17ab18..4e2d31b21 100644 --- a/test/integration/test_raw_api.py +++ b/test/integration/test_raw_api.py @@ -21,7 +21,7 @@ from b2sdk.b2http import B2Http from b2sdk.encryption.setting import EncryptionAlgorithm, EncryptionMode, EncryptionSetting -from b2sdk.exception import DisablingFileLockNotSupported +from b2sdk.exception import DisablingFileLockNotSupported, InvalidAuthToken from b2sdk.file_lock import ( NO_RETENTION_FILE_SETTING, BucketRetentionSetting, @@ -35,7 +35,7 @@ # TODO: rewrite to separate test cases -def test_raw_api(dont_cleanup_old_buckets): +def test_raw_api(dont_cleanup_old_buckets, monkeypatch): """ Exercise the code in B2RawHTTPApi by making each call once, just to make sure the parameters are passed in, and the result is @@ -60,7 +60,7 @@ def test_raw_api(dont_cleanup_old_buckets): try: raw_api = B2RawHTTPApi(B2Http()) - raw_api_test_helper(raw_api, not dont_cleanup_old_buckets) + raw_api_test_helper(raw_api, not dont_cleanup_old_buckets, monkeypatch) except Exception: traceback.print_exc(file=sys.stdout) pytest.fail('test_raw_api failed') @@ -83,7 +83,7 @@ def authorize_raw_api(raw_api): return auth_dict -def raw_api_test_helper(raw_api, should_cleanup_old_buckets): +def raw_api_test_helper(raw_api, should_cleanup_old_buckets, monkeypatch): """ Try each of the calls to the raw api. Raise an exception if anything goes wrong. @@ -550,6 +550,60 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): is_file_lock_enabled=False, ) + # b2_100_continue + print('b2_100_continue') + file_name = 'test-100-continue.txt' + file_contents = b'hello world' + file_sha1 = hex_sha1_of_stream(io.BytesIO(file_contents), len(file_contents)) + + def set_100_header(fn): + def wrapper(url, headers, *args, **kwargs): + headers.update({ + 'Expect': '100-continue', + }) + return fn(url, headers, *args, **kwargs) + + return wrapper + + with monkeypatch.context() as monkey: + monkey.setattr( + raw_api.b2_http, + 'post_content_return_json', + set_100_header(raw_api.b2_http.post_content_return_json), + ) + data = io.BytesIO(file_contents) + + with pytest.raises(InvalidAuthToken): + raw_api.upload_file( + upload_url, + upload_auth_token + 'x', + file_name, + len(file_contents), + 'text/plain', + file_sha1, + {'color': 'blue'}, + data, + server_side_encryption=sse_b2_aes, + ) + + # NOTE: disabling the following check as urllib3 consumes body even if we don't send it + # Checking if any data was read. + # assert data.tell() == 0 + # TODO: add test to check the data is not sent with Expect: 100-continue header + + data.seek(0) + raw_api.upload_file( + upload_url, + upload_auth_token, + file_name, + len(file_contents), + 'text/plain', + file_sha1, + {'color': 'blue'}, + data, + server_side_encryption=sse_b2_aes, + ) + # Clean up this test. _clean_and_delete_bucket(raw_api, api_url, account_auth_token, account_id, bucket_id) From cee26655fca1b94b06a71eee8973db48896bf013 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sat, 12 Aug 2023 21:56:43 +0600 Subject: [PATCH 02/33] fix header type when checking for Expect: 100-continue --- b2sdk/requests/_continue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/b2sdk/requests/_continue.py b/b2sdk/requests/_continue.py index 926f3c2a9..7bd060900 100644 --- a/b2sdk/requests/_continue.py +++ b/b2sdk/requests/_continue.py @@ -82,7 +82,7 @@ def request(self, method, url, body=None, headers=None, *args, **kwargs): if headers is None: headers = {} self._response_received = False - if headers.get('Expect', b'') == b'100-continue': + if headers.get('Expect', b'') in [b'100-continue', '100-continue']: self._expect_header_set = True else: self._expect_header_set = False From b72b83890191f8f4415391b41b48c9638762fe27 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sat, 12 Aug 2023 21:58:27 +0600 Subject: [PATCH 03/33] Handle b2 server responding with empty text phrase in status line --- b2sdk/b2http.py | 4 ++-- b2sdk/requests/_continue.py | 30 +++++++++++------------------- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/b2sdk/b2http.py b/b2sdk/b2http.py index e02f0195f..54591c8e7 100644 --- a/b2sdk/b2http.py +++ b/b2sdk/b2http.py @@ -23,7 +23,7 @@ import requests -from .requests._continue import HTTPAdapter +from .requests._continue import HTTPAdapterWithContinue from .api_config import DEFAULT_HTTP_API_CONFIG, B2HttpApiConfig from .exception import ( B2ConnectionError, @@ -534,7 +534,7 @@ def _translate_and_retry(cls, fcn, try_count, post_params=None): return cls._translate_errors(fcn, post_params) -class NotDecompressingHTTPAdapter(HTTPAdapter): +class NotDecompressingHTTPAdapter(HTTPAdapterWithContinue): """ HTTP adapter that uses :class:`b2sdk.requests.NotDecompressingResponse` instead of the default :code:`requests.Response` class. diff --git a/b2sdk/requests/_continue.py b/b2sdk/requests/_continue.py index 7bd060900..ddaa8e98a 100644 --- a/b2sdk/requests/_continue.py +++ b/b2sdk/requests/_continue.py @@ -1,4 +1,6 @@ -# Code taken and modified from: https://github.com/boto/botocore/blob/1.31.20/botocore/awsrequest.py +# Code taken from: +# https://github.com/boto/botocore/blob/754b699bbf34261eae47c9dece3b11d7b58eb03c/botocore/awsrequest.py +# The code has been modified to work with urllib3>=2.0 # Copyright (c) 2012-2013 Mitch Garnaat http://garnaat.org/ # Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. @@ -13,18 +15,15 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. - import functools import logging from http.client import HTTPResponse -import requests import urllib3 from requests import adapters from urllib3.connection import HTTPConnection, VerifiedHTTPSConnection from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool - logger = logging.getLogger(__name__) @@ -106,7 +105,7 @@ def _convert_to_bytes(self, mixed_buffer): def _send_output(self, message_body=None, *args, **kwargs): self._buffer.extend((b"", b"")) - msg = self._convert_to_bytes(self._buffer) + msg = b"\r\n".join(self._buffer) del self._buffer[:] # If msg and message_body are sent in a single send() call, # it will avoid performance problems caused by the interaction @@ -120,7 +119,7 @@ def _send_output(self, message_body=None, *args, **kwargs): # set, it will trigger this custom behavior. logger.debug("Waiting for 100 Continue response.") # Wait for 1 second for the server to send a response. - if urllib3.util.wait_for_read(self.sock, 1): + if urllib3.util.wait_for_read(self.sock, 2): self._handle_expect_response(message_body) return else: @@ -162,13 +161,15 @@ def _handle_expect_response(self, message_body): try: maybe_status_line = fp.readline() parts = maybe_status_line.split(None, 2) - if self._is_100_continue_status(maybe_status_line): + + # Check for 'HTTP/ 100 Continue\r\n' or, 'HTTP/ 100\r\n' + if len(parts) >= 2 and parts[0].startswith(b'HTTP/') and parts[1] == b'100': self._consume_headers(fp) logger.debug( "100 Continue response seen, now sending request body." ) self._send_message_body(message_body) - elif len(parts) == 3 and parts[0].startswith(b'HTTP/'): + elif len(parts) >= 2 and parts[0].startswith(b'HTTP/'): # From the RFC: # Requirements for HTTP/1.1 origin servers: # @@ -188,7 +189,7 @@ def _handle_expect_response(self, message_body): status_tuple = ( parts[0].decode('ascii'), int(parts[1]), - parts[2].decode('ascii'), + parts[2].decode('ascii') if len(parts) > 2 else '', ) response_class = functools.partial( AWSHTTPResponse, status_tuple=status_tuple @@ -216,15 +217,6 @@ def send(self, str): return return super().send(str) - def _is_100_continue_status(self, maybe_status_line): - parts = maybe_status_line.split(None, 2) - # Check for HTTP/ 100 Continue\r\n - return ( - len(parts) >= 3 - and parts[0].startswith(b'HTTP/') - and parts[1] == b'100' - ) - class AWSHTTPConnection(AWSConnection, HTTPConnection): """An HTTPConnection that supports 100 Continue behavior.""" @@ -245,7 +237,7 @@ class AWSHTTPSConnectionPool(HTTPSConnectionPool): pool_classes_by_scheme = {"http": AWSHTTPConnectionPool, "https": AWSHTTPSConnectionPool} -class HTTPAdapter(adapters.HTTPAdapter): +class HTTPAdapterWithContinue(adapters.HTTPAdapter): def init_poolmanager( self, connections, maxsize, block=adapters.DEFAULT_POOLBLOCK, **pool_kwargs ): From f2c2bde0a37b2f22a2110a9cf3d2807fcfa1c477 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sat, 12 Aug 2023 22:01:50 +0600 Subject: [PATCH 04/33] Update NOTICE file to include urllib3 changes --- b2sdk/requests/NOTICE | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/b2sdk/requests/NOTICE b/b2sdk/requests/NOTICE index 12e0576bb..18ddc6b3f 100644 --- a/b2sdk/requests/NOTICE +++ b/b2sdk/requests/NOTICE @@ -4,4 +4,15 @@ Copyright 2019 Kenneth Reitz Copyright 2021 Backblaze Inc. Changes made to the original source: requests.models.Response.iter_content has been overridden to pass `decode_content=False` argument to `self.raw.stream` -in order to NOT decompress data based on Content-Encoding header \ No newline at end of file +in order to NOT decompress data based on Content-Encoding header +requests.adapters.HTTPAdapter has been overriden to use patched Urllib3 connection pools + + +Urllib3 +Copyright 2008-2011 Andrey Petrov and contributors. + +Copyright 2023 Backblaze Inc. +Changes made to the original source: +The following classes have been overriden for support HTTP 100-continue: +urllib3.connection.HTTPConnection, urllib3.connection.VerifiedHTTPSConnection, +urllib3.connectionpool.HTTPConnectionPool, urllib3.connectionpool.HTTPSConnectionPool From 08c4f1630675d2876466927087312edbfdfde532 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sat, 12 Aug 2023 22:02:13 +0600 Subject: [PATCH 05/33] Update test for 100-continue --- test/integration/test_raw_api.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/test/integration/test_raw_api.py b/test/integration/test_raw_api.py index 4e2d31b21..aa4fbc284 100644 --- a/test/integration/test_raw_api.py +++ b/test/integration/test_raw_api.py @@ -9,6 +9,7 @@ ###################################################################### from __future__ import annotations +import http.client import io import os import random @@ -565,12 +566,24 @@ def wrapper(url, headers, *args, **kwargs): return wrapper + orig_send = http.client.HTTPConnection.send + sent_data = bytearray() + + def patched_send(self, data): + sent_data.extend(data) + return orig_send(self, data) + with monkeypatch.context() as monkey: monkey.setattr( raw_api.b2_http, 'post_content_return_json', set_100_header(raw_api.b2_http.post_content_return_json), ) + monkey.setattr( + http.client.HTTPConnection, + "send", + patched_send, + ) data = io.BytesIO(file_contents) with pytest.raises(InvalidAuthToken): @@ -586,23 +599,8 @@ def wrapper(url, headers, *args, **kwargs): server_side_encryption=sse_b2_aes, ) - # NOTE: disabling the following check as urllib3 consumes body even if we don't send it - # Checking if any data was read. - # assert data.tell() == 0 - # TODO: add test to check the data is not sent with Expect: 100-continue header - - data.seek(0) - raw_api.upload_file( - upload_url, - upload_auth_token, - file_name, - len(file_contents), - 'text/plain', - file_sha1, - {'color': 'blue'}, - data, - server_side_encryption=sse_b2_aes, - ) + # Check if data was sent + assert file_contents not in sent_data # Clean up this test. _clean_and_delete_bucket(raw_api, api_url, account_auth_token, account_id, bucket_id) From 56a741e3de669339d5fda8c15601230717c5624f Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sat, 12 Aug 2023 22:22:36 +0600 Subject: [PATCH 06/33] Add test for 100-continue success case --- test/integration/test_raw_api.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/integration/test_raw_api.py b/test/integration/test_raw_api.py index aa4fbc284..3eb6984aa 100644 --- a/test/integration/test_raw_api.py +++ b/test/integration/test_raw_api.py @@ -602,6 +602,20 @@ def patched_send(self, data): # Check if data was sent assert file_contents not in sent_data + # this should not fail + data.seek(0) + raw_api.upload_file( + upload_url, + upload_auth_token, + file_name, + len(file_contents), + 'text/plain', + file_sha1, + {'color': 'blue'}, + data, + server_side_encryption=sse_b2_aes, + ) + # Clean up this test. _clean_and_delete_bucket(raw_api, api_url, account_auth_token, account_id, bucket_id) From 9b7f80c01f6a60d7e8d7e85d520fb59c3cdabba3 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sat, 12 Aug 2023 22:24:15 +0600 Subject: [PATCH 07/33] Format code --- b2sdk/b2http.py | 2 +- b2sdk/requests/_continue.py | 13 +++---------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/b2sdk/b2http.py b/b2sdk/b2http.py index 54591c8e7..4c159f98f 100644 --- a/b2sdk/b2http.py +++ b/b2sdk/b2http.py @@ -23,7 +23,6 @@ import requests -from .requests._continue import HTTPAdapterWithContinue from .api_config import DEFAULT_HTTP_API_CONFIG, B2HttpApiConfig from .exception import ( B2ConnectionError, @@ -41,6 +40,7 @@ interpret_b2_error, ) from .requests import NotDecompressingResponse +from .requests._continue import HTTPAdapterWithContinue from .version import USER_AGENT LOCALE_LOCK = threading.Lock() diff --git a/b2sdk/requests/_continue.py b/b2sdk/requests/_continue.py index ddaa8e98a..067ebb11e 100644 --- a/b2sdk/requests/_continue.py +++ b/b2sdk/requests/_continue.py @@ -165,9 +165,7 @@ def _handle_expect_response(self, message_body): # Check for 'HTTP/ 100 Continue\r\n' or, 'HTTP/ 100\r\n' if len(parts) >= 2 and parts[0].startswith(b'HTTP/') and parts[1] == b'100': self._consume_headers(fp) - logger.debug( - "100 Continue response seen, now sending request body." - ) + logger.debug("100 Continue response seen, now sending request body.") self._send_message_body(message_body) elif len(parts) >= 2 and parts[0].startswith(b'HTTP/'): # From the RFC: @@ -191,9 +189,7 @@ def _handle_expect_response(self, message_body): int(parts[1]), parts[2].decode('ascii') if len(parts) > 2 else '', ) - response_class = functools.partial( - AWSHTTPResponse, status_tuple=status_tuple - ) + response_class = functools.partial(AWSHTTPResponse, status_tuple=status_tuple) self.response_class = response_class self._response_received = True finally: @@ -209,10 +205,7 @@ def send(self, str): # urllib3 2.0 chunks and calls send potentially # thousands of times inside `request` unlike the # standard library. Only log this once for sanity. - logger.debug( - "send() called, but response already received. " - "Not sending data." - ) + logger.debug("send() called, but response already received. " "Not sending data.") self._send_called = True return return super().send(str) From 334ef687c76990b12b9f04a66d5537c38e965b77 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sat, 12 Aug 2023 22:35:56 +0600 Subject: [PATCH 08/33] Move urllib3 subclasses into separate package --- b2sdk/b2http.py | 2 +- b2sdk/requests/NOTICE | 12 +-------- b2sdk/requests/included_source_meta.py | 3 ++- b2sdk/urllib3/NOTICE | 8 ++++++ b2sdk/urllib3/__init__.py | 0 .../_continue.py => urllib3/_connection.py} | 0 b2sdk/urllib3/included_source_meta.py | 25 +++++++++++++++++++ 7 files changed, 37 insertions(+), 13 deletions(-) create mode 100644 b2sdk/urllib3/NOTICE create mode 100644 b2sdk/urllib3/__init__.py rename b2sdk/{requests/_continue.py => urllib3/_connection.py} (100%) create mode 100644 b2sdk/urllib3/included_source_meta.py diff --git a/b2sdk/b2http.py b/b2sdk/b2http.py index 4c159f98f..01bcfc35e 100644 --- a/b2sdk/b2http.py +++ b/b2sdk/b2http.py @@ -23,6 +23,7 @@ import requests +from b2sdk.urllib3._connection import HTTPAdapterWithContinue from .api_config import DEFAULT_HTTP_API_CONFIG, B2HttpApiConfig from .exception import ( B2ConnectionError, @@ -40,7 +41,6 @@ interpret_b2_error, ) from .requests import NotDecompressingResponse -from .requests._continue import HTTPAdapterWithContinue from .version import USER_AGENT LOCALE_LOCK = threading.Lock() diff --git a/b2sdk/requests/NOTICE b/b2sdk/requests/NOTICE index 18ddc6b3f..84dadbf44 100644 --- a/b2sdk/requests/NOTICE +++ b/b2sdk/requests/NOTICE @@ -5,14 +5,4 @@ Copyright 2021 Backblaze Inc. Changes made to the original source: requests.models.Response.iter_content has been overridden to pass `decode_content=False` argument to `self.raw.stream` in order to NOT decompress data based on Content-Encoding header -requests.adapters.HTTPAdapter has been overriden to use patched Urllib3 connection pools - - -Urllib3 -Copyright 2008-2011 Andrey Petrov and contributors. - -Copyright 2023 Backblaze Inc. -Changes made to the original source: -The following classes have been overriden for support HTTP 100-continue: -urllib3.connection.HTTPConnection, urllib3.connection.VerifiedHTTPSConnection, -urllib3.connectionpool.HTTPConnectionPool, urllib3.connectionpool.HTTPSConnectionPool +requests.adapters.HTTPAdapter has been overriden to use patched Urllib3 connection pools \ No newline at end of file diff --git a/b2sdk/requests/included_source_meta.py b/b2sdk/requests/included_source_meta.py index 40bfcbe2d..1f9df3172 100644 --- a/b2sdk/requests/included_source_meta.py +++ b/b2sdk/requests/included_source_meta.py @@ -18,7 +18,8 @@ Copyright 2021 Backblaze Inc. Changes made to the original source: requests.models.Response.iter_content has been overridden to pass `decode_content=False` argument to `self.raw.stream` -in order to NOT decompress data based on Content-Encoding header""" +in order to NOT decompress data based on Content-Encoding header +requests.adapters.HTTPAdapter has been overriden to use patched Urllib3 connection pools""" } ) add_included_source(included_source_meta) diff --git a/b2sdk/urllib3/NOTICE b/b2sdk/urllib3/NOTICE new file mode 100644 index 000000000..e024f31c1 --- /dev/null +++ b/b2sdk/urllib3/NOTICE @@ -0,0 +1,8 @@ +Urllib3 +Copyright 2008-2011 Andrey Petrov and contributors. + +Copyright 2023 Backblaze Inc. +Changes made to the original source: +The following classes have been overriden for support HTTP 100-continue: +urllib3.connection.HTTPConnection, urllib3.connection.VerifiedHTTPSConnection, +urllib3.connectionpool.HTTPConnectionPool, urllib3.connectionpool.HTTPSConnectionPool \ No newline at end of file diff --git a/b2sdk/urllib3/__init__.py b/b2sdk/urllib3/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/b2sdk/requests/_continue.py b/b2sdk/urllib3/_connection.py similarity index 100% rename from b2sdk/requests/_continue.py rename to b2sdk/urllib3/_connection.py diff --git a/b2sdk/urllib3/included_source_meta.py b/b2sdk/urllib3/included_source_meta.py new file mode 100644 index 000000000..bdf9cd7f6 --- /dev/null +++ b/b2sdk/urllib3/included_source_meta.py @@ -0,0 +1,25 @@ +###################################################################### +# +# File: b2sdk/requests/included_source_meta.py +# +# Copyright 2022 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +from b2sdk.included_sources import IncludedSourceMeta, add_included_source + +included_source_meta = IncludedSourceMeta( + 'urllib3', 'Included in a revised form', { + 'NOTICE': + """Urllib3 +Copyright 2008-2011 Andrey Petrov and contributors. + +Copyright 2023 Backblaze Inc. +Changes made to the original source: +The following classes have been overriden for support HTTP 100-continue: +urllib3.connection.HTTPConnection, urllib3.connection.VerifiedHTTPSConnection, +urllib3.connectionpool.HTTPConnectionPool, urllib3.connectionpool.HTTPSConnectionPool""" + } +) +add_included_source(included_source_meta) From 3c23b92000decbefba7b8b510b33e2d305d5d7c3 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sun, 13 Aug 2023 20:36:03 +0600 Subject: [PATCH 09/33] Fix NOTICE of vendored requests --- b2sdk/requests/NOTICE | 6 +++--- b2sdk/requests/included_source_meta.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/b2sdk/requests/NOTICE b/b2sdk/requests/NOTICE index 84dadbf44..cf7c367fa 100644 --- a/b2sdk/requests/NOTICE +++ b/b2sdk/requests/NOTICE @@ -3,6 +3,6 @@ Copyright 2019 Kenneth Reitz Copyright 2021 Backblaze Inc. Changes made to the original source: -requests.models.Response.iter_content has been overridden to pass `decode_content=False` argument to `self.raw.stream` -in order to NOT decompress data based on Content-Encoding header -requests.adapters.HTTPAdapter has been overriden to use patched Urllib3 connection pools \ No newline at end of file +* requests.models.Response.iter_content has been overridden to pass `decode_content=False` argument to `self.raw.stream` + in order to NOT decompress data based on Content-Encoding header +* requests.adapters.HTTPAdapter has been overriden to use patched Urllib3 connection pools \ No newline at end of file diff --git a/b2sdk/requests/included_source_meta.py b/b2sdk/requests/included_source_meta.py index 1f9df3172..b47f2b0af 100644 --- a/b2sdk/requests/included_source_meta.py +++ b/b2sdk/requests/included_source_meta.py @@ -17,9 +17,9 @@ Copyright 2021 Backblaze Inc. Changes made to the original source: -requests.models.Response.iter_content has been overridden to pass `decode_content=False` argument to `self.raw.stream` -in order to NOT decompress data based on Content-Encoding header -requests.adapters.HTTPAdapter has been overriden to use patched Urllib3 connection pools""" +* requests.models.Response.iter_content has been overridden to pass `decode_content=False` argument to `self.raw.stream` + in order to NOT decompress data based on Content-Encoding header +* requests.adapters.HTTPAdapter has been overriden to use patched Urllib3 connection pools""" } ) add_included_source(included_source_meta) From 3c1c10c664f31b2a9b05d71fb77f2ef964927802 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sun, 13 Aug 2023 20:36:59 +0600 Subject: [PATCH 10/33] Move HTTPAdapterWithContinue to b2sdk.requests --- b2sdk/b2http.py | 3 +-- b2sdk/requests/__init__.py | 10 ++++++++++ b2sdk/urllib3/_connection.py | 12 ------------ 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/b2sdk/b2http.py b/b2sdk/b2http.py index 01bcfc35e..4d083ce25 100644 --- a/b2sdk/b2http.py +++ b/b2sdk/b2http.py @@ -23,7 +23,6 @@ import requests -from b2sdk.urllib3._connection import HTTPAdapterWithContinue from .api_config import DEFAULT_HTTP_API_CONFIG, B2HttpApiConfig from .exception import ( B2ConnectionError, @@ -40,7 +39,7 @@ UnknownHost, interpret_b2_error, ) -from .requests import NotDecompressingResponse +from .requests import NotDecompressingResponse, HTTPAdapterWithContinue from .version import USER_AGENT LOCALE_LOCK = threading.Lock() diff --git a/b2sdk/requests/__init__.py b/b2sdk/requests/__init__.py index d1b657bd4..6afc2f373 100644 --- a/b2sdk/requests/__init__.py +++ b/b2sdk/requests/__init__.py @@ -16,11 +16,13 @@ """ from requests import Response, ConnectionError +from requests.adapters import HTTPAdapter, DEFAULT_POOLBLOCK from requests.exceptions import ChunkedEncodingError, ContentDecodingError, StreamConsumedError from requests.utils import iter_slices, stream_decode_response_unicode from urllib3.exceptions import ProtocolError, DecodeError, ReadTimeoutError from . import included_source_meta +from ..urllib3._connection import AWSHTTPConnectionPool, AWSHTTPSConnectionPool class NotDecompressingResponse(Response): @@ -77,3 +79,11 @@ def from_builtin_response(cls, response: Response): setattr(new_response, attr_name, getattr(response, attr_name)) new_response.raw = response.raw return new_response + + +class HTTPAdapterWithContinue(HTTPAdapter): + def init_poolmanager( + self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs + ): + super().init_poolmanager(connections, maxsize, block, **pool_kwargs) + self.poolmanager.pool_classes_by_scheme = {"http": AWSHTTPConnectionPool, "https": AWSHTTPSConnectionPool} diff --git a/b2sdk/urllib3/_connection.py b/b2sdk/urllib3/_connection.py index 067ebb11e..8cb4230e9 100644 --- a/b2sdk/urllib3/_connection.py +++ b/b2sdk/urllib3/_connection.py @@ -20,7 +20,6 @@ from http.client import HTTPResponse import urllib3 -from requests import adapters from urllib3.connection import HTTPConnection, VerifiedHTTPSConnection from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool @@ -225,14 +224,3 @@ class AWSHTTPConnectionPool(HTTPConnectionPool): class AWSHTTPSConnectionPool(HTTPSConnectionPool): ConnectionCls = AWSHTTPSConnection - - -pool_classes_by_scheme = {"http": AWSHTTPConnectionPool, "https": AWSHTTPSConnectionPool} - - -class HTTPAdapterWithContinue(adapters.HTTPAdapter): - def init_poolmanager( - self, connections, maxsize, block=adapters.DEFAULT_POOLBLOCK, **pool_kwargs - ): - super().init_poolmanager(connections, maxsize, block, **pool_kwargs) - self.poolmanager.pool_classes_by_scheme = pool_classes_by_scheme From b0e406384b1d37924125f63b5519ad0c91ad5650 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sun, 13 Aug 2023 20:59:39 +0600 Subject: [PATCH 11/33] Fix botocore vendor package name & NOTICE --- b2sdk/_botocore/NOTICE | 10 ++++++++++ b2sdk/{urllib3 => _botocore}/__init__.py | 0 .../_connection.py => _botocore/awsrequest.py} | 6 +++--- .../{urllib3 => _botocore}/included_source_meta.py | 14 ++++++++------ b2sdk/requests/__init__.py | 2 +- b2sdk/urllib3/NOTICE | 8 -------- 6 files changed, 22 insertions(+), 18 deletions(-) create mode 100644 b2sdk/_botocore/NOTICE rename b2sdk/{urllib3 => _botocore}/__init__.py (100%) rename b2sdk/{urllib3/_connection.py => _botocore/awsrequest.py} (98%) rename b2sdk/{urllib3 => _botocore}/included_source_meta.py (51%) delete mode 100644 b2sdk/urllib3/NOTICE diff --git a/b2sdk/_botocore/NOTICE b/b2sdk/_botocore/NOTICE new file mode 100644 index 000000000..5cfb4e738 --- /dev/null +++ b/b2sdk/_botocore/NOTICE @@ -0,0 +1,10 @@ +botocore +Copyright 2012-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. + +b2sdk includes vendorized parts of the botocore python library for 100-continue functionality. + +Copyright 2023 Backblaze Inc. +Changes made to the original source: +* Updated botocore.awsrequest.request method to work with str/byte header values (urllib 1.x vs 2.x) +* Updated botocore.awsrequest._handle_expect_response method to work with b2 responses +* Updated botocore.awsrequest._send_output method to change 100 response timeout \ No newline at end of file diff --git a/b2sdk/urllib3/__init__.py b/b2sdk/_botocore/__init__.py similarity index 100% rename from b2sdk/urllib3/__init__.py rename to b2sdk/_botocore/__init__.py diff --git a/b2sdk/urllib3/_connection.py b/b2sdk/_botocore/awsrequest.py similarity index 98% rename from b2sdk/urllib3/_connection.py rename to b2sdk/_botocore/awsrequest.py index 8cb4230e9..982934607 100644 --- a/b2sdk/urllib3/_connection.py +++ b/b2sdk/_botocore/awsrequest.py @@ -1,6 +1,6 @@ # Code taken from: # https://github.com/boto/botocore/blob/754b699bbf34261eae47c9dece3b11d7b58eb03c/botocore/awsrequest.py -# The code has been modified to work with urllib3>=2.0 +# The code has been modified to also work with urllib3>=2.0 # Copyright (c) 2012-2013 Mitch Garnaat http://garnaat.org/ # Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. @@ -117,8 +117,8 @@ def _send_output(self, message_body=None, *args, **kwargs): # This is our custom behavior. If the Expect header was # set, it will trigger this custom behavior. logger.debug("Waiting for 100 Continue response.") - # Wait for 1 second for the server to send a response. - if urllib3.util.wait_for_read(self.sock, 2): + # Wait for 10 seconds for the server to send a response. + if urllib3.util.wait_for_read(self.sock, 10): self._handle_expect_response(message_body) return else: diff --git a/b2sdk/urllib3/included_source_meta.py b/b2sdk/_botocore/included_source_meta.py similarity index 51% rename from b2sdk/urllib3/included_source_meta.py rename to b2sdk/_botocore/included_source_meta.py index bdf9cd7f6..6c77841ec 100644 --- a/b2sdk/urllib3/included_source_meta.py +++ b/b2sdk/_botocore/included_source_meta.py @@ -10,16 +10,18 @@ from b2sdk.included_sources import IncludedSourceMeta, add_included_source included_source_meta = IncludedSourceMeta( - 'urllib3', 'Included in a revised form', { + 'botocore', 'Included in a revised form', { 'NOTICE': - """Urllib3 -Copyright 2008-2011 Andrey Petrov and contributors. + """botocore +Copyright 2012-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. + +b2sdk includes vendorized parts of the botocore python library for 100-continue functionality. Copyright 2023 Backblaze Inc. Changes made to the original source: -The following classes have been overriden for support HTTP 100-continue: -urllib3.connection.HTTPConnection, urllib3.connection.VerifiedHTTPSConnection, -urllib3.connectionpool.HTTPConnectionPool, urllib3.connectionpool.HTTPSConnectionPool""" +* Updated botocore.awsrequest.request method to work with str/byte header values (urllib 1.x vs 2.x) +* Updated botocore.awsrequest._handle_expect_response method to work with b2 responses +* Updated botocore.awsrequest._send_output method to change 100 response timeout""" } ) add_included_source(included_source_meta) diff --git a/b2sdk/requests/__init__.py b/b2sdk/requests/__init__.py index 6afc2f373..318389193 100644 --- a/b2sdk/requests/__init__.py +++ b/b2sdk/requests/__init__.py @@ -22,7 +22,7 @@ from urllib3.exceptions import ProtocolError, DecodeError, ReadTimeoutError from . import included_source_meta -from ..urllib3._connection import AWSHTTPConnectionPool, AWSHTTPSConnectionPool +from .._botocore.awsrequest import AWSHTTPConnectionPool, AWSHTTPSConnectionPool class NotDecompressingResponse(Response): diff --git a/b2sdk/urllib3/NOTICE b/b2sdk/urllib3/NOTICE deleted file mode 100644 index e024f31c1..000000000 --- a/b2sdk/urllib3/NOTICE +++ /dev/null @@ -1,8 +0,0 @@ -Urllib3 -Copyright 2008-2011 Andrey Petrov and contributors. - -Copyright 2023 Backblaze Inc. -Changes made to the original source: -The following classes have been overriden for support HTTP 100-continue: -urllib3.connection.HTTPConnection, urllib3.connection.VerifiedHTTPSConnection, -urllib3.connectionpool.HTTPConnectionPool, urllib3.connectionpool.HTTPSConnectionPool \ No newline at end of file From 456b93b8a536594ceed51cd281baa387abb5f025 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sun, 13 Aug 2023 21:07:50 +0600 Subject: [PATCH 12/33] Update botocore LICENCE & NOTICE --- NOTES | 5 + b2sdk/_botocore/LICENSE | 177 ++++++++++++++++++++++++ b2sdk/_botocore/NOTICE | 69 ++++++++- b2sdk/_botocore/README.md | 3 + b2sdk/_botocore/included_source_meta.py | 70 +++++++++- 5 files changed, 318 insertions(+), 6 deletions(-) create mode 100644 NOTES create mode 100644 b2sdk/_botocore/LICENSE create mode 100644 b2sdk/_botocore/README.md diff --git a/NOTES b/NOTES new file mode 100644 index 000000000..4aa922ea8 --- /dev/null +++ b/NOTES @@ -0,0 +1,5 @@ +response status line = b'HTTP/1.1 401 \r\n' + + +`Backblaze / Opensource` :: `B2-40 support 100 Continue` +https://github.com/reef-technologies/b2-sdk-python/pull/300 B2-40 support 100 Continue \ No newline at end of file diff --git a/b2sdk/_botocore/LICENSE b/b2sdk/_botocore/LICENSE new file mode 100644 index 000000000..4947287f7 --- /dev/null +++ b/b2sdk/_botocore/LICENSE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS \ No newline at end of file diff --git a/b2sdk/_botocore/NOTICE b/b2sdk/_botocore/NOTICE index 5cfb4e738..7cd30d18a 100644 --- a/b2sdk/_botocore/NOTICE +++ b/b2sdk/_botocore/NOTICE @@ -1,5 +1,5 @@ -botocore -Copyright 2012-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. +b2sdk +Copyright 2023 Backblaze Inc. b2sdk includes vendorized parts of the botocore python library for 100-continue functionality. @@ -7,4 +7,67 @@ Copyright 2023 Backblaze Inc. Changes made to the original source: * Updated botocore.awsrequest.request method to work with str/byte header values (urllib 1.x vs 2.x) * Updated botocore.awsrequest._handle_expect_response method to work with b2 responses -* Updated botocore.awsrequest._send_output method to change 100 response timeout \ No newline at end of file +* Updated botocore.awsrequest._send_output method to change 100 response timeout + +--- + +Botocore +Copyright 2012-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. + +---- + +Botocore includes vendorized parts of the requests python library for backwards compatibility. + +Requests License +================ + +Copyright 2013 Kenneth Reitz + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Botocore includes vendorized parts of the urllib3 library for backwards compatibility. + +Urllib3 License +=============== + +This is the MIT license: http://www.opensource.org/licenses/mit-license.php + +Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt), +Modifications copyright 2012 Kenneth Reitz. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE +FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +Bundle of CA Root Certificates +============================== + +***** BEGIN LICENSE BLOCK ***** +This Source Code Form is subject to the terms of the +Mozilla Public License, v. 2.0. If a copy of the MPL +was not distributed with this file, You can obtain +one at http://mozilla.org/MPL/2.0/. + +***** END LICENSE BLOCK ***** diff --git a/b2sdk/_botocore/README.md b/b2sdk/_botocore/README.md new file mode 100644 index 000000000..cb77d81bc --- /dev/null +++ b/b2sdk/_botocore/README.md @@ -0,0 +1,3 @@ +This module contains modified parts of the botocore module (https://github.com/boto/botocore). +The modules original license is included in LICENSE. +Changes made to the original source are listed in NOTICE, along with original NOTICE. \ No newline at end of file diff --git a/b2sdk/_botocore/included_source_meta.py b/b2sdk/_botocore/included_source_meta.py index 6c77841ec..d12b8eb15 100644 --- a/b2sdk/_botocore/included_source_meta.py +++ b/b2sdk/_botocore/included_source_meta.py @@ -12,8 +12,8 @@ included_source_meta = IncludedSourceMeta( 'botocore', 'Included in a revised form', { 'NOTICE': - """botocore -Copyright 2012-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. + """b2sdk +Copyright 2023 Backblaze Inc. b2sdk includes vendorized parts of the botocore python library for 100-continue functionality. @@ -21,7 +21,71 @@ Changes made to the original source: * Updated botocore.awsrequest.request method to work with str/byte header values (urllib 1.x vs 2.x) * Updated botocore.awsrequest._handle_expect_response method to work with b2 responses -* Updated botocore.awsrequest._send_output method to change 100 response timeout""" +* Updated botocore.awsrequest._send_output method to change 100 response timeout + +--- + +Botocore +Copyright 2012-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. + +---- + +Botocore includes vendorized parts of the requests python library for backwards compatibility. + +Requests License +================ + +Copyright 2013 Kenneth Reitz + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Botocore includes vendorized parts of the urllib3 library for backwards compatibility. + +Urllib3 License +=============== + +This is the MIT license: http://www.opensource.org/licenses/mit-license.php + +Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt), +Modifications copyright 2012 Kenneth Reitz. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE +FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +Bundle of CA Root Certificates +============================== + +***** BEGIN LICENSE BLOCK ***** +This Source Code Form is subject to the terms of the +Mozilla Public License, v. 2.0. If a copy of the MPL +was not distributed with this file, You can obtain +one at http://mozilla.org/MPL/2.0/. + +***** END LICENSE BLOCK ***** +""" } ) add_included_source(included_source_meta) From 2d905b53e78da54e10f107d049a94aba540b9e4c Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sun, 13 Aug 2023 21:14:43 +0600 Subject: [PATCH 13/33] Add urllib3 as explicit dependency --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index e99bd83ef..208309ab7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ importlib-metadata>=3.3.0; python_version < '3.8' logfury>=1.0.1,<2.0.0 requests>=2.9.1,<3.0.0 +urllib3>=1.21.1,<3 tqdm>=4.5.0,<5.0.0 typing-extensions>=4.7.1; python_version < '3.12' From 73b7237a9571ef7d4e7306ae2e9f50950cff1f83 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sun, 13 Aug 2023 21:22:40 +0600 Subject: [PATCH 14/33] Add file contents inclusion assertion to 100-continue test --- test/integration/test_raw_api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/integration/test_raw_api.py b/test/integration/test_raw_api.py index 3eb6984aa..720726921 100644 --- a/test/integration/test_raw_api.py +++ b/test/integration/test_raw_api.py @@ -604,6 +604,7 @@ def patched_send(self, data): # this should not fail data.seek(0) + sent_data.clear() raw_api.upload_file( upload_url, upload_auth_token, @@ -615,6 +616,7 @@ def patched_send(self, data): data, server_side_encryption=sse_b2_aes, ) + assert file_contents in sent_data # Clean up this test. _clean_and_delete_bucket(raw_api, api_url, account_auth_token, account_id, bucket_id) From 3c2aa6243a3de2f75982f9d73e9d39e005a56e6c Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Mon, 14 Aug 2023 01:07:27 +0600 Subject: [PATCH 15/33] Enable 100-continue for file uploads by default Also added two configurations in B2HttpApiConfig to disable this feature or to configure 100-continue response timeout --- b2sdk/_botocore/awsrequest.py | 8 +++++++- b2sdk/api_config.py | 6 +++++- b2sdk/file_version.py | 2 ++ b2sdk/raw_api.py | 6 ++++++ b2sdk/raw_simulator.py | 4 ++++ test/integration/test_raw_api.py | 30 ------------------------------ 6 files changed, 24 insertions(+), 32 deletions(-) diff --git a/b2sdk/_botocore/awsrequest.py b/b2sdk/_botocore/awsrequest.py index 982934607..9c85cbe94 100644 --- a/b2sdk/_botocore/awsrequest.py +++ b/b2sdk/_botocore/awsrequest.py @@ -67,6 +67,7 @@ def __init__(self, *args, **kwargs): self._response_received = False self._expect_header_set = False self._send_called = False + self._continue_timeout = 10.0 def close(self): super().close() @@ -82,6 +83,11 @@ def request(self, method, url, body=None, headers=None, *args, **kwargs): self._response_received = False if headers.get('Expect', b'') in [b'100-continue', '100-continue']: self._expect_header_set = True + timeout = headers.pop('X-Expect-100-Continue-Timeout-Seconds', self._continue_timeout) + try: + self._continue_timeout = float(timeout) + except (ValueError, TypeError): + pass else: self._expect_header_set = False self.response_class = self._original_response_cls @@ -118,7 +124,7 @@ def _send_output(self, message_body=None, *args, **kwargs): # set, it will trigger this custom behavior. logger.debug("Waiting for 100 Continue response.") # Wait for 10 seconds for the server to send a response. - if urllib3.util.wait_for_read(self.sock, 10): + if urllib3.util.wait_for_read(self.sock, self._continue_timeout): self._handle_expect_response(message_body) return else: diff --git a/b2sdk/api_config.py b/b2sdk/api_config.py index 1dd10d314..e7871f5d8 100644 --- a/b2sdk/api_config.py +++ b/b2sdk/api_config.py @@ -26,7 +26,9 @@ def __init__( install_clock_skew_hook: bool = True, user_agent_append: str | None = None, _raw_api_class: type[AbstractRawApi] | None = None, - decode_content: bool = False + decode_content: bool = False, + expect_100_continue: bool = True, + expect_100_continue_timeout_seconds: float = 10.0, ): """ A structure with params to be passed to low level API. @@ -43,6 +45,8 @@ def __init__( self.user_agent_append = user_agent_append self.raw_api_class = _raw_api_class or self.DEFAULT_RAW_API_CLASS self.decode_content = decode_content + self.expect_100_continue = expect_100_continue + self.expect_100_continue_timeout_seconds = expect_100_continue_timeout_seconds DEFAULT_HTTP_API_CONFIG = B2HttpApiConfig() diff --git a/b2sdk/file_version.py b/b2sdk/file_version.py index 0c3b5524d..5c899ed68 100644 --- a/b2sdk/file_version.py +++ b/b2sdk/file_version.py @@ -351,6 +351,8 @@ def _get_upload_headers(self) -> bytes: file_retention=self.file_retention, legal_hold=self.legal_hold, cache_control=self.cache_control, + expect_100_continue=self.api.api_config.expect_100_continue, + expect_100_timeout_seconds=self.api.api_config.expect_100_continue_timeout_seconds, ) headers_str = ''.join( diff --git a/b2sdk/raw_api.py b/b2sdk/raw_api.py index cd6302d1d..1469c2172 100644 --- a/b2sdk/raw_api.py +++ b/b2sdk/raw_api.py @@ -351,6 +351,8 @@ def get_upload_file_headers( legal_hold: LegalHold | None, custom_upload_timestamp: int | None = None, cache_control: str | None = None, + expect_100_continue: bool = True, + expect_100_timeout_seconds: float = 10.0, ) -> dict: headers = { 'Authorization': upload_auth_token, @@ -379,6 +381,10 @@ def get_upload_file_headers( if custom_upload_timestamp is not None: headers['X-Bz-Custom-Upload-Timestamp'] = str(custom_upload_timestamp) + if expect_100_continue: + headers['Expect'] = '100-continue' + headers['X-Expect-100-Continue-Timeout-Seconds'] = str(expect_100_timeout_seconds) + return headers @abstractmethod diff --git a/b2sdk/raw_simulator.py b/b2sdk/raw_simulator.py index 9d01c7b4f..10641b304 100644 --- a/b2sdk/raw_simulator.py +++ b/b2sdk/raw_simulator.py @@ -1800,6 +1800,8 @@ def get_upload_file_headers( legal_hold: LegalHold | None, custom_upload_timestamp: int | None = None, cache_control: str | None = None, + expect_100_continue: bool = True, + expect_100_timeout_seconds: float = 10.0, ) -> dict: # fix to allow calculating headers on unknown key - only for simulation @@ -1820,6 +1822,8 @@ def get_upload_file_headers( legal_hold=legal_hold, custom_upload_timestamp=custom_upload_timestamp, cache_control=cache_control, + expect_100_continue=expect_100_continue, + expect_100_timeout_seconds=expect_100_timeout_seconds, ) def upload_file( diff --git a/test/integration/test_raw_api.py b/test/integration/test_raw_api.py index 720726921..b9e8293fe 100644 --- a/test/integration/test_raw_api.py +++ b/test/integration/test_raw_api.py @@ -557,15 +557,6 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets, monkeypatch): file_contents = b'hello world' file_sha1 = hex_sha1_of_stream(io.BytesIO(file_contents), len(file_contents)) - def set_100_header(fn): - def wrapper(url, headers, *args, **kwargs): - headers.update({ - 'Expect': '100-continue', - }) - return fn(url, headers, *args, **kwargs) - - return wrapper - orig_send = http.client.HTTPConnection.send sent_data = bytearray() @@ -574,11 +565,6 @@ def patched_send(self, data): return orig_send(self, data) with monkeypatch.context() as monkey: - monkey.setattr( - raw_api.b2_http, - 'post_content_return_json', - set_100_header(raw_api.b2_http.post_content_return_json), - ) monkey.setattr( http.client.HTTPConnection, "send", @@ -602,22 +588,6 @@ def patched_send(self, data): # Check if data was sent assert file_contents not in sent_data - # this should not fail - data.seek(0) - sent_data.clear() - raw_api.upload_file( - upload_url, - upload_auth_token, - file_name, - len(file_contents), - 'text/plain', - file_sha1, - {'color': 'blue'}, - data, - server_side_encryption=sse_b2_aes, - ) - assert file_contents in sent_data - # Clean up this test. _clean_and_delete_bucket(raw_api, api_url, account_auth_token, account_id, bucket_id) From dd1ecd55b26e4aaa8a5e205960dacdc0477f8062 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Mon, 14 Aug 2023 01:28:33 +0600 Subject: [PATCH 16/33] Fix broken b2sdk/file_version.py --- b2sdk/file_version.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/b2sdk/file_version.py b/b2sdk/file_version.py index 5c899ed68..99ec74c49 100644 --- a/b2sdk/file_version.py +++ b/b2sdk/file_version.py @@ -351,8 +351,7 @@ def _get_upload_headers(self) -> bytes: file_retention=self.file_retention, legal_hold=self.legal_hold, cache_control=self.cache_control, - expect_100_continue=self.api.api_config.expect_100_continue, - expect_100_timeout_seconds=self.api.api_config.expect_100_continue_timeout_seconds, + expect_100_continue=False, ) headers_str = ''.join( From 42a6fc1aa0c492b69ea8843772fb8932b6b49e30 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Mon, 14 Aug 2023 02:27:38 +0600 Subject: [PATCH 17/33] Propagate api_config.expect_100_* settings --- b2sdk/raw_api.py | 17 +++++++++++++++-- b2sdk/raw_simulator.py | 12 ++++++++++-- b2sdk/session.py | 7 +++++++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/b2sdk/raw_api.py b/b2sdk/raw_api.py index 1469c2172..64de5b6ce 100644 --- a/b2sdk/raw_api.py +++ b/b2sdk/raw_api.py @@ -352,7 +352,7 @@ def get_upload_file_headers( custom_upload_timestamp: int | None = None, cache_control: str | None = None, expect_100_continue: bool = True, - expect_100_timeout_seconds: float = 10.0, + expect_100_continue_timeout_seconds: float = 10.0, ) -> dict: headers = { 'Authorization': upload_auth_token, @@ -383,7 +383,7 @@ def get_upload_file_headers( if expect_100_continue: headers['Expect'] = '100-continue' - headers['X-Expect-100-Continue-Timeout-Seconds'] = str(expect_100_timeout_seconds) + headers['X-Expect-100-Continue-Timeout-Seconds'] = str(expect_100_continue_timeout_seconds) return headers @@ -403,6 +403,8 @@ def upload_file( legal_hold: LegalHold | None = None, custom_upload_timestamp: int | None = None, cache_control: str | None = None, + expect_100_continue: bool = True, + expect_100_continue_timeout_seconds: float = 10.0, ): pass @@ -938,6 +940,8 @@ def upload_file( legal_hold: LegalHold | None = None, custom_upload_timestamp: int | None = None, cache_control: str | None = None, + expect_100_continue: bool = True, + expect_100_continue_timeout_seconds: float = 10.0, ): """ Upload one, small file to b2. @@ -955,6 +959,8 @@ def upload_file( :param legal_hold: legal hold setting for the file :param custom_upload_timestamp: custom upload timestamp for the file :param cache_control: an optional cache control setting. Syntax based on the section 14.9 of RFC 2616. Example string value: 'public, max-age=86400, s-maxage=3600, no-transform'. + :param expect_100_continue: whether to use 'Expect: 100-continue' header + :param expect_100_continue_timeout_seconds: timeout of 100 response when expect_100_continue is True :return: """ # Raise UnusableFileName if the file_name doesn't meet the rules. @@ -971,6 +977,8 @@ def upload_file( legal_hold=legal_hold, custom_upload_timestamp=custom_upload_timestamp, cache_control=cache_control, + expect_100_continue=expect_100_continue, + expect_100_continue_timeout_seconds=expect_100_continue_timeout_seconds, ) return self.b2_http.post_content_return_json(upload_url, headers, data_stream) @@ -983,6 +991,8 @@ def upload_part( content_sha1, data_stream, server_side_encryption: EncryptionSetting | None = None, + expect_100_continue: bool = True, + expect_100_continue_timeout_seconds: float = 10.0, ): headers = { 'Authorization': upload_auth_token, @@ -995,6 +1005,9 @@ def upload_part( EncryptionMode.NONE, EncryptionMode.SSE_B2, EncryptionMode.SSE_C ) server_side_encryption.add_to_upload_headers(headers) + if expect_100_continue: + headers['Expect'] = '100-continue' + headers['X-Expect-100-Continue-Timeout-Seconds'] = str(expect_100_continue_timeout_seconds) return self.b2_http.post_content_return_json(upload_url, headers, data_stream) diff --git a/b2sdk/raw_simulator.py b/b2sdk/raw_simulator.py index 10641b304..31f2aeb2c 100644 --- a/b2sdk/raw_simulator.py +++ b/b2sdk/raw_simulator.py @@ -1014,6 +1014,8 @@ def upload_file( legal_hold: LegalHold | None = None, custom_upload_timestamp: int | None = None, cache_control: str | None = None, + expect_100_continue: bool = True, + expect_100_continue_timeout_seconds: float = 10.0, ): data_bytes = self._simulate_chunked_post(data_stream, content_length) assert len(data_bytes) == content_length @@ -1801,7 +1803,7 @@ def get_upload_file_headers( custom_upload_timestamp: int | None = None, cache_control: str | None = None, expect_100_continue: bool = True, - expect_100_timeout_seconds: float = 10.0, + expect_100_continue_timeout_seconds: float = 10.0, ) -> dict: # fix to allow calculating headers on unknown key - only for simulation @@ -1823,7 +1825,7 @@ def get_upload_file_headers( custom_upload_timestamp=custom_upload_timestamp, cache_control=cache_control, expect_100_continue=expect_100_continue, - expect_100_timeout_seconds=expect_100_timeout_seconds, + expect_100_continue_timeout_seconds=expect_100_continue_timeout_seconds, ) def upload_file( @@ -1841,6 +1843,8 @@ def upload_file( legal_hold: LegalHold | None = None, custom_upload_timestamp: int | None = None, cache_control: str | None = None, + expect_100_continue: bool = True, + expect_100_continue_timeout_seconds: float = 10.0, ): with ConcurrentUsedAuthTokenGuard( self.currently_used_auth_tokens[upload_auth_token], upload_auth_token @@ -1873,6 +1877,8 @@ def upload_file( legal_hold=legal_hold, custom_upload_timestamp=custom_upload_timestamp, cache_control=cache_control, + expect_100_continue=expect_100_continue, + expect_100_continue_timeout_seconds=expect_100_continue_timeout_seconds, ) response = bucket.upload_file( @@ -1904,6 +1910,8 @@ def upload_part( sha1_sum, input_stream, server_side_encryption: EncryptionSetting | None = None, + expect_100_continue: bool = True, + expect_100_continue_timeout_seconds: float = 10.0, ): with ConcurrentUsedAuthTokenGuard( self.currently_used_auth_tokens[upload_auth_token], upload_auth_token diff --git a/b2sdk/session.py b/b2sdk/session.py index f3214dd24..23c793fb7 100644 --- a/b2sdk/session.py +++ b/b2sdk/session.py @@ -87,6 +87,9 @@ def __init__( TokenType.UPLOAD_PART: self._upload_part, } + self.expect_100_continue = api_config.expect_100_continue + self.expect_100_continue_timeout_seconds = api_config.expect_100_continue_timeout_seconds + def authorize_automatically(self): """ Perform automatic account authorization, retrieving all account data @@ -370,6 +373,8 @@ def upload_file( legal_hold=legal_hold, custom_upload_timestamp=custom_upload_timestamp, cache_control=cache_control, + expect_100_continue=self.expect_100_continue, + expect_100_continue_timeout_seconds=self.expect_100_continue_timeout_seconds, ) def upload_part( @@ -390,6 +395,8 @@ def upload_part( sha1_sum, input_stream, server_side_encryption, + expect_100_continue=self.expect_100_continue, + expect_100_continue_timeout_seconds=self.expect_100_continue_timeout_seconds, ) def get_download_url_by_id(self, file_id): From 5009d6ee16a9a80593e31febd439e9f35053b98f Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Mon, 14 Aug 2023 02:46:04 +0600 Subject: [PATCH 18/33] Rename except-100 vars & headers --- NOTES | 5 ----- b2sdk/_botocore/awsrequest.py | 9 +++------ b2sdk/api_config.py | 4 ++-- b2sdk/raw_api.py | 16 ++++++++-------- b2sdk/raw_simulator.py | 12 ++++++------ b2sdk/session.py | 6 +++--- 6 files changed, 22 insertions(+), 30 deletions(-) delete mode 100644 NOTES diff --git a/NOTES b/NOTES deleted file mode 100644 index 4aa922ea8..000000000 --- a/NOTES +++ /dev/null @@ -1,5 +0,0 @@ -response status line = b'HTTP/1.1 401 \r\n' - - -`Backblaze / Opensource` :: `B2-40 support 100 Continue` -https://github.com/reef-technologies/b2-sdk-python/pull/300 B2-40 support 100 Continue \ No newline at end of file diff --git a/b2sdk/_botocore/awsrequest.py b/b2sdk/_botocore/awsrequest.py index 9c85cbe94..e0a7f0428 100644 --- a/b2sdk/_botocore/awsrequest.py +++ b/b2sdk/_botocore/awsrequest.py @@ -19,7 +19,7 @@ import logging from http.client import HTTPResponse -import urllib3 +import urllib3.util from urllib3.connection import HTTPConnection, VerifiedHTTPSConnection from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool @@ -83,11 +83,8 @@ def request(self, method, url, body=None, headers=None, *args, **kwargs): self._response_received = False if headers.get('Expect', b'') in [b'100-continue', '100-continue']: self._expect_header_set = True - timeout = headers.pop('X-Expect-100-Continue-Timeout-Seconds', self._continue_timeout) - try: - self._continue_timeout = float(timeout) - except (ValueError, TypeError): - pass + timeout = headers.pop('X-Expect-100-Timeout', self._continue_timeout) + self._continue_timeout = float(timeout) else: self._expect_header_set = False self.response_class = self._original_response_cls diff --git a/b2sdk/api_config.py b/b2sdk/api_config.py index e7871f5d8..0ff62d608 100644 --- a/b2sdk/api_config.py +++ b/b2sdk/api_config.py @@ -28,7 +28,7 @@ def __init__( _raw_api_class: type[AbstractRawApi] | None = None, decode_content: bool = False, expect_100_continue: bool = True, - expect_100_continue_timeout_seconds: float = 10.0, + expect_100_timeout: float = 10.0, ): """ A structure with params to be passed to low level API. @@ -46,7 +46,7 @@ def __init__( self.raw_api_class = _raw_api_class or self.DEFAULT_RAW_API_CLASS self.decode_content = decode_content self.expect_100_continue = expect_100_continue - self.expect_100_continue_timeout_seconds = expect_100_continue_timeout_seconds + self.expect_100_timeout = expect_100_timeout DEFAULT_HTTP_API_CONFIG = B2HttpApiConfig() diff --git a/b2sdk/raw_api.py b/b2sdk/raw_api.py index 64de5b6ce..2ac52c911 100644 --- a/b2sdk/raw_api.py +++ b/b2sdk/raw_api.py @@ -352,7 +352,7 @@ def get_upload_file_headers( custom_upload_timestamp: int | None = None, cache_control: str | None = None, expect_100_continue: bool = True, - expect_100_continue_timeout_seconds: float = 10.0, + expect_100_timeout: float = 10.0, ) -> dict: headers = { 'Authorization': upload_auth_token, @@ -383,7 +383,7 @@ def get_upload_file_headers( if expect_100_continue: headers['Expect'] = '100-continue' - headers['X-Expect-100-Continue-Timeout-Seconds'] = str(expect_100_continue_timeout_seconds) + headers['X-Expect-100-Timeout'] = str(expect_100_timeout) return headers @@ -404,7 +404,7 @@ def upload_file( custom_upload_timestamp: int | None = None, cache_control: str | None = None, expect_100_continue: bool = True, - expect_100_continue_timeout_seconds: float = 10.0, + expect_100_timeout: float = 10.0, ): pass @@ -941,7 +941,7 @@ def upload_file( custom_upload_timestamp: int | None = None, cache_control: str | None = None, expect_100_continue: bool = True, - expect_100_continue_timeout_seconds: float = 10.0, + expect_100_timeout: float = 10.0, ): """ Upload one, small file to b2. @@ -960,7 +960,7 @@ def upload_file( :param custom_upload_timestamp: custom upload timestamp for the file :param cache_control: an optional cache control setting. Syntax based on the section 14.9 of RFC 2616. Example string value: 'public, max-age=86400, s-maxage=3600, no-transform'. :param expect_100_continue: whether to use 'Expect: 100-continue' header - :param expect_100_continue_timeout_seconds: timeout of 100 response when expect_100_continue is True + :param expect_100_timeout: timeout of 100 response when expect_100_continue is True :return: """ # Raise UnusableFileName if the file_name doesn't meet the rules. @@ -978,7 +978,7 @@ def upload_file( custom_upload_timestamp=custom_upload_timestamp, cache_control=cache_control, expect_100_continue=expect_100_continue, - expect_100_continue_timeout_seconds=expect_100_continue_timeout_seconds, + expect_100_timeout=expect_100_timeout, ) return self.b2_http.post_content_return_json(upload_url, headers, data_stream) @@ -992,7 +992,7 @@ def upload_part( data_stream, server_side_encryption: EncryptionSetting | None = None, expect_100_continue: bool = True, - expect_100_continue_timeout_seconds: float = 10.0, + expect_100_timeout: float = 10.0, ): headers = { 'Authorization': upload_auth_token, @@ -1007,7 +1007,7 @@ def upload_part( server_side_encryption.add_to_upload_headers(headers) if expect_100_continue: headers['Expect'] = '100-continue' - headers['X-Expect-100-Continue-Timeout-Seconds'] = str(expect_100_continue_timeout_seconds) + headers['X-Expect-100-Timeout'] = str(expect_100_timeout) return self.b2_http.post_content_return_json(upload_url, headers, data_stream) diff --git a/b2sdk/raw_simulator.py b/b2sdk/raw_simulator.py index 31f2aeb2c..d9fc54c43 100644 --- a/b2sdk/raw_simulator.py +++ b/b2sdk/raw_simulator.py @@ -1015,7 +1015,7 @@ def upload_file( custom_upload_timestamp: int | None = None, cache_control: str | None = None, expect_100_continue: bool = True, - expect_100_continue_timeout_seconds: float = 10.0, + expect_100_timeout: float = 10.0, ): data_bytes = self._simulate_chunked_post(data_stream, content_length) assert len(data_bytes) == content_length @@ -1803,7 +1803,7 @@ def get_upload_file_headers( custom_upload_timestamp: int | None = None, cache_control: str | None = None, expect_100_continue: bool = True, - expect_100_continue_timeout_seconds: float = 10.0, + expect_100_timeout: float = 10.0, ) -> dict: # fix to allow calculating headers on unknown key - only for simulation @@ -1825,7 +1825,7 @@ def get_upload_file_headers( custom_upload_timestamp=custom_upload_timestamp, cache_control=cache_control, expect_100_continue=expect_100_continue, - expect_100_continue_timeout_seconds=expect_100_continue_timeout_seconds, + expect_100_timeout=expect_100_timeout, ) def upload_file( @@ -1844,7 +1844,7 @@ def upload_file( custom_upload_timestamp: int | None = None, cache_control: str | None = None, expect_100_continue: bool = True, - expect_100_continue_timeout_seconds: float = 10.0, + expect_100_timeout: float = 10.0, ): with ConcurrentUsedAuthTokenGuard( self.currently_used_auth_tokens[upload_auth_token], upload_auth_token @@ -1878,7 +1878,7 @@ def upload_file( custom_upload_timestamp=custom_upload_timestamp, cache_control=cache_control, expect_100_continue=expect_100_continue, - expect_100_continue_timeout_seconds=expect_100_continue_timeout_seconds, + expect_100_timeout=expect_100_timeout, ) response = bucket.upload_file( @@ -1911,7 +1911,7 @@ def upload_part( input_stream, server_side_encryption: EncryptionSetting | None = None, expect_100_continue: bool = True, - expect_100_continue_timeout_seconds: float = 10.0, + expect_100_timeout: float = 10.0, ): with ConcurrentUsedAuthTokenGuard( self.currently_used_auth_tokens[upload_auth_token], upload_auth_token diff --git a/b2sdk/session.py b/b2sdk/session.py index 23c793fb7..66876ddf1 100644 --- a/b2sdk/session.py +++ b/b2sdk/session.py @@ -88,7 +88,7 @@ def __init__( } self.expect_100_continue = api_config.expect_100_continue - self.expect_100_continue_timeout_seconds = api_config.expect_100_continue_timeout_seconds + self.expect_100_timeout = api_config.expect_100_timeout def authorize_automatically(self): """ @@ -374,7 +374,7 @@ def upload_file( custom_upload_timestamp=custom_upload_timestamp, cache_control=cache_control, expect_100_continue=self.expect_100_continue, - expect_100_continue_timeout_seconds=self.expect_100_continue_timeout_seconds, + expect_100_timeout=self.expect_100_timeout, ) def upload_part( @@ -396,7 +396,7 @@ def upload_part( input_stream, server_side_encryption, expect_100_continue=self.expect_100_continue, - expect_100_continue_timeout_seconds=self.expect_100_continue_timeout_seconds, + expect_100_timeout=self.expect_100_timeout, ) def get_download_url_by_id(self, file_id): From 710c36101907dba87768abbe7668ffaa13c8c1f7 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Tue, 15 Aug 2023 17:59:49 +0600 Subject: [PATCH 19/33] Remove misleading comment --- b2sdk/_botocore/awsrequest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/b2sdk/_botocore/awsrequest.py b/b2sdk/_botocore/awsrequest.py index e0a7f0428..4ea175f9c 100644 --- a/b2sdk/_botocore/awsrequest.py +++ b/b2sdk/_botocore/awsrequest.py @@ -120,7 +120,6 @@ def _send_output(self, message_body=None, *args, **kwargs): # This is our custom behavior. If the Expect header was # set, it will trigger this custom behavior. logger.debug("Waiting for 100 Continue response.") - # Wait for 10 seconds for the server to send a response. if urllib3.util.wait_for_read(self.sock, self._continue_timeout): self._handle_expect_response(message_body) return From 9c92658cd7be74abf00c4ef07b03e019bc0e86c3 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Tue, 15 Aug 2023 18:00:59 +0600 Subject: [PATCH 20/33] Add changelog entry for 100-continue --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e19835d3..c4bccd4ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Require `typing_extensions` on Python 3.11 (already required on earlier versinons) for better compatibility with pydantic v2 * Fix `RawSimulator` handling of `cache_control` parameter during tests. +### Changed +* Uploading files now makes use of `Expect: 100-continue` header + ## [1.22.1] - 2023-07-24 ### Fixed From 015eb4da7c48fd75f14712beb5f4a07bf998c294 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Tue, 15 Aug 2023 19:41:25 +0600 Subject: [PATCH 21/33] Move expect-100 tests into separate file --- test/integration/test_raw_api.py | 46 +------- test/integration/test_raw_expect_100.py | 141 ++++++++++++++++++++++++ 2 files changed, 145 insertions(+), 42 deletions(-) create mode 100644 test/integration/test_raw_expect_100.py diff --git a/test/integration/test_raw_api.py b/test/integration/test_raw_api.py index b9e8293fe..89d17ab18 100644 --- a/test/integration/test_raw_api.py +++ b/test/integration/test_raw_api.py @@ -9,7 +9,6 @@ ###################################################################### from __future__ import annotations -import http.client import io import os import random @@ -22,7 +21,7 @@ from b2sdk.b2http import B2Http from b2sdk.encryption.setting import EncryptionAlgorithm, EncryptionMode, EncryptionSetting -from b2sdk.exception import DisablingFileLockNotSupported, InvalidAuthToken +from b2sdk.exception import DisablingFileLockNotSupported from b2sdk.file_lock import ( NO_RETENTION_FILE_SETTING, BucketRetentionSetting, @@ -36,7 +35,7 @@ # TODO: rewrite to separate test cases -def test_raw_api(dont_cleanup_old_buckets, monkeypatch): +def test_raw_api(dont_cleanup_old_buckets): """ Exercise the code in B2RawHTTPApi by making each call once, just to make sure the parameters are passed in, and the result is @@ -61,7 +60,7 @@ def test_raw_api(dont_cleanup_old_buckets, monkeypatch): try: raw_api = B2RawHTTPApi(B2Http()) - raw_api_test_helper(raw_api, not dont_cleanup_old_buckets, monkeypatch) + raw_api_test_helper(raw_api, not dont_cleanup_old_buckets) except Exception: traceback.print_exc(file=sys.stdout) pytest.fail('test_raw_api failed') @@ -84,7 +83,7 @@ def authorize_raw_api(raw_api): return auth_dict -def raw_api_test_helper(raw_api, should_cleanup_old_buckets, monkeypatch): +def raw_api_test_helper(raw_api, should_cleanup_old_buckets): """ Try each of the calls to the raw api. Raise an exception if anything goes wrong. @@ -551,43 +550,6 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets, monkeypatch): is_file_lock_enabled=False, ) - # b2_100_continue - print('b2_100_continue') - file_name = 'test-100-continue.txt' - file_contents = b'hello world' - file_sha1 = hex_sha1_of_stream(io.BytesIO(file_contents), len(file_contents)) - - orig_send = http.client.HTTPConnection.send - sent_data = bytearray() - - def patched_send(self, data): - sent_data.extend(data) - return orig_send(self, data) - - with monkeypatch.context() as monkey: - monkey.setattr( - http.client.HTTPConnection, - "send", - patched_send, - ) - data = io.BytesIO(file_contents) - - with pytest.raises(InvalidAuthToken): - raw_api.upload_file( - upload_url, - upload_auth_token + 'x', - file_name, - len(file_contents), - 'text/plain', - file_sha1, - {'color': 'blue'}, - data, - server_side_encryption=sse_b2_aes, - ) - - # Check if data was sent - assert file_contents not in sent_data - # Clean up this test. _clean_and_delete_bucket(raw_api, api_url, account_auth_token, account_id, bucket_id) diff --git a/test/integration/test_raw_expect_100.py b/test/integration/test_raw_expect_100.py new file mode 100644 index 000000000..016a748af --- /dev/null +++ b/test/integration/test_raw_expect_100.py @@ -0,0 +1,141 @@ +import http.client +import io +import os +import random +import secrets +import time + +from b2sdk.encryption.setting import EncryptionSetting +from b2sdk.encryption.types import EncryptionMode, EncryptionAlgorithm +from b2sdk.exception import InvalidAuthToken +from b2sdk.utils import hex_sha1_of_stream +import pytest +from b2sdk.b2http import B2Http +from b2sdk.raw_api import REALM_URLS, B2RawHTTPApi + +from .fixtures import b2_auth_data + + +@pytest.fixture +def expect_100_setup(b2_auth_data, monkeypatch): + application_key_id, application_key = b2_auth_data + raw_api = B2RawHTTPApi(B2Http()) + realm = os.environ.get('B2_TEST_ENVIRONMENT', 'production') + realm_url = REALM_URLS.get(realm, realm) + auth_dict = raw_api.authorize_account(realm_url, application_key_id, application_key) + + account_id = auth_dict['accountId'] + account_auth_token = auth_dict['authorizationToken'] + api_url = auth_dict['apiUrl'] + + sse_b2_aes = EncryptionSetting( + mode=EncryptionMode.SSE_B2, + algorithm=EncryptionAlgorithm.AES256, + ) + + bucket_name = 'test-raw-api-%s-%d-%d' % ( + account_id, int(time.time()), random.randint(1000, 9999) + ) + + bucket_dict = raw_api.create_bucket( + api_url, + account_auth_token, + account_id, + bucket_name, + 'allPublic', + is_file_lock_enabled=True, + ) + + upload_url_dict = raw_api.get_upload_url(api_url, account_auth_token, bucket_dict['bucketId']) + upload_url = upload_url_dict['uploadUrl'] + upload_auth_token = upload_url_dict['authorizationToken'] + + orig_send = http.client.HTTPConnection.send + sent_data = bytearray() + + def patched_send(self, data): + sent_data.extend(data) + return orig_send(self, data) + + monkeypatch.setattr( + http.client.HTTPConnection, + "send", + patched_send, + ) + + file_name = 'test-100-continue.txt' + file_contents = secrets.token_bytes() + file_length = len(file_contents) + file_sha1 = hex_sha1_of_stream(io.BytesIO(file_contents), file_length) + + return { + "raw_api": raw_api, + "upload_url": upload_url, + "upload_auth_token": upload_auth_token, + "sse_b2_aes": sse_b2_aes, + "sent_data": sent_data, + "file_name": file_name, + "file_contents": file_contents, + "file_length": file_length, + "file_sha1": file_sha1, + } + + +@pytest.fixture(autouse=True) +def rest_sent_data(expect_100_setup): + expect_100_setup["sent_data"].clear() + + +def test_expect_100_non_100_response(expect_100_setup): + raw_api = expect_100_setup["raw_api"] + data = io.BytesIO(expect_100_setup["file_contents"]) + with pytest.raises(InvalidAuthToken): + raw_api.upload_file( + expect_100_setup["upload_url"], + expect_100_setup["upload_auth_token"] + 'wrong token', + expect_100_setup["file_name"], + expect_100_setup["file_contents"], + 'text/plain', + expect_100_setup["file_sha1"], + {'color': 'blue'}, + data, + server_side_encryption=expect_100_setup["sse_b2_aes"], + ) + assert expect_100_setup["file_contents"] not in expect_100_setup["sent_data"] + + +def test_expect_100_timeout(expect_100_setup): + raw_api = expect_100_setup["raw_api"] + data = io.BytesIO(expect_100_setup["file_contents"]) + raw_api.upload_file( + expect_100_setup["upload_url"], + expect_100_setup["upload_auth_token"], + expect_100_setup["file_name"], + expect_100_setup["file_contents"], + 'text/plain', + expect_100_setup["file_sha1"], + {'color': 'blue'}, + data, + server_side_encryption=expect_100_setup["sse_b2_aes"], + expect_100_timeout=0, + ) + assert expect_100_setup["file_contents"] in expect_100_setup["sent_data"] + + +def test_expect_100_disabled(expect_100_setup): + raw_api = expect_100_setup["raw_api"] + data = io.BytesIO(expect_100_setup["file_contents"]) + raw_api.upload_file( + expect_100_setup["upload_url"], + expect_100_setup["upload_auth_token"], + expect_100_setup["file_name"], + expect_100_setup["file_contents"], + 'text/plain', + expect_100_setup["file_sha1"], + {'color': 'blue'}, + data, + server_side_encryption=expect_100_setup["sse_b2_aes"], + expect_100_continue=False, + ) + + assert expect_100_setup["file_contents"] in expect_100_setup["sent_data"] From 1ed88bb69c06a8133a5c120b486a03f14a8c7552 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 16 Aug 2023 00:45:36 +0600 Subject: [PATCH 22/33] Fix formatting --- b2sdk/b2http.py | 2 +- b2sdk/requests/__init__.py | 9 +++++---- test/integration/test_raw_expect_100.py | 11 +++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/b2sdk/b2http.py b/b2sdk/b2http.py index 4d083ce25..b4396db16 100644 --- a/b2sdk/b2http.py +++ b/b2sdk/b2http.py @@ -39,7 +39,7 @@ UnknownHost, interpret_b2_error, ) -from .requests import NotDecompressingResponse, HTTPAdapterWithContinue +from .requests import HTTPAdapterWithContinue, NotDecompressingResponse from .version import USER_AGENT LOCALE_LOCK = threading.Lock() diff --git a/b2sdk/requests/__init__.py b/b2sdk/requests/__init__.py index 318389193..716b0fd4c 100644 --- a/b2sdk/requests/__init__.py +++ b/b2sdk/requests/__init__.py @@ -82,8 +82,9 @@ def from_builtin_response(cls, response: Response): class HTTPAdapterWithContinue(HTTPAdapter): - def init_poolmanager( - self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs - ): + def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs): super().init_poolmanager(connections, maxsize, block, **pool_kwargs) - self.poolmanager.pool_classes_by_scheme = {"http": AWSHTTPConnectionPool, "https": AWSHTTPSConnectionPool} + self.poolmanager.pool_classes_by_scheme = { + "http": AWSHTTPConnectionPool, + "https": AWSHTTPSConnectionPool + } diff --git a/test/integration/test_raw_expect_100.py b/test/integration/test_raw_expect_100.py index 016a748af..00a794965 100644 --- a/test/integration/test_raw_expect_100.py +++ b/test/integration/test_raw_expect_100.py @@ -5,15 +5,14 @@ import secrets import time -from b2sdk.encryption.setting import EncryptionSetting -from b2sdk.encryption.types import EncryptionMode, EncryptionAlgorithm -from b2sdk.exception import InvalidAuthToken -from b2sdk.utils import hex_sha1_of_stream import pytest + from b2sdk.b2http import B2Http +from b2sdk.encryption.setting import EncryptionSetting +from b2sdk.encryption.types import EncryptionAlgorithm, EncryptionMode +from b2sdk.exception import InvalidAuthToken from b2sdk.raw_api import REALM_URLS, B2RawHTTPApi - -from .fixtures import b2_auth_data +from b2sdk.utils import hex_sha1_of_stream @pytest.fixture From 2e53be1ce43f419631632e5a1da3c99058296a24 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 16 Aug 2023 00:46:35 +0600 Subject: [PATCH 23/33] Fix file headers --- b2sdk/_botocore/__init__.py | 9 +++++++++ b2sdk/_botocore/awsrequest.py | 27 +++++++++++++------------ b2sdk/_botocore/included_source_meta.py | 4 ++-- test/integration/test_raw_expect_100.py | 9 +++++++++ 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/b2sdk/_botocore/__init__.py b/b2sdk/_botocore/__init__.py index e69de29bb..dd542af95 100644 --- a/b2sdk/_botocore/__init__.py +++ b/b2sdk/_botocore/__init__.py @@ -0,0 +1,9 @@ +###################################################################### +# +# File: b2sdk/_botocore/__init__.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### diff --git a/b2sdk/_botocore/awsrequest.py b/b2sdk/_botocore/awsrequest.py index 4ea175f9c..75477a146 100644 --- a/b2sdk/_botocore/awsrequest.py +++ b/b2sdk/_botocore/awsrequest.py @@ -1,20 +1,21 @@ -# Code taken from: -# https://github.com/boto/botocore/blob/754b699bbf34261eae47c9dece3b11d7b58eb03c/botocore/awsrequest.py -# The code has been modified to also work with urllib3>=2.0 - +###################################################################### +# +# File: b2sdk/_botocore/awsrequest.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. # Copyright (c) 2012-2013 Mitch Garnaat http://garnaat.org/ # Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. # -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at +# License https://www.backblaze.com/using_b2_code.html +# License Apache License 2.0 (http://www.apache.org/licenses/ and LICENSE file in this directory) # -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. +###################################################################### +"""\ +This module contains modified parts of the botocore module (https://github.com/boto/botocore). +The modules original license is included in LICENSE. +Changes made to the original source are listed in NOTICE, along with original NOTICE. +""" + import functools import logging from http.client import HTTPResponse diff --git a/b2sdk/_botocore/included_source_meta.py b/b2sdk/_botocore/included_source_meta.py index d12b8eb15..27f315d7a 100644 --- a/b2sdk/_botocore/included_source_meta.py +++ b/b2sdk/_botocore/included_source_meta.py @@ -1,8 +1,8 @@ ###################################################################### # -# File: b2sdk/requests/included_source_meta.py +# File: b2sdk/_botocore/included_source_meta.py # -# Copyright 2022 Backblaze Inc. All Rights Reserved. +# Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # diff --git a/test/integration/test_raw_expect_100.py b/test/integration/test_raw_expect_100.py index 00a794965..4bd5762c4 100644 --- a/test/integration/test_raw_expect_100.py +++ b/test/integration/test_raw_expect_100.py @@ -1,3 +1,12 @@ +###################################################################### +# +# File: test/integration/test_raw_expect_100.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### import http.client import io import os From 4478dd29fc4649895c81470a20f96bbcae2dbc23 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 16 Aug 2023 01:30:43 +0600 Subject: [PATCH 24/33] Fix failing test --- test/integration/test_raw_expect_100.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/integration/test_raw_expect_100.py b/test/integration/test_raw_expect_100.py index 4bd5762c4..8d9980718 100644 --- a/test/integration/test_raw_expect_100.py +++ b/test/integration/test_raw_expect_100.py @@ -23,9 +23,11 @@ from b2sdk.raw_api import REALM_URLS, B2RawHTTPApi from b2sdk.utils import hex_sha1_of_stream +from .fixtures import b2_auth_data # noqa + @pytest.fixture -def expect_100_setup(b2_auth_data, monkeypatch): +def expect_100_setup(b2_auth_data, monkeypatch): # noqa application_key_id, application_key = b2_auth_data raw_api = B2RawHTTPApi(B2Http()) realm = os.environ.get('B2_TEST_ENVIRONMENT', 'production') From 28b4d401b85d8f418177fdd9b649913db2cc8b2f Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sat, 2 Sep 2023 23:21:11 +0600 Subject: [PATCH 25/33] Fix typos --- b2sdk/requests/NOTICE | 2 +- b2sdk/requests/included_source_meta.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/b2sdk/requests/NOTICE b/b2sdk/requests/NOTICE index cf7c367fa..c90dc5c78 100644 --- a/b2sdk/requests/NOTICE +++ b/b2sdk/requests/NOTICE @@ -5,4 +5,4 @@ Copyright 2021 Backblaze Inc. Changes made to the original source: * requests.models.Response.iter_content has been overridden to pass `decode_content=False` argument to `self.raw.stream` in order to NOT decompress data based on Content-Encoding header -* requests.adapters.HTTPAdapter has been overriden to use patched Urllib3 connection pools \ No newline at end of file +* requests.adapters.HTTPAdapter has been overridden to use patched Urllib3 connection pools \ No newline at end of file diff --git a/b2sdk/requests/included_source_meta.py b/b2sdk/requests/included_source_meta.py index b47f2b0af..c789d4741 100644 --- a/b2sdk/requests/included_source_meta.py +++ b/b2sdk/requests/included_source_meta.py @@ -19,7 +19,7 @@ Changes made to the original source: * requests.models.Response.iter_content has been overridden to pass `decode_content=False` argument to `self.raw.stream` in order to NOT decompress data based on Content-Encoding header -* requests.adapters.HTTPAdapter has been overriden to use patched Urllib3 connection pools""" +* requests.adapters.HTTPAdapter has been overridden to use patched Urllib3 connection pools""" } ) add_included_source(included_source_meta) From c55f8136383379ace3aed354d66b62f74a61a269 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sun, 3 Sep 2023 00:58:05 +0600 Subject: [PATCH 26/33] Separate fixtures from big expect_100_setup fixture --- test/integration/conftest.py | 64 ++++++++++ test/integration/fixtures/__init__.py | 2 +- test/integration/test_raw_expect_100.py | 153 ++++++++---------------- 3 files changed, 113 insertions(+), 106 deletions(-) diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 210cbaab4..0bb85d3a6 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -9,8 +9,18 @@ ###################################################################### from __future__ import annotations +import http.client +import os +import random +import time + import pytest +from b2sdk.b2http import B2Http +from b2sdk.raw_api import REALM_URLS, B2RawHTTPApi + +from .fixtures import b2_auth_data + def pytest_addoption(parser): """Add a flag for not cleaning up old buckets""" @@ -24,3 +34,57 @@ def pytest_addoption(parser): @pytest.fixture def dont_cleanup_old_buckets(request): return request.config.getoption("--dont-cleanup-old-buckets") + + +@pytest.fixture(scope="session") +def raw_api(): + return B2RawHTTPApi(B2Http()) + + +@pytest.fixture(scope="session") +def auth_dict(raw_api, b2_auth_data): + application_key_id, application_key = b2_auth_data + realm = os.environ.get('B2_TEST_ENVIRONMENT', 'production') + realm_url = REALM_URLS.get(realm, realm) + return raw_api.authorize_account(realm_url, application_key_id, application_key) + + +@pytest.fixture(scope="session") +def bucket_dict(raw_api, auth_dict): + bucket_name = 'test-raw-api-%s-%d-%d' % ( + auth_dict['accountId'], int(time.time()), random.randint(1000, 9999) + ) + + return raw_api.create_bucket( + auth_dict['apiUrl'], + auth_dict['authorizationToken'], + auth_dict['accountId'], + bucket_name, + 'allPublic', + is_file_lock_enabled=True, + ) + + +@pytest.fixture(scope="session") +def upload_url_dict(raw_api, auth_dict, bucket_dict): + return raw_api.get_upload_url( + auth_dict['apiUrl'], auth_dict['authorizationToken'], bucket_dict['bucketId'] + ) + + +@pytest.fixture +def http_sent_data(monkeypatch): + orig_send = http.client.HTTPConnection.send + sent_data = bytearray() + + def patched_send(self, data): + sent_data.extend(data) + return orig_send(self, data) + + monkeypatch.setattr( + http.client.HTTPConnection, + "send", + patched_send, + ) + + return sent_data diff --git a/test/integration/fixtures/__init__.py b/test/integration/fixtures/__init__.py index eaa285700..de886f2ed 100644 --- a/test/integration/fixtures/__init__.py +++ b/test/integration/fixtures/__init__.py @@ -15,7 +15,7 @@ from .. import get_b2_auth_data -@pytest.fixture +@pytest.fixture(scope="session") def b2_auth_data(): try: return get_b2_auth_data() diff --git a/test/integration/test_raw_expect_100.py b/test/integration/test_raw_expect_100.py index 016a748af..9aab706bb 100644 --- a/test/integration/test_raw_expect_100.py +++ b/test/integration/test_raw_expect_100.py @@ -1,141 +1,84 @@ -import http.client import io -import os -import random import secrets -import time + +import pytest from b2sdk.encryption.setting import EncryptionSetting from b2sdk.encryption.types import EncryptionMode, EncryptionAlgorithm from b2sdk.exception import InvalidAuthToken from b2sdk.utils import hex_sha1_of_stream -import pytest -from b2sdk.b2http import B2Http -from b2sdk.raw_api import REALM_URLS, B2RawHTTPApi - -from .fixtures import b2_auth_data - - -@pytest.fixture -def expect_100_setup(b2_auth_data, monkeypatch): - application_key_id, application_key = b2_auth_data - raw_api = B2RawHTTPApi(B2Http()) - realm = os.environ.get('B2_TEST_ENVIRONMENT', 'production') - realm_url = REALM_URLS.get(realm, realm) - auth_dict = raw_api.authorize_account(realm_url, application_key_id, application_key) - - account_id = auth_dict['accountId'] - account_auth_token = auth_dict['authorizationToken'] - api_url = auth_dict['apiUrl'] - - sse_b2_aes = EncryptionSetting( - mode=EncryptionMode.SSE_B2, - algorithm=EncryptionAlgorithm.AES256, - ) - - bucket_name = 'test-raw-api-%s-%d-%d' % ( - account_id, int(time.time()), random.randint(1000, 9999) - ) - - bucket_dict = raw_api.create_bucket( - api_url, - account_auth_token, - account_id, - bucket_name, - 'allPublic', - is_file_lock_enabled=True, - ) - - upload_url_dict = raw_api.get_upload_url(api_url, account_auth_token, bucket_dict['bucketId']) - upload_url = upload_url_dict['uploadUrl'] - upload_auth_token = upload_url_dict['authorizationToken'] - - orig_send = http.client.HTTPConnection.send - sent_data = bytearray() - - def patched_send(self, data): - sent_data.extend(data) - return orig_send(self, data) - monkeypatch.setattr( - http.client.HTTPConnection, - "send", - patched_send, - ) +def test_expect_100_non_100_response(raw_api, upload_url_dict, http_sent_data): file_name = 'test-100-continue.txt' file_contents = secrets.token_bytes() file_length = len(file_contents) file_sha1 = hex_sha1_of_stream(io.BytesIO(file_contents), file_length) + data = io.BytesIO(file_contents) - return { - "raw_api": raw_api, - "upload_url": upload_url, - "upload_auth_token": upload_auth_token, - "sse_b2_aes": sse_b2_aes, - "sent_data": sent_data, - "file_name": file_name, - "file_contents": file_contents, - "file_length": file_length, - "file_sha1": file_sha1, - } - - -@pytest.fixture(autouse=True) -def rest_sent_data(expect_100_setup): - expect_100_setup["sent_data"].clear() - - -def test_expect_100_non_100_response(expect_100_setup): - raw_api = expect_100_setup["raw_api"] - data = io.BytesIO(expect_100_setup["file_contents"]) with pytest.raises(InvalidAuthToken): raw_api.upload_file( - expect_100_setup["upload_url"], - expect_100_setup["upload_auth_token"] + 'wrong token', - expect_100_setup["file_name"], - expect_100_setup["file_contents"], + upload_url_dict['uploadUrl'], + upload_url_dict['authorizationToken'] + 'wrong token', + file_name, + file_contents, 'text/plain', - expect_100_setup["file_sha1"], + file_sha1, {'color': 'blue'}, data, - server_side_encryption=expect_100_setup["sse_b2_aes"], + server_side_encryption=EncryptionSetting( + mode=EncryptionMode.SSE_B2, + algorithm=EncryptionAlgorithm.AES256, + ), ) - assert expect_100_setup["file_contents"] not in expect_100_setup["sent_data"] + assert file_contents not in http_sent_data + +def test_expect_100_timeout(raw_api, upload_url_dict, http_sent_data): + file_name = 'test-100-continue.txt' + file_contents = secrets.token_bytes() + file_length = len(file_contents) + file_sha1 = hex_sha1_of_stream(io.BytesIO(file_contents), file_length) + data = io.BytesIO(file_contents) -def test_expect_100_timeout(expect_100_setup): - raw_api = expect_100_setup["raw_api"] - data = io.BytesIO(expect_100_setup["file_contents"]) raw_api.upload_file( - expect_100_setup["upload_url"], - expect_100_setup["upload_auth_token"], - expect_100_setup["file_name"], - expect_100_setup["file_contents"], + upload_url_dict['uploadUrl'], + upload_url_dict['authorizationToken'], + file_name, + file_contents, 'text/plain', - expect_100_setup["file_sha1"], + file_sha1, {'color': 'blue'}, data, - server_side_encryption=expect_100_setup["sse_b2_aes"], + server_side_encryption=EncryptionSetting( + mode=EncryptionMode.SSE_B2, + algorithm=EncryptionAlgorithm.AES256, + ), expect_100_timeout=0, ) - assert expect_100_setup["file_contents"] in expect_100_setup["sent_data"] + assert file_contents in http_sent_data -def test_expect_100_disabled(expect_100_setup): - raw_api = expect_100_setup["raw_api"] - data = io.BytesIO(expect_100_setup["file_contents"]) +def test_expect_100_disabled(raw_api, upload_url_dict, http_sent_data): + file_name = 'test-100-continue.txt' + file_contents = secrets.token_bytes() + file_length = len(file_contents) + file_sha1 = hex_sha1_of_stream(io.BytesIO(file_contents), file_length) + data = io.BytesIO(file_contents) + raw_api.upload_file( - expect_100_setup["upload_url"], - expect_100_setup["upload_auth_token"], - expect_100_setup["file_name"], - expect_100_setup["file_contents"], + upload_url_dict['uploadUrl'], + upload_url_dict['authorizationToken'], + file_name, + file_contents, 'text/plain', - expect_100_setup["file_sha1"], + file_sha1, {'color': 'blue'}, data, - server_side_encryption=expect_100_setup["sse_b2_aes"], + server_side_encryption=EncryptionSetting( + mode=EncryptionMode.SSE_B2, + algorithm=EncryptionAlgorithm.AES256, + ), expect_100_continue=False, ) - - assert expect_100_setup["file_contents"] in expect_100_setup["sent_data"] + assert file_contents in http_sent_data From 504b3f7a9465224631cc201379eb29637424d807 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sun, 3 Sep 2023 01:20:15 +0600 Subject: [PATCH 27/33] Fix lint issues --- b2sdk/_botocore/__init__.py | 10 ++++++++++ b2sdk/_botocore/awsrequest.py | 21 ++++++++------------- b2sdk/_botocore/included_source_meta.py | 4 ++-- b2sdk/b2http.py | 2 +- b2sdk/requests/__init__.py | 9 +++++---- test/integration/conftest.py | 16 ++++++++++------ test/integration/fixtures/__init__.py | 2 +- test/integration/test_raw_expect_100.py | 11 ++++++++++- 8 files changed, 47 insertions(+), 28 deletions(-) diff --git a/b2sdk/_botocore/__init__.py b/b2sdk/_botocore/__init__.py index e69de29bb..d00a42864 100644 --- a/b2sdk/_botocore/__init__.py +++ b/b2sdk/_botocore/__init__.py @@ -0,0 +1,10 @@ +###################################################################### +# +# File: b2sdk/_botocore/__init__.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# License Apache License 2.0 (http://www.apache.org/licenses/ and LICENSE file in this directory) +# +###################################################################### \ No newline at end of file diff --git a/b2sdk/_botocore/awsrequest.py b/b2sdk/_botocore/awsrequest.py index 4ea175f9c..9029d539c 100644 --- a/b2sdk/_botocore/awsrequest.py +++ b/b2sdk/_botocore/awsrequest.py @@ -1,20 +1,15 @@ -# Code taken from: -# https://github.com/boto/botocore/blob/754b699bbf34261eae47c9dece3b11d7b58eb03c/botocore/awsrequest.py -# The code has been modified to also work with urllib3>=2.0 - +###################################################################### +# +# File: b2sdk/_botocore/awsrequest.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. # Copyright (c) 2012-2013 Mitch Garnaat http://garnaat.org/ # Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. # -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ +# License https://www.backblaze.com/using_b2_code.html +# License Apache License 2.0 (http://www.apache.org/licenses/ and LICENSE file in this directory) # -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. +###################################################################### import functools import logging from http.client import HTTPResponse diff --git a/b2sdk/_botocore/included_source_meta.py b/b2sdk/_botocore/included_source_meta.py index d12b8eb15..27f315d7a 100644 --- a/b2sdk/_botocore/included_source_meta.py +++ b/b2sdk/_botocore/included_source_meta.py @@ -1,8 +1,8 @@ ###################################################################### # -# File: b2sdk/requests/included_source_meta.py +# File: b2sdk/_botocore/included_source_meta.py # -# Copyright 2022 Backblaze Inc. All Rights Reserved. +# Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # diff --git a/b2sdk/b2http.py b/b2sdk/b2http.py index 4d083ce25..b4396db16 100644 --- a/b2sdk/b2http.py +++ b/b2sdk/b2http.py @@ -39,7 +39,7 @@ UnknownHost, interpret_b2_error, ) -from .requests import NotDecompressingResponse, HTTPAdapterWithContinue +from .requests import HTTPAdapterWithContinue, NotDecompressingResponse from .version import USER_AGENT LOCALE_LOCK = threading.Lock() diff --git a/b2sdk/requests/__init__.py b/b2sdk/requests/__init__.py index 318389193..60adfbae3 100644 --- a/b2sdk/requests/__init__.py +++ b/b2sdk/requests/__init__.py @@ -82,8 +82,9 @@ def from_builtin_response(cls, response: Response): class HTTPAdapterWithContinue(HTTPAdapter): - def init_poolmanager( - self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs - ): + def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs): super().init_poolmanager(connections, maxsize, block, **pool_kwargs) - self.poolmanager.pool_classes_by_scheme = {"http": AWSHTTPConnectionPool, "https": AWSHTTPSConnectionPool} + self.poolmanager.pool_classes_by_scheme = { + "http": AWSHTTPConnectionPool, + "https": AWSHTTPSConnectionPool, + } diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 0bb85d3a6..e7f3d7bf2 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -19,7 +19,7 @@ from b2sdk.b2http import B2Http from b2sdk.raw_api import REALM_URLS, B2RawHTTPApi -from .fixtures import b2_auth_data +from . import get_b2_auth_data def pytest_addoption(parser): @@ -42,11 +42,15 @@ def raw_api(): @pytest.fixture(scope="session") -def auth_dict(raw_api, b2_auth_data): - application_key_id, application_key = b2_auth_data - realm = os.environ.get('B2_TEST_ENVIRONMENT', 'production') - realm_url = REALM_URLS.get(realm, realm) - return raw_api.authorize_account(realm_url, application_key_id, application_key) +def auth_dict(raw_api): + try: + application_key_id, application_key = get_b2_auth_data() + except ValueError as ex: + pytest.fail(ex.args[0]) + else: + realm = os.environ.get('B2_TEST_ENVIRONMENT', 'production') + realm_url = REALM_URLS.get(realm, realm) + return raw_api.authorize_account(realm_url, application_key_id, application_key) @pytest.fixture(scope="session") diff --git a/test/integration/fixtures/__init__.py b/test/integration/fixtures/__init__.py index de886f2ed..eaa285700 100644 --- a/test/integration/fixtures/__init__.py +++ b/test/integration/fixtures/__init__.py @@ -15,7 +15,7 @@ from .. import get_b2_auth_data -@pytest.fixture(scope="session") +@pytest.fixture def b2_auth_data(): try: return get_b2_auth_data() diff --git a/test/integration/test_raw_expect_100.py b/test/integration/test_raw_expect_100.py index 9aab706bb..7b2455ba2 100644 --- a/test/integration/test_raw_expect_100.py +++ b/test/integration/test_raw_expect_100.py @@ -1,10 +1,19 @@ +###################################################################### +# +# File: test/integration/test_raw_expect_100.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### import io import secrets import pytest from b2sdk.encryption.setting import EncryptionSetting -from b2sdk.encryption.types import EncryptionMode, EncryptionAlgorithm +from b2sdk.encryption.types import EncryptionAlgorithm, EncryptionMode from b2sdk.exception import InvalidAuthToken from b2sdk.utils import hex_sha1_of_stream From 42bc4ab694036ec6b153c6a4240ab6358751a111 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 15 Sep 2023 17:57:45 +0600 Subject: [PATCH 28/33] add the unit tests from botocore's AWSHTTPConnection --- test/unit/botocore/__init__.py | 10 + test/unit/botocore/test_awsrequest.py | 348 ++++++++++++++++++++++++++ 2 files changed, 358 insertions(+) create mode 100644 test/unit/botocore/__init__.py create mode 100644 test/unit/botocore/test_awsrequest.py diff --git a/test/unit/botocore/__init__.py b/test/unit/botocore/__init__.py new file mode 100644 index 000000000..ca334aed4 --- /dev/null +++ b/test/unit/botocore/__init__.py @@ -0,0 +1,10 @@ +###################################################################### +# +# File: test/unit/botocore/__init__.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +from __future__ import annotations diff --git a/test/unit/botocore/test_awsrequest.py b/test/unit/botocore/test_awsrequest.py new file mode 100644 index 000000000..5006141bd --- /dev/null +++ b/test/unit/botocore/test_awsrequest.py @@ -0,0 +1,348 @@ +###################################################################### +# +# File: test/unit/botocore/test_awsrequest.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# Copyright (c) 2012-2013 Mitch Garnaat http://garnaat.org/ +# Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# See NOTICE and LICENSE files in b2sdk/_botocore directory. +# +###################################################################### +from __future__ import annotations + +import io +import socket +import unittest +from unittest import mock + +from b2sdk._botocore.awsrequest import AWSHTTPConnection + + +class IgnoreCloseBytesIO(io.BytesIO): + def close(self): + pass + + +class FakeSocket: + def __init__(self, read_data, fileclass=IgnoreCloseBytesIO): + self.sent_data = b'' + self.read_data = read_data + self.fileclass = fileclass + self._fp_object = None + + def sendall(self, data): + self.sent_data += data + + def makefile(self, mode, bufsize=None): + if self._fp_object is None: + self._fp_object = self.fileclass(self.read_data) + return self._fp_object + + def close(self): + pass + + def settimeout(self, value): + pass + + +class BytesIOWithLen(io.BytesIO): + def __len__(self): + return len(self.getvalue()) + + +class TestAWSHTTPConnection(unittest.TestCase): + def create_tunneled_connection(self, url, port, response): + s = FakeSocket(response) + conn = AWSHTTPConnection(url, port) + conn.sock = s + conn._tunnel_host = url + conn._tunnel_port = port + conn._tunnel_headers = {'key': 'value'} + + # Create a mock response. + self.mock_response = mock.Mock() + self.mock_response.fp = mock.Mock() + + # Imitate readline function by creating a list to be sent as + # a side effect of the mocked readline to be able to track how the + # response is processed in ``_tunnel()``. + delimiter = b'\r\n' + side_effect = [] + response_components = response.split(delimiter) + for i in range(len(response_components)): + new_component = response_components[i] + # Only add the delimiter on if it is not the last component + # which should be an empty string. + if i != len(response_components) - 1: + new_component += delimiter + side_effect.append(new_component) + + self.mock_response.fp.readline.side_effect = side_effect + + response_components = response.split(b' ') + self.mock_response._read_status.return_value = ( + response_components[0], + int(response_components[1]), + response_components[2], + ) + conn.response_class = mock.Mock() + conn.response_class.return_value = self.mock_response + return conn + + def test_expect_100_continue_returned(self): + with mock.patch('urllib3.util.wait_for_read') as wait_mock: + # Shows the server first sending a 100 continue response + # then a 200 ok response. + s = FakeSocket(b'HTTP/1.1 100 Continue\r\n\r\nHTTP/1.1 200 OK\r\n') + conn = AWSHTTPConnection('s3.amazonaws.com', 443) + conn.sock = s + wait_mock.return_value = True + conn.request('GET', '/bucket/foo', b'body', {'Expect': b'100-continue'}) + response = conn.getresponse() + # Assert that we waited for the 100-continue response + self.assertEqual(wait_mock.call_count, 1) + # Now we should verify that our final response is the 200 OK + self.assertEqual(response.status, 200) + + def test_handles_expect_100_with_different_reason_phrase(self): + with mock.patch('urllib3.util.wait_for_read') as wait_mock: + # Shows the server first sending a 100 continue response + # then a 200 ok response. + s = FakeSocket(b'HTTP/1.1 100 (Continue)\r\n\r\nHTTP/1.1 200 OK\r\n') + conn = AWSHTTPConnection('s3.amazonaws.com', 443) + conn.sock = s + wait_mock.return_value = True + conn.request( + 'GET', + '/bucket/foo', + io.BytesIO(b'body'), + { + 'Expect': b'100-continue', + 'Content-Length': b'4' + }, + ) + response = conn.getresponse() + # Now we should verify that our final response is the 200 OK. + self.assertEqual(response.status, 200) + # Assert that we waited for the 100-continue response + self.assertEqual(wait_mock.call_count, 1) + # Verify that we went the request body because we got a 100 + # continue. + self.assertIn(b'body', s.sent_data) + + def test_expect_100_sends_connection_header(self): + # When using squid as an HTTP proxy, it will also send + # a Connection: keep-alive header back with the 100 continue + # response. We need to ensure we handle this case. + with mock.patch('urllib3.util.wait_for_read') as wait_mock: + # Shows the server first sending a 100 continue response + # then a 500 response. We're picking 500 to confirm we + # actually parse the response instead of getting the + # default status of 200 which happens when we can't parse + # the response. + s = FakeSocket( + b'HTTP/1.1 100 Continue\r\n' + b'Connection: keep-alive\r\n' + b'\r\n' + b'HTTP/1.1 500 Internal Service Error\r\n' + ) + conn = AWSHTTPConnection('s3.amazonaws.com', 443) + conn.sock = s + wait_mock.return_value = True + conn.request('GET', '/bucket/foo', b'body', {'Expect': b'100-continue'}) + # Assert that we waited for the 100-continue response + self.assertEqual(wait_mock.call_count, 1) + response = conn.getresponse() + self.assertEqual(response.status, 500) + + def test_expect_100_continue_sends_307(self): + # This is the case where we send a 100 continue and the server + # immediately sends a 307 + with mock.patch('urllib3.util.wait_for_read') as wait_mock: + # Shows the server first sending a 100 continue response + # then a 200 ok response. + s = FakeSocket( + b'HTTP/1.1 307 Temporary Redirect\r\n' + b'Location: http://example.org\r\n' + ) + conn = AWSHTTPConnection('s3.amazonaws.com', 443) + conn.sock = s + wait_mock.return_value = True + conn.request('GET', '/bucket/foo', b'body', {'Expect': b'100-continue'}) + # Assert that we waited for the 100-continue response + self.assertEqual(wait_mock.call_count, 1) + response = conn.getresponse() + # Now we should verify that our final response is the 307. + self.assertEqual(response.status, 307) + + def test_expect_100_continue_no_response_from_server(self): + with mock.patch('urllib3.util.wait_for_read') as wait_mock: + # Shows the server first sending a 100 continue response + # then a 200 ok response. + s = FakeSocket( + b'HTTP/1.1 307 Temporary Redirect\r\n' + b'Location: http://example.org\r\n' + ) + conn = AWSHTTPConnection('s3.amazonaws.com', 443) + conn.sock = s + # By settings wait_mock to return False, this indicates + # that the server did not send any response. In this situation + # we should just send the request anyways. + wait_mock.return_value = False + conn.request('GET', '/bucket/foo', b'body', {'Expect': b'100-continue'}) + # Assert that we waited for the 100-continue response + self.assertEqual(wait_mock.call_count, 1) + response = conn.getresponse() + self.assertEqual(response.status, 307) + + def test_message_body_is_file_like_object(self): + # Shows the server first sending a 100 continue response + # then a 200 ok response. + body = BytesIOWithLen(b'body contents') + s = FakeSocket(b'HTTP/1.1 200 OK\r\n') + conn = AWSHTTPConnection('s3.amazonaws.com', 443) + conn.sock = s + conn.request('GET', '/bucket/foo', body) + response = conn.getresponse() + self.assertEqual(response.status, 200) + + def test_no_expect_header_set(self): + # Shows the server first sending a 100 continue response + # then a 200 ok response. + s = FakeSocket(b'HTTP/1.1 200 OK\r\n') + conn = AWSHTTPConnection('s3.amazonaws.com', 443) + conn.sock = s + conn.request('GET', '/bucket/foo', b'body') + response = conn.getresponse() + self.assertEqual(response.status, 200) + + def test_tunnel_readline_none_bugfix(self): + # Tests whether ``_tunnel`` function is able to work around the + # py26 bug of avoiding infinite while loop if nothing is returned. + conn = self.create_tunneled_connection( + url='s3.amazonaws.com', + port=443, + response=b'HTTP/1.1 200 OK\r\n', + ) + conn._tunnel() + # Ensure proper amount of readline calls were made. + self.assertEqual(self.mock_response.fp.readline.call_count, 2) + + def test_tunnel_readline_normal(self): + # Tests that ``_tunnel`` function behaves normally when it comes + # across the usual http ending. + conn = self.create_tunneled_connection( + url='s3.amazonaws.com', + port=443, + response=b'HTTP/1.1 200 OK\r\n\r\n', + ) + conn._tunnel() + # Ensure proper amount of readline calls were made. + self.assertEqual(self.mock_response.fp.readline.call_count, 2) + + def test_tunnel_raises_socket_error(self): + # Tests that ``_tunnel`` function throws appropriate error when + # not 200 status. + conn = self.create_tunneled_connection( + url='s3.amazonaws.com', + port=443, + response=b'HTTP/1.1 404 Not Found\r\n\r\n', + ) + with self.assertRaises(socket.error): + conn._tunnel() + + def test_tunnel_uses_std_lib(self): + s = FakeSocket(b'HTTP/1.1 200 OK\r\n') + conn = AWSHTTPConnection('s3.amazonaws.com', 443) + conn.sock = s + # Test that the standard library method was used by patching out + # the ``_tunnel`` method and seeing if the std lib method was called. + with mock.patch('urllib3.connection.HTTPConnection._tunnel') as mock_tunnel: + conn._tunnel() + self.assertTrue(mock_tunnel.called) + + def test_encodes_unicode_method_line(self): + s = FakeSocket(b'HTTP/1.1 200 OK\r\n') + conn = AWSHTTPConnection('s3.amazonaws.com', 443) + conn.sock = s + # Note the combination of unicode 'GET' and + # bytes 'Utf8-Header' value. + conn.request( + 'GET', + '/bucket/foo', + b'body', + headers={"Utf8-Header": b"\xe5\xb0\x8f"}, + ) + response = conn.getresponse() + self.assertEqual(response.status, 200) + + def test_state_reset_on_connection_close(self): + # This simulates what urllib3 does with connections + # in its connection pool logic. + with mock.patch('urllib3.util.wait_for_read') as wait_mock: + # First fast fail with a 500 response when we first + # send the expect header. + s = FakeSocket(b'HTTP/1.1 500 Internal Server Error\r\n') + conn = AWSHTTPConnection('s3.amazonaws.com', 443) + conn.sock = s + wait_mock.return_value = True + + conn.request('GET', '/bucket/foo', b'body', {'Expect': b'100-continue'}) + self.assertEqual(wait_mock.call_count, 1) + response = conn.getresponse() + self.assertEqual(response.status, 500) + + # Now what happens in urllib3 is that when the next + # request comes along and this connection gets checked + # out. We see that the connection needs to be + # reset. So first the connection is closed. + conn.close() + + # And then a new connection is established. + new_conn = FakeSocket(b'HTTP/1.1 100 (Continue)\r\n\r\nHTTP/1.1 200 OK\r\n') + conn.sock = new_conn + + # And we make a request, we should see the 200 response + # that was sent back. + wait_mock.return_value = True + + conn.request('GET', '/bucket/foo', b'body', {'Expect': b'100-continue'}) + # Assert that we waited for the 100-continue response + self.assertEqual(wait_mock.call_count, 2) + response = conn.getresponse() + # This should be 200. If it's a 500 then + # the prior response was leaking into our + # current response., + self.assertEqual(response.status, 200) + + def test_handles_expect_100_with_no_reason_phrase(self): + with mock.patch('urllib3.util.wait_for_read') as wait_mock: + # Shows the server first sending a 100 continue response + # then a 200 ok response. + s = FakeSocket(b'HTTP/1.1 100\r\n\r\nHTTP/1.1 200 OK\r\n') + conn = AWSHTTPConnection('s3.amazonaws.com', 443) + conn.sock = s + wait_mock.return_value = True + conn.request( + 'GET', + '/bucket/foo', + io.BytesIO(b'body'), + { + 'Expect': b'100-continue', + 'Content-Length': b'4' + }, + ) + response = conn.getresponse() + # Now we should verify that our final response is the 200 OK. + self.assertEqual(response.status, 200) + # Assert that we waited for the 100-continue response + self.assertEqual(wait_mock.call_count, 1) + # Verify that we went the request body because we got a 100 + # continue. + self.assertIn(b'body', s.sent_data) + + +if __name__ == "__main__": + unittest.main() From 5819641b4df07735c32e5dd4e49b81f8f5568f06 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 15 Sep 2023 18:11:12 +0600 Subject: [PATCH 29/33] Update botocore notice about its unittest --- b2sdk/_botocore/NOTICE | 1 + b2sdk/_botocore/included_source_meta.py | 1 + 2 files changed, 2 insertions(+) diff --git a/b2sdk/_botocore/NOTICE b/b2sdk/_botocore/NOTICE index 7cd30d18a..57420250c 100644 --- a/b2sdk/_botocore/NOTICE +++ b/b2sdk/_botocore/NOTICE @@ -8,6 +8,7 @@ Changes made to the original source: * Updated botocore.awsrequest.request method to work with str/byte header values (urllib 1.x vs 2.x) * Updated botocore.awsrequest._handle_expect_response method to work with b2 responses * Updated botocore.awsrequest._send_output method to change 100 response timeout +* Add a new test test_handles_expect_100_with_no_reason_phrase to TestAWSHTTPConnection test class --- diff --git a/b2sdk/_botocore/included_source_meta.py b/b2sdk/_botocore/included_source_meta.py index 27f315d7a..dae8e4573 100644 --- a/b2sdk/_botocore/included_source_meta.py +++ b/b2sdk/_botocore/included_source_meta.py @@ -22,6 +22,7 @@ * Updated botocore.awsrequest.request method to work with str/byte header values (urllib 1.x vs 2.x) * Updated botocore.awsrequest._handle_expect_response method to work with b2 responses * Updated botocore.awsrequest._send_output method to change 100 response timeout +* Add a new test test_handles_expect_100_with_no_reason_phrase to TestAWSHTTPConnection test class --- From b54bb3aeaced0ceaafed2a4a5c09dc9196ecaa20 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Tue, 19 Sep 2023 00:53:39 +0600 Subject: [PATCH 30/33] check for waiting in 100 continue integration tests --- test/integration/test_raw_expect_100.py | 78 +++++++++++++++---------- 1 file changed, 47 insertions(+), 31 deletions(-) diff --git a/test/integration/test_raw_expect_100.py b/test/integration/test_raw_expect_100.py index 7b2455ba2..b2308e34a 100644 --- a/test/integration/test_raw_expect_100.py +++ b/test/integration/test_raw_expect_100.py @@ -9,8 +9,10 @@ ###################################################################### import io import secrets +from unittest import mock import pytest +from urllib3.util import wait_for_read from b2sdk.encryption.setting import EncryptionSetting from b2sdk.encryption.types import EncryptionAlgorithm, EncryptionMode @@ -25,7 +27,9 @@ def test_expect_100_non_100_response(raw_api, upload_url_dict, http_sent_data): file_sha1 = hex_sha1_of_stream(io.BytesIO(file_contents), file_length) data = io.BytesIO(file_contents) - with pytest.raises(InvalidAuthToken): + with pytest.raises(InvalidAuthToken), mock.patch( + "urllib3.util.wait_for_read", side_effect=wait_for_read + ) as wait_mock: raw_api.upload_file( upload_url_dict['uploadUrl'], upload_url_dict['authorizationToken'] + 'wrong token', @@ -41,6 +45,7 @@ def test_expect_100_non_100_response(raw_api, upload_url_dict, http_sent_data): ), ) assert file_contents not in http_sent_data + assert wait_mock.call_count == 1 def test_expect_100_timeout(raw_api, upload_url_dict, http_sent_data): @@ -49,23 +54,30 @@ def test_expect_100_timeout(raw_api, upload_url_dict, http_sent_data): file_length = len(file_contents) file_sha1 = hex_sha1_of_stream(io.BytesIO(file_contents), file_length) data = io.BytesIO(file_contents) + timeout = 0 - raw_api.upload_file( - upload_url_dict['uploadUrl'], - upload_url_dict['authorizationToken'], - file_name, - file_contents, - 'text/plain', - file_sha1, - {'color': 'blue'}, - data, - server_side_encryption=EncryptionSetting( - mode=EncryptionMode.SSE_B2, - algorithm=EncryptionAlgorithm.AES256, - ), - expect_100_timeout=0, - ) + with mock.patch( + "urllib3.util.wait_for_read", side_effect=wait_for_read + ) as wait_mock: + raw_api.upload_file( + upload_url_dict['uploadUrl'], + upload_url_dict['authorizationToken'], + file_name, + file_contents, + 'text/plain', + file_sha1, + {'color': 'blue'}, + data, + server_side_encryption=EncryptionSetting( + mode=EncryptionMode.SSE_B2, + algorithm=EncryptionAlgorithm.AES256, + ), + expect_100_timeout=timeout, + ) assert file_contents in http_sent_data + assert wait_mock.call_count == 1 + args, _ = wait_mock.call_args + assert args[1] == timeout def test_expect_100_disabled(raw_api, upload_url_dict, http_sent_data): @@ -75,19 +87,23 @@ def test_expect_100_disabled(raw_api, upload_url_dict, http_sent_data): file_sha1 = hex_sha1_of_stream(io.BytesIO(file_contents), file_length) data = io.BytesIO(file_contents) - raw_api.upload_file( - upload_url_dict['uploadUrl'], - upload_url_dict['authorizationToken'], - file_name, - file_contents, - 'text/plain', - file_sha1, - {'color': 'blue'}, - data, - server_side_encryption=EncryptionSetting( - mode=EncryptionMode.SSE_B2, - algorithm=EncryptionAlgorithm.AES256, - ), - expect_100_continue=False, - ) + with mock.patch( + "urllib3.util.wait_for_read", side_effect=wait_for_read + ) as wait_mock: + raw_api.upload_file( + upload_url_dict['uploadUrl'], + upload_url_dict['authorizationToken'], + file_name, + file_contents, + 'text/plain', + file_sha1, + {'color': 'blue'}, + data, + server_side_encryption=EncryptionSetting( + mode=EncryptionMode.SSE_B2, + algorithm=EncryptionAlgorithm.AES256, + ), + expect_100_continue=False, + ) assert file_contents in http_sent_data + assert wait_mock.call_count == 0 From 58662627955dba7939caad424d8414d929e60c67 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Tue, 19 Sep 2023 01:08:06 +0600 Subject: [PATCH 31/33] add test to verify data is sent AFTER waiting for 100 Continue response --- test/integration/test_raw_expect_100.py | 40 +++++++++++++++++++++---- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/test/integration/test_raw_expect_100.py b/test/integration/test_raw_expect_100.py index b2308e34a..c8fc1a2e0 100644 --- a/test/integration/test_raw_expect_100.py +++ b/test/integration/test_raw_expect_100.py @@ -56,9 +56,7 @@ def test_expect_100_timeout(raw_api, upload_url_dict, http_sent_data): data = io.BytesIO(file_contents) timeout = 0 - with mock.patch( - "urllib3.util.wait_for_read", side_effect=wait_for_read - ) as wait_mock: + with mock.patch("urllib3.util.wait_for_read", side_effect=wait_for_read) as wait_mock: raw_api.upload_file( upload_url_dict['uploadUrl'], upload_url_dict['authorizationToken'], @@ -87,9 +85,7 @@ def test_expect_100_disabled(raw_api, upload_url_dict, http_sent_data): file_sha1 = hex_sha1_of_stream(io.BytesIO(file_contents), file_length) data = io.BytesIO(file_contents) - with mock.patch( - "urllib3.util.wait_for_read", side_effect=wait_for_read - ) as wait_mock: + with mock.patch("urllib3.util.wait_for_read", side_effect=wait_for_read) as wait_mock: raw_api.upload_file( upload_url_dict['uploadUrl'], upload_url_dict['authorizationToken'], @@ -107,3 +103,35 @@ def test_expect_100_disabled(raw_api, upload_url_dict, http_sent_data): ) assert file_contents in http_sent_data assert wait_mock.call_count == 0 + + +def test_expect_100_data_sent_after_wait(raw_api, upload_url_dict, http_sent_data): + file_name = 'test-100-continue.txt' + file_contents = secrets.token_bytes() + file_length = len(file_contents) + file_sha1 = hex_sha1_of_stream(io.BytesIO(file_contents), file_length) + data = io.BytesIO(file_contents) + + def patched_wait(*args, **kwargs): + # verify that, data is not sent before waiting + assert file_contents not in http_sent_data, "data sent before waiting for 100 Continue" + wait_for_read(*args, **kwargs) + + with mock.patch("urllib3.util.wait_for_read", side_effect=patched_wait) as wait_mock: + raw_api.upload_file( + upload_url_dict['uploadUrl'], + upload_url_dict['authorizationToken'], + file_name, + file_contents, + 'text/plain', + file_sha1, + {'color': 'blue'}, + data, + server_side_encryption=EncryptionSetting( + mode=EncryptionMode.SSE_B2, + algorithm=EncryptionAlgorithm.AES256, + ), + ) + assert file_contents in http_sent_data + assert wait_mock.call_count == 1 + args, _ = wait_mock.call_args From 2f337cfbef1a6a711b38774b271ac8711a8578b0 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Tue, 19 Sep 2023 01:16:48 +0600 Subject: [PATCH 32/33] fix patched wait_for_read in test --- test/integration/test_raw_expect_100.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/test_raw_expect_100.py b/test/integration/test_raw_expect_100.py index c8fc1a2e0..680b9db82 100644 --- a/test/integration/test_raw_expect_100.py +++ b/test/integration/test_raw_expect_100.py @@ -115,7 +115,7 @@ def test_expect_100_data_sent_after_wait(raw_api, upload_url_dict, http_sent_dat def patched_wait(*args, **kwargs): # verify that, data is not sent before waiting assert file_contents not in http_sent_data, "data sent before waiting for 100 Continue" - wait_for_read(*args, **kwargs) + return wait_for_read(*args, **kwargs) with mock.patch("urllib3.util.wait_for_read", side_effect=patched_wait) as wait_mock: raw_api.upload_file( From abde2c1cd4b3890cc1d0433460cc340862fe906d Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Tue, 19 Sep 2023 01:53:47 +0600 Subject: [PATCH 33/33] remove unnecessary tests for 100 continue --- test/integration/test_raw_expect_100.py | 1 - test/unit/botocore/test_awsrequest.py | 16 ---------------- 2 files changed, 17 deletions(-) diff --git a/test/integration/test_raw_expect_100.py b/test/integration/test_raw_expect_100.py index 680b9db82..c5e751b0f 100644 --- a/test/integration/test_raw_expect_100.py +++ b/test/integration/test_raw_expect_100.py @@ -134,4 +134,3 @@ def patched_wait(*args, **kwargs): ) assert file_contents in http_sent_data assert wait_mock.call_count == 1 - args, _ = wait_mock.call_args diff --git a/test/unit/botocore/test_awsrequest.py b/test/unit/botocore/test_awsrequest.py index 5006141bd..a80990f6a 100644 --- a/test/unit/botocore/test_awsrequest.py +++ b/test/unit/botocore/test_awsrequest.py @@ -218,18 +218,6 @@ def test_no_expect_header_set(self): response = conn.getresponse() self.assertEqual(response.status, 200) - def test_tunnel_readline_none_bugfix(self): - # Tests whether ``_tunnel`` function is able to work around the - # py26 bug of avoiding infinite while loop if nothing is returned. - conn = self.create_tunneled_connection( - url='s3.amazonaws.com', - port=443, - response=b'HTTP/1.1 200 OK\r\n', - ) - conn._tunnel() - # Ensure proper amount of readline calls were made. - self.assertEqual(self.mock_response.fp.readline.call_count, 2) - def test_tunnel_readline_normal(self): # Tests that ``_tunnel`` function behaves normally when it comes # across the usual http ending. @@ -342,7 +330,3 @@ def test_handles_expect_100_with_no_reason_phrase(self): # Verify that we went the request body because we got a 100 # continue. self.assertIn(b'body', s.sent_data) - - -if __name__ == "__main__": - unittest.main()