From 1497c8bf86a1468e17c4d8f369358b0b33598aae Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Wed, 16 Oct 2024 18:05:23 -0500 Subject: [PATCH] Replace requests-auth-aws-sigv4 with boto3 signing Signed-off-by: Andre Kurait --- .../migrationConsole/console_api/Pipfile.lock | 8 -- .../migrationConsole/lib/console_link/Pipfile | 1 - .../lib/console_link/Pipfile.lock | 11 +- .../console_link/models/cluster.py | 8 +- .../console_link/console_link/models/utils.py | 33 ++++- .../lib/console_link/setup.py | 3 +- .../lib/console_link/tests/test_cluster.py | 129 +++++++++++++++++- .../lib/integ_test/Pipfile.lock | 8 -- 8 files changed, 159 insertions(+), 42 deletions(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_api/Pipfile.lock b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_api/Pipfile.lock index f64f5a10b..263ab8c81 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_api/Pipfile.lock +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_api/Pipfile.lock @@ -471,14 +471,6 @@ "markers": "python_version >= '3.8'", "version": "==2.32.3" }, - "requests-auth-aws-sigv4": { - "hashes": [ - "sha256:1f6c7f63a0696a8f131a2ff21a544380f43c11f54d72600f6f2a1d402bd41d41", - "sha256:3d2a475cccbf85d4c93b8bd052d072e5c3f8e77022fd621b69a5b11ac2c139c8" - ], - "markers": "python_version >= '2.7'", - "version": "==0.7" - }, "s3transfer": { "hashes": [ "sha256:0711534e9356d3cc692fdde846b4a1e4b0cb6519971860796e6bc4c7aea00ef6", diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/Pipfile b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/Pipfile index cf5dc3003..91c833799 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/Pipfile +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/Pipfile @@ -10,7 +10,6 @@ pyyaml = "*" Click = "*" cerberus = "*" awscli = "*" -requests-auth-aws-sigv4 = ">=0.7" [dev-packages] pytest = "*" diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/Pipfile.lock b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/Pipfile.lock index 132f495ea..370f379bf 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/Pipfile.lock +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "78bd6c8b8764c6d11ab77832542313282b29d29bcc577a23167049253fcc0f54" + "sha256": "732106b5c0463690ea660a9021d614109dc94fb9882d1474fbfef5f892fe6a8a" }, "pipfile-spec": 6, "requires": { @@ -280,15 +280,6 @@ "markers": "python_version >= '3.8'", "version": "==2.32.3" }, - "requests-auth-aws-sigv4": { - "hashes": [ - "sha256:1f6c7f63a0696a8f131a2ff21a544380f43c11f54d72600f6f2a1d402bd41d41", - "sha256:3d2a475cccbf85d4c93b8bd052d072e5c3f8e77022fd621b69a5b11ac2c139c8" - ], - "index": "pypi", - "markers": "python_version >= '2.7'", - "version": "==0.7" - }, "rsa": { "hashes": [ "sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2", diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/cluster.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/cluster.py index 32bc46531..1d6856be0 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/cluster.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/cluster.py @@ -8,11 +8,9 @@ import requests import requests.auth from requests.auth import HTTPBasicAuth -from requests_auth_aws_sigv4 import AWSSigV4 - from console_link.models.client_options import ClientOptions from console_link.models.schema_tools import contains_one_of -from console_link.models.utils import create_boto3_client, append_user_agent_header_for_requests +from console_link.models.utils import SigV4AuthPlugin, create_boto3_client, append_user_agent_header_for_requests requests.packages.urllib3.disable_warnings() # ignore: type @@ -136,8 +134,8 @@ def _generate_auth_object(self) -> requests.auth.AuthBase | None: password ) elif self.auth_type == AuthMethod.SIGV4: - sigv4_details = self._get_sigv4_details() - return AWSSigV4(sigv4_details[0], region=sigv4_details[1]) + service_name, region_name = self._get_sigv4_details(force_region=True) + return SigV4AuthPlugin(service_name, region_name) elif self.auth_type is AuthMethod.NO_AUTH: return None raise NotImplementedError(f"Auth type {self.auth_type} not implemented") diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/utils.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/utils.py index a78614248..8dffe109a 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/utils.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/utils.py @@ -4,6 +4,10 @@ from datetime import datetime import boto3 import requests.utils +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest +from requests.models import PreparedRequest + from console_link.models.client_options import ClientOptions @@ -15,8 +19,8 @@ def __init__(self, message, status_code=None): def raise_for_aws_api_error(response: Dict) -> None: if ( - "ResponseMetadata" in response - and "HTTPStatusCode" in response["ResponseMetadata"] # noqa: W503 + "ResponseMetadata" in response and + "HTTPStatusCode" in response["ResponseMetadata"] # noqa: W503 ): status_code = response["ResponseMetadata"]["HTTPStatusCode"] else: @@ -54,3 +58,28 @@ def append_user_agent_header_for_requests(headers: Optional[dict], user_agent_ex else: adjusted_headers["User-Agent"] = f"{requests.utils.default_user_agent()} {user_agent_extra}" return adjusted_headers + +# The SigV4AuthPlugin allows us to use boto3 with the requests library by integrating +# AWS Signature Version 4 signing. This enables the requests library to authenticate +# requests to AWS services using SigV4. + + +class SigV4AuthPlugin(requests.auth.AuthBase): + def __init__(self, service, region): + self.service = service + self.region = region + session = boto3.Session() + self.credentials = session.get_credentials() + + def __call__(self, r: PreparedRequest) -> PreparedRequest: + # Exclude signing headers that may change after signing + default_headers = requests.utils.default_headers() + excluded_headers = default_headers.keys() + filtered_headers = {k: v for k, v in r.headers.items() if k.lower() not in excluded_headers} + aws_request = AWSRequest(method=r.method, url=r.url, data=r.body, headers=filtered_headers) + signer = SigV4Auth(self.credentials, self.service, self.region) + if aws_request.body is not None: + aws_request.headers['x-amz-content-sha256'] = signer.payload(aws_request) + signer.add_auth(aws_request) + r.headers.update(dict(aws_request.headers)) + return r diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/setup.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/setup.py index 68077fca9..1606c3295 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/setup.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/setup.py @@ -5,8 +5,7 @@ version="1.0.0", description="A Python module to create a console application from a Python script", packages=find_packages(exclude=("tests")), - install_requires=["requests", "boto3", "pyyaml", "Click", "cerberus", - "requests-auth-aws-sigv4"], + install_requires=["requests", "boto3", "pyyaml", "Click", "cerberus"], entry_points={ "console_scripts": [ "console = console_link.cli:cli", diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_cluster.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_cluster.py index ef0f8e46e..7dc375f07 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_cluster.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_cluster.py @@ -1,14 +1,19 @@ -from base64 import b64encode +import boto3 +import console_link.middleware.clusters as clusters_ +import hashlib import os - import pytest -from moto import mock_aws -import boto3 +import re +import json -import console_link.middleware.clusters as clusters_ +from base64 import b64encode +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest from console_link.models.client_options import ClientOptions -from console_link.models.cluster import AuthMethod, Cluster +from console_link.models.cluster import AuthMethod, Cluster, HttpMethod +from moto import mock_aws from tests.utils import create_valid_cluster +import requests @pytest.fixture(scope="function") @@ -383,3 +388,115 @@ def test_run_benchmark_executes_correctly_basic_auth_and_https(mocker): "--client-options=verify_certs:false,use_ssl:true," f"basic_auth_user:{auth_details['username']}," f"basic_auth_password:{auth_details['password']}", shell=True) + + +@pytest.mark.parametrize("method, endpoint, data, has_body", [ + (HttpMethod.GET, "/_cluster/health", None, False), + (HttpMethod.POST, "/_search", {"query": {"match_all": {}}}, True) +]) +def test_sigv4_authentication_signature(requests_mock, method, endpoint, data, has_body): + # Set up a valid cluster configuration with SIGV4 authentication + sigv4_cluster_config = { + "endpoint": "https://opensearchtarget:9200", + "allow_insecure": True, + "sigv4": { + "region": "us-east-1", + "service": "es" + } + } + cluster = Cluster(sigv4_cluster_config) + + # Prepare the mocked API response + url = f"{cluster.endpoint}{endpoint}" + if method == HttpMethod.GET: + requests_mock.get(url, json={'status': 'green'}) + elif method == HttpMethod.POST: + requests_mock.post(url, json={'hits': {'total': 0, 'hits': []}}) + + with mock_aws(): + # Add default headers to the request + headers = { + # These headers are excluded from signing since they are in default request headers + 'User-Agent': 'my-test-agent', + 'Accept': 'application/json', + 'Accept-Encoding': 'gzip, deflate', + 'Connection': 'keep-alive', + # Custom headers to be included in the request + 'X-Custom-Header-1': 'CustomValue1', + 'X-Custom-Header-2': 'CustomValue2' + } + if data is not None: + response = cluster.call_api(endpoint, method=method, data=json.dumps(data), headers=headers) + else: + response = cluster.call_api(endpoint, method=method, headers=headers) + assert response.status_code == 200 + + # Retrieve the last request made + last_request = requests_mock.last_request + + # Verify the default headers are present in the request headers + for header_name, header_value in headers.items(): + assert last_request.headers.get(header_name) == header_value + + # Verify the Authorization header + auth_header = last_request.headers.get('Authorization') + assert auth_header is not None, "Authorization header is missing" + assert auth_header.startswith("AWS4-HMAC-SHA256"), "Incorrect Authorization header format" + + # Extract SignedHeaders and Signature from the Authorization header + signed_headers_match = re.search(r"SignedHeaders=([^,]+)", auth_header) + signature_match = re.search(r"Signature=([a-f0-9]+)", auth_header) + assert signed_headers_match is not None, "SignedHeaders not found in Authorization header" + assert signature_match is not None, "Signature not found in Authorization header" + + # Verify that default headers are not included in SignedHeaders + signed_headers = signed_headers_match.group(1).split(';') + default_headers = [header.lower() for header in headers.keys() if header.lower() + in requests.utils.default_headers().keys()] + assert len(default_headers) > 0, "Default headers should contain at least one header" + for header in default_headers: + assert header not in signed_headers, f"Default header '{header}' should not be included in SignedHeaders" + + # Verify that essential headers are included in SignedHeaders + required_headers = ['host', 'x-amz-date', 'x-custom-header-1', 'x-custom-header-2'] + for header in required_headers: + assert header in signed_headers, f"Header '{header}' not found in SignedHeaders, actual headers: {signed_headers}" + + # Check that the x-amz-date header is present + amz_date_header = last_request.headers.get('x-amz-date') + assert amz_date_header is not None, "x-amz-date header is missing" + + if has_body: + # Verify that the 'x-amz-content-sha256' header is present + content_sha256 = last_request.headers.get('x-amz-content-sha256') + assert content_sha256 is not None, 'x-amz-content-sha256 header is missing' + # Compute the SHA256 hash of the body + body_hash = hashlib.sha256(last_request.body.encode('utf-8')).hexdigest() + assert content_sha256 == body_hash, "x-amz-content-sha256 does not match body hash" + + # Re-sign the request using botocore to verify the signature + session = boto3.Session() + credentials = session.get_credentials() + service_name = cluster.auth_details.get("service", "es") + region_name = cluster.auth_details.get("region", "us-east-1") + # Create a new AWSRequest + aws_request = AWSRequest( + method=last_request.method, + url=last_request.url, + data=last_request.body, + headers={k: v for k, v in last_request.headers.items() if k.lower() not in requests.utils.default_headers().keys()} + ) + # Sign the request + SigV4Auth(credentials, service_name, region_name).add_auth(aws_request) + + # Extract the new signature + new_auth_header = aws_request.headers.get('Authorization') + assert new_auth_header is not None, "Failed to generate new Authorization header" + + # Compare signatures + original_signature = signature_match.group(1) + new_signature_match = re.search(r"Signature=([a-f0-9]+)", new_auth_header) + assert new_signature_match is not None, "New signature not found in Authorization header" + new_signature = new_signature_match.group(1) + + assert original_signature == new_signature, "Signatures do not match" diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/Pipfile.lock b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/Pipfile.lock index 5c11fe6c1..4b4663fd3 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/Pipfile.lock +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/Pipfile.lock @@ -312,14 +312,6 @@ "markers": "python_version >= '3.8'", "version": "==2.32.3" }, - "requests-auth-aws-sigv4": { - "hashes": [ - "sha256:1f6c7f63a0696a8f131a2ff21a544380f43c11f54d72600f6f2a1d402bd41d41", - "sha256:3d2a475cccbf85d4c93b8bd052d072e5c3f8e77022fd621b69a5b11ac2c139c8" - ], - "markers": "python_version >= '2.7'", - "version": "==0.7" - }, "requests-aws4auth": { "hashes": [ "sha256:2969b5379ae6e60ee666638caf6cb94a32d67033f6bfcf0d50c95cd5474f2419",