diff --git a/pyproject.toml b/pyproject.toml index d983b3b2..efb6f1d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,9 @@ classifiers = [ # Applications that consume this library should be the ones that are more strictly # limiting dependencies if they want/need to. dependencies = [ - "boto3 >= 1.34.75", + # See CR platform and Python version support here: https://pypi.org/project/awscrt/#files + "boto3[crt] >= 1.34.75; python_version >= '3.7' and python_version <= '3.11'", + "boto3 >= 1.34.75; python_version < '3.7' or python_version > '3.11'", "click >= 8.1.7", "pyyaml >= 6.0", # Job Attachments diff --git a/src/deadline/job_attachments/_aws/aws_clients.py b/src/deadline/job_attachments/_aws/aws_clients.py index 4a603216..4cbec8db 100644 --- a/src/deadline/job_attachments/_aws/aws_clients.py +++ b/src/deadline/job_attachments/_aws/aws_clients.py @@ -7,8 +7,9 @@ from typing import Optional import boto3 +from boto3.s3.transfer import create_crt_transfer_manager, create_transfer_manager + import botocore -from boto3.s3.transfer import create_transfer_manager from botocore.client import BaseClient, Config from deadline.client.config import config_file @@ -104,6 +105,13 @@ def get_s3_max_pool_connections() -> int: @lru_cache(maxsize=MAX_SIZE_CACHE) def get_s3_transfer_manager(s3_client: BaseClient): transfer_config = boto3.s3.transfer.TransferConfig() + + crt_transfer_manager = create_crt_transfer_manager(client=s3_client, config=transfer_config) + if crt_transfer_manager: + return crt_transfer_manager + + # Fallback to regular transfer manager if CRT transfer manager does not support the configuration, which can happen if the client + # and bucket are in different regions. return create_transfer_manager(client=s3_client, config=transfer_config) diff --git a/test/unit/deadline_job_attachments/test_asset_sync.py b/test/unit/deadline_job_attachments/test_asset_sync.py index 7a2ab0d0..7a82a13c 100644 --- a/test/unit/deadline_job_attachments/test_asset_sync.py +++ b/test/unit/deadline_job_attachments/test_asset_sync.py @@ -44,6 +44,10 @@ from ..conftest import is_windows_non_admin +@patch( + f"{deadline.__package__}.job_attachments._aws.aws_clients.create_crt_transfer_manager", + MagicMock(return_value=None), +) class TestAssetSync: @pytest.fixture(autouse=True) def before_test( diff --git a/test/unit/deadline_job_attachments/test_download.py b/test/unit/deadline_job_attachments/test_download.py index 9eee0943..abf1766e 100644 --- a/test/unit/deadline_job_attachments/test_download.py +++ b/test/unit/deadline_job_attachments/test_download.py @@ -566,6 +566,10 @@ def assert_get_job_input_output_paths_by_asset_root( @pytest.mark.docker @pytest.mark.parametrize("manifest_version", [ManifestVersion.v2023_03_03]) +@patch( + f"{deadline.__package__}.job_attachments._aws.aws_clients.create_crt_transfer_manager", + MagicMock(return_value=None), +) class TestFullDownload: """ Tests for downloads from cas. @@ -1965,6 +1969,10 @@ def test_download_file_error_message_on_timeout(self): @pytest.mark.parametrize("manifest_version", [ManifestVersion.v2023_03_03]) +@patch( + f"{deadline.__package__}.job_attachments._aws.aws_clients.create_crt_transfer_manager", + MagicMock(return_value=None), +) class TestFullDownloadPrefixesWithSlashes: """ Tests for downloads from cas when the queue prefixes are created. diff --git a/test/unit/deadline_job_attachments/test_upload.py b/test/unit/deadline_job_attachments/test_upload.py index ac8d4ac6..fcd18b8b 100644 --- a/test/unit/deadline_job_attachments/test_upload.py +++ b/test/unit/deadline_job_attachments/test_upload.py @@ -57,6 +57,10 @@ from ..conftest import is_windows_non_admin +@patch( + f"{deadline.__package__}.job_attachments._aws.aws_clients.create_crt_transfer_manager", + MagicMock(return_value=None), +) class TestUpload: """ Tests for handling uploading assets.