feat: use AWS CRT for faster transfers (aws-deadline#319)

* feat: support AWS CRT for faster transfers

Signed-off-by: Stephen Crowe <crowest@amazon.com>

pyproject.toml

@@ -35,7 +35,9 @@ classifiers = [
 # Applications that consume this library should be the ones that are more strictly
 # limiting dependencies if they want/need to.
 dependencies = [
-    "boto3 >= 1.34.75",
+    # See CR platform and Python version support here: https://pypi.org/project/awscrt/#files
+    "boto3[crt] >= 1.34.75; python_version >= '3.7' and python_version <= '3.11'",
+    "boto3 >= 1.34.75; python_version < '3.7' or python_version > '3.11'",
     "click >= 8.1.7",
     "pyyaml >= 6.0",
     # Job Attachments

src/deadline/job_attachments/_aws/aws_clients.py

@@ -7,8 +7,9 @@
 from typing import Optional
 
 import boto3
+from boto3.s3.transfer import create_crt_transfer_manager, create_transfer_manager
+
 import botocore
-from boto3.s3.transfer import create_transfer_manager
 from botocore.client import BaseClient, Config
 
 from deadline.client.config import config_file
@@ -104,6 +105,13 @@ def get_s3_max_pool_connections() -> int:
 @lru_cache(maxsize=MAX_SIZE_CACHE)
 def get_s3_transfer_manager(s3_client: BaseClient):
     transfer_config = boto3.s3.transfer.TransferConfig()
+
+    crt_transfer_manager = create_crt_transfer_manager(client=s3_client, config=transfer_config)
+    if crt_transfer_manager:
+        return crt_transfer_manager
+
+    # Fallback to regular transfer manager if CRT transfer manager does not support the configuration, which can happen if the client
+    # and bucket are in different regions.
     return create_transfer_manager(client=s3_client, config=transfer_config)
 
 

test/unit/deadline_job_attachments/test_asset_sync.py

@@ -44,6 +44,10 @@
 from ..conftest import is_windows_non_admin
 
 
+@patch(
+    f"{deadline.__package__}.job_attachments._aws.aws_clients.create_crt_transfer_manager",
+    MagicMock(return_value=None),
+)
 class TestAssetSync:
     @pytest.fixture(autouse=True)
     def before_test(

test/unit/deadline_job_attachments/test_download.py

@@ -566,6 +566,10 @@ def assert_get_job_input_output_paths_by_asset_root(
 
 @pytest.mark.docker
 @pytest.mark.parametrize("manifest_version", [ManifestVersion.v2023_03_03])
+@patch(
+    f"{deadline.__package__}.job_attachments._aws.aws_clients.create_crt_transfer_manager",
+    MagicMock(return_value=None),
+)
 class TestFullDownload:
     """
     Tests for downloads from cas.
@@ -1965,6 +1969,10 @@ def test_download_file_error_message_on_timeout(self):
 
 
 @pytest.mark.parametrize("manifest_version", [ManifestVersion.v2023_03_03])
+@patch(
+    f"{deadline.__package__}.job_attachments._aws.aws_clients.create_crt_transfer_manager",
+    MagicMock(return_value=None),
+)
 class TestFullDownloadPrefixesWithSlashes:
     """
     Tests for downloads from cas when the queue prefixes are created.

test/unit/deadline_job_attachments/test_upload.py

@@ -57,6 +57,10 @@
 from ..conftest import is_windows_non_admin
 
 
+@patch(
+    f"{deadline.__package__}.job_attachments._aws.aws_clients.create_crt_transfer_manager",
+    MagicMock(return_value=None),
+)
 class TestUpload:
     """
     Tests for handling uploading assets.