feat: Add support for flexible checksums on streaming payloads

.github/workflows/integration-test.yml

@@ -20,18 +20,21 @@ jobs:
       matrix:
         # This matrix runs tests on Mac, on oldest & newest supported Xcodes
         runner:
+          - macos-13
+          - macos-14
           - macos-13-xlarge
           - macos-14-xlarge
         xcode:
-          - Xcode_14.1
+          - Xcode_14.3.1
+          - Xcode_15.0.1
           - Xcode_15.2
-        exclude:
+        #exclude:
           # Don't run old macOS with new Xcode
-          - runner: macos-13-xlarge
-            xcode: Xcode_15.2
+          #- runner: macos-13
+          #  xcode: Xcode_15.2
           # Don't run new macOS with old Xcode
-          - runner: macos-14-xlarge
-            xcode: Xcode_14.1
+          #- runner: macos-14
+          #  xcode: Xcode_14.1
     steps:
       - name: Configure AWS Credentials for Integration Tests
         uses: aws-actions/configure-aws-credentials@v4
@@ -95,13 +98,15 @@ jobs:
       matrix:
         os:
           - focal
-          #- amazonlinux2
+          - amazonlinux2
         version:
-          - 5.7
-          - 5.9
+          - "5.7"
+          - "5.8"
+          - "5.9"
+          - "5.10"
     steps:
       - name: Configure AWS Credentials for Integration Tests
-        uses: aws-actions/configure-aws-credentials@v4
+        uses: aws-actions/configure-aws-credentials@v3
         with:
           role-to-assume: ${{ secrets.INTEGRATION_TEST_ROLE_ARN }}
           aws-region: us-west-2

IntegrationTests/Services/AWSS3IntegrationTests/S3FlexibleChecksumsTests.swift

@@ -0,0 +1,134 @@
+//
+// Copyright Amazon.com Inc. or its affiliates.
+// All Rights Reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+import XCTest
+import AWSS3
+@testable import ClientRuntime
+
+final class S3FlexibleChecksumsTests: S3XCTestCase {
+    var originalData: Data!
+
+    override func setUp() {
+        super.setUp()
+        // Fill one MB with random data.  Data is refreshed for each flexible checksums tests below.
+        originalData = Data((0..<(1024 * 1024)).map { _ in UInt8.random(in: UInt8.min...UInt8.max) })
+    }
+
+    // MARK: - Data uploads
+
+    func test_putGetObject_data_crc32() async throws {
+        try await _testPutGetObject(withChecksumAlgorithm: .crc32, objectNameSuffix: "crc32-data", upload: .data(originalData))
+    }
+
+    func test_putGetObject_data_crc32c() async throws {
+        try await _testPutGetObject(withChecksumAlgorithm: .crc32c, objectNameSuffix: "crc32c-data", upload: .data(originalData))
+    }
+
+    func test_putGetObject_data_sha1() async throws {
+        try await _testPutGetObject(withChecksumAlgorithm: .sha1, objectNameSuffix: "sha1-data", upload: .data(originalData))
+    }
+
+    func test_putGetObject_data_sha256() async throws {
+        try await _testPutGetObject(withChecksumAlgorithm: .sha256, objectNameSuffix: "sha256-data", upload: .data(originalData))
+    }
+
+    // MARK: - Streaming uploads
+
+    func test_putGetObject_streaming_crc32() async throws {
+        let bufferedStream = BufferedStream(data: originalData, isClosed: true)
+        try await _testPutGetObject(withChecksumAlgorithm: .crc32, objectNameSuffix: "crc32", upload: .stream(bufferedStream))
+    }
+
+    func test_putGetObject_streaming_crc32c() async throws {
+        let bufferedStream = BufferedStream(data: originalData, isClosed: true)
+        try await _testPutGetObject(withChecksumAlgorithm: .crc32c, objectNameSuffix: "crc32c", upload: .stream(bufferedStream))
+    }
+
+    func test_putGetObject_streaming_sha1() async throws {
+        let bufferedStream = BufferedStream(data: originalData, isClosed: true)
+        try await _testPutGetObject(withChecksumAlgorithm: .sha1, objectNameSuffix: "sha1", upload: .stream(bufferedStream))
+    }
+
+    func test_putGetObject_streaming_sha256() async throws {
+        let bufferedStream = BufferedStream(data: originalData, isClosed: true)
+        try await _testPutGetObject(withChecksumAlgorithm: .sha256, objectNameSuffix: "sha256", upload: .stream(bufferedStream))
+    }
+
+    // Streaming without checksum (chunked encoding)
+    func test_putGetObject_streaming_chunked() async throws {
+        let bufferedStream = BufferedStream(data: originalData, isClosed: true)
+        let objectName = "flexible-checksums-s3-test-chunked"
+
+        let putObjectInput = PutObjectInput(
+            body: .stream(bufferedStream),
+            bucket: bucketName,
+            key: objectName
+        )
+
+        _ = try await client.putObject(input: putObjectInput)
+
+        let getObjectInput = GetObjectInput(bucket: bucketName, key: objectName)
+        let getObjectOutput = try await client.getObject(input: getObjectInput)
+        let body = try XCTUnwrap(getObjectOutput.body)
+        let data = try await body.readData()
+        XCTAssertEqual(data, originalData)
+    }
+
+    // MARK: - Private methods
+
+    private func _testPutGetObject(
+        withChecksumAlgorithm algorithm: S3ClientTypes.ChecksumAlgorithm,
+        objectNameSuffix: String, upload: ByteStream, file: StaticString = #filePath, line: UInt = #line
+    ) async throws {
+        let objectName = "flexible-checksums-s3-test-\(objectNameSuffix)"
+
+        let input = PutObjectInput(
+            body: upload,
+            bucket: bucketName,
+            checksumAlgorithm: algorithm,
+            key: objectName
+        )
+
+        let output = try await client.putObject(input: input)
+
+        // Verify the checksum response based on the algorithm used.
+        let checksumResponse = try XCTUnwrap(getChecksumResponse(from: output, with: algorithm), file: file, line: line)
+        XCTAssertNotNil(checksumResponse, file: file, line: line)
+
+        let getInput = GetObjectInput(bucket: bucketName, checksumMode: S3ClientTypes.ChecksumMode.enabled, key: objectName)
+        let getOutput = try await client.getObject(input: getInput) // will error for normal payloads if checksum mismatch
+        XCTAssertNotNil(getOutput.body, file: file, line: line) // Ensure there's a body in the response.
+
+        // Additional step for stream: Validate stream and read data.
+        if case .stream = upload {
+            let streamingBody = try XCTUnwrap(getOutput.body, file: file, line: line)
+            if case .stream(let stream) = streamingBody {
+                XCTAssert(stream is ValidatingBufferedStream, "Expected ValidatingBufferedStream for streaming upload", file: file, line: line)
+                let data = try await streamingBody.readData() // will error if checksum mismatch
+                XCTAssertEqual(data, originalData, file: file, line: line)
+            } else {
+                XCTFail("Did not receive a stream when expected for checksum validation!", file: file, line: line)
+            }
+        }
+    }
+
+    private func getChecksumResponse(from response: PutObjectOutput, with algorithm: S3ClientTypes.ChecksumAlgorithm) throws -> String? {
+        switch algorithm {
+        case .crc32:
+            return response.checksumCRC32
+        case .crc32c:
+            return response.checksumCRC32C
+        case .sha1:
+            return response.checksumSHA1
+        case .sha256:
+            return response.checksumSHA256
+        default:
+            XCTFail("Unsupported checksum algorithm")
+            return nil
+        }
+    }
+}

Package.swift

@@ -242,7 +242,7 @@ func addResolvedTargets() {
 
 addDependencies(
     clientRuntimeVersion: "0.44.0",
-    crtVersion: "0.26.0"
+    crtVersion: "0.28.0"
 )
 
 // Uncomment this line to exclude runtime unit tests

Sources/Core/AWSClientRuntime/Auth/AuthSchemes/SigV4AuthScheme.swift

@@ -55,6 +55,15 @@ public struct SigV4AuthScheme: ClientRuntime.AuthScheme {
         updatedSigningProperties.set(key: AttributeKeys.shouldNormalizeURIPath, value: true)
         updatedSigningProperties.set(key: AttributeKeys.omitSessionToken, value: false)
 
+        // Copy checksum from middleware context to signing properties
+        updatedSigningProperties.set(key: AttributeKeys.checksum, value: context.getChecksum())
+
+        // Copy chunked streaming eligiblity from middleware context to signing properties
+        updatedSigningProperties.set(
+            key: AttributeKeys.isChunkedEligibleStream,
+            value: context.getIsChunkedEligibleStream()
+        )
+
         // Set service-specific signing properties if needed.
         try CustomSigningPropertiesSetter().setServiceSpecificSigningProperties(
             signingProperties: &updatedSigningProperties,

Sources/Core/AWSClientRuntime/Signing/AWSSigV4Signer.swift

@@ -41,13 +41,31 @@ public class AWSSigV4Signer: ClientRuntime.Signer {
             try unsignedRequest.toHttp2Request() :
             try unsignedRequest.toHttpRequest()
 
+        let crtSigningConfig = try signingConfig.toCRTType()
+
         let crtSignedRequest = try await Signer.signRequest(
             request: crtUnsignedRequest,
-            config: signingConfig.toCRTType()
+            config: crtSigningConfig
         )
 
+        let sdkSignedRequest = requestBuilder.update(from: crtSignedRequest, originalRequest: unsignedRequest)
+
+        if crtSigningConfig.useAwsChunkedEncoding {
+            guard let requestSignature = crtSignedRequest.signature else {
+                throw ClientError.dataNotFound("Could not get request signature!")
+            }
+
+            // Set streaming body to an AwsChunked wrapped type
+            try sdkSignedRequest.setAwsChunkedBody(
+                signingConfig: crtSigningConfig,
+                signature: requestSignature,
+                trailingHeaders: unsignedRequest.trailingHeaders,
+                checksumAlgorithm: signingProperties.get(key: AttributeKeys.checksum)
+            )
+        }
+
         // Return signed request
-        return requestBuilder.update(from: crtSignedRequest, originalRequest: unsignedRequest)
+        return sdkSignedRequest
     }
 
     private func constructSigningConfig(
@@ -77,7 +95,17 @@ public class AWSSigV4Signer: ClientRuntime.Signer {
 
         let expiration: TimeInterval = signingProperties.get(key: AttributeKeys.expiration) ?? 0
         let signedBodyHeader: AWSSignedBodyHeader = signingProperties.get(key: AttributeKeys.signedBodyHeader) ?? .none
-        let signedBodyValue: AWSSignedBodyValue = unsignedBody ? .unsignedPayload : .empty
+
+        // Determine signed body value
+        let checksum = signingProperties.get(key: AttributeKeys.checksum)
+        let isChunkedEligibleStream = signingProperties.get(key: AttributeKeys.isChunkedEligibleStream) ?? false
+
+        let signedBodyValue: AWSSignedBodyValue = determineSignedBodyValue(
+            checksum: checksum,
+            isChunkedEligbleStream: isChunkedEligibleStream,
+            isUnsignedBody: unsignedBody
+        )
+
         let flags: SigningFlags = SigningFlags(
             useDoubleURIEncode: signingProperties.get(key: AttributeKeys.useDoubleURIEncode) ?? true,
             shouldNormalizeURIPath: signingProperties.get(key: AttributeKeys.shouldNormalizeURIPath) ?? true,
@@ -216,4 +244,23 @@ public class AWSSigV4Signer: ClientRuntime.Signer {
             signingAlgorithm: signingConfig.signingAlgorithm
         )
     }
+
+    private func determineSignedBodyValue(
+        checksum: ChecksumAlgorithm?,
+        isChunkedEligbleStream: Bool,
+        isUnsignedBody: Bool
+    ) -> AWSSignedBodyValue {
+        if !isChunkedEligbleStream {
+            // Normal Payloads, Event Streams, etc.
+            return isUnsignedBody ? .unsignedPayload : .empty
+        }
+
+        // streaming + eligible for chunked transfer
+        if checksum == nil {
+            return isUnsignedBody ? .unsignedPayload : .streamingSha256Payload
+        } else {
+            // checksum is present
+            return isUnsignedBody ? .streamingUnsignedPayloadTrailer : .streamingSha256PayloadTrailer
+        }
+    }
 }

Sources/Core/AWSClientRuntime/Signing/AWSSignedBodyValue.swift

@@ -13,6 +13,8 @@ public enum AWSSignedBodyValue {
     case unsignedPayload
     case streamingSha256Payload
     case streamingSha256Events
+    case streamingSha256PayloadTrailer
+    case streamingUnsignedPayloadTrailer
 }
 
 extension AWSSignedBodyValue {
@@ -23,6 +25,8 @@ extension AWSSignedBodyValue {
         case .unsignedPayload: return .unsignedPayload
         case .streamingSha256Payload: return .streamingSha256Payload
         case .streamingSha256Events: return .streamingSha256Events
+        case .streamingSha256PayloadTrailer: return .streamingSha256PayloadTrailer
+        case .streamingUnsignedPayloadTrailer: return .streamingUnSignedPayloadTrailer
         }
     }
 }

Sources/Core/AWSClientRuntime/Signing/AWSSigningConfig.swift

@@ -53,7 +53,23 @@ public struct AWSSigningConfig {
 
 extension AWSSigningConfig {
     func toCRTType() throws -> SigningConfig {
-        SigningConfig(
+        // Never include the Transfer-Encoding header in the signature,
+        // since older versions of URLSession will modify this header's value
+        // prior to sending a request.
+        //
+        // The Transfer-Encoding header does not affect the AWS operation being
+        // performed and is just there to coordinate the flow of data to the server.
+        //
+        // For all other headers, use the shouldSignHeaders block that was passed to
+        // determine if the header should be included in the signature.  If the
+        // shouldSignHeaders block was not provided, then include all headers other
+        // than Transfer-Encoding.
+        let modifiedShouldSignHeader = { (name: String) in
+            guard name.lowercased(with: Locale(identifier: "en_US_POSIX")) != "transfer-encoding" else { return false }
+            return shouldSignHeader?(name) ?? true
+        }
+
+        return SigningConfig(
             algorithm: signingAlgorithm.toCRTType(),
             signatureType: signatureType.toCRTType(),
             service: service,
@@ -67,7 +83,7 @@ extension AWSSigningConfig {
             expiration: expiration,
             signedBodyHeader: signedBodyHeader.toCRTType(),
             signedBodyValue: signedBodyValue.toCRTType(),
-            shouldSignHeader: shouldSignHeader,
+            shouldSignHeader: modifiedShouldSignHeader,
             useDoubleURIEncode: flags.useDoubleURIEncode,
             shouldNormalizeURIPath: flags.shouldNormalizeURIPath,
             omitSessionToken: flags.omitSessionToken

codegen/smithy-aws-swift-codegen/src/main/kotlin/software/amazon/smithy/aws/swift/codegen/customization/flexiblechecksums/FlexibleChecksumsRequestIntegration.kt

@@ -56,7 +56,6 @@ private object FlexibleChecksumRequestMiddleware : MiddlewareRenderable {
         val httpChecksumTrait = op.getTrait(HttpChecksumTrait::class.java).orElse(null)
         val inputMemberName = httpChecksumTrait?.requestAlgorithmMember?.get()?.lowercaseFirstLetter()
 
-        // Convert algorithmNames list to a Swift array representation
         val middlewareInit = "${ClientRuntimeTypes.Middleware.FlexibleChecksumsRequestMiddleware}<$inputShapeName, $outputShapeName>(checksumAlgorithm: input.$inputMemberName?.rawValue)"
 
         writer.write("$operationStackName.${middlewareStep.stringValue()}.intercept(position: ${position.stringValue()}, middleware: $middlewareInit)")

packageDependencies.plist

@@ -5,7 +5,7 @@
 	<key>awsCRTSwiftBranch</key>
 	<string>main</string>
 	<key>awsCRTSwiftVersion</key>
-	<string>0.26.0</string>
+	<string>0.28.0</string>
 	<key>clientRuntimeBranch</key>
 	<string>main</string>
 	<key>clientRuntimeVersion</key>

scripts/ci_steps/log_tool_versions.sh

@@ -4,6 +4,17 @@ set -e
 
 echo
 
+# Log OS version (sw_vers on Mac, uname -a on Linux)
+
+if command -v sw_vers &> /dev/null
+then
+  sw_vers
+else
+  uname -a
+fi
+
+
+
 # Log CPU for hardware in use, if running on Mac
 
 if [[ "$OSTYPE" == "darwin"* ]];