feat: modularize checksums

AWSSDKSwiftCLI/Sources/AWSSDKSwiftCLI/Resources/Package.Base.swift

@@ -22,6 +22,7 @@ extension Target.Dependency {
     static var awsSDKEventStreamsAuth: Self { "AWSSDKEventStreamsAuth" }
     static var awsSDKHTTPAuth: Self { "AWSSDKHTTPAuth" }
     static var awsSDKIdentity: Self { "AWSSDKIdentity" }
+    static var awsSDKChecksums: Self { "AWSSDKChecksums" }
 
     // CRT module
     static var crt: Self { .product(name: "AwsCommonRuntimeKit", package: "aws-crt-swift") }
@@ -30,6 +31,7 @@ extension Target.Dependency {
     static var clientRuntime: Self { .product(name: "ClientRuntime", package: "smithy-swift") }
     static var smithy: Self { .product(name: "Smithy", package: "smithy-swift") }
     static var smithyChecksumsAPI: Self { .product(name: "SmithyChecksumsAPI", package: "smithy-swift") }
+    static var smithyChecksums: Self { .product(name: "SmithyChecksums", package: "smithy-swift") }
     static var smithyEventStreams: Self { .product(name: "SmithyEventStreams", package: "smithy-swift") }
     static var smithyEventStreamsAPI: Self { .product(name: "SmithyEventStreamsAPI", package: "smithy-swift") }
     static var smithyEventStreamsAuthAPI: Self { .product(name: "SmithyEventStreamsAuthAPI", package: "smithy-swift") }
@@ -41,6 +43,7 @@ extension Target.Dependency {
     static var smithyRetriesAPI: Self { .product(name: "SmithyRetriesAPI", package: "smithy-swift") }
     static var smithyWaitersAPI: Self { .product(name: "SmithyWaitersAPI", package: "smithy-swift") }
     static var smithyTestUtils: Self { .product(name: "SmithyTestUtil", package: "smithy-swift") }
+    static var smithyStreams: Self { .product(name: "SmithyStreams", package: "smithy-swift") }
 }
 
 // MARK: - Base Package
@@ -59,6 +62,7 @@ let package = Package(
         .library(name: "AWSSDKEventStreamsAuth", targets: ["AWSSDKEventStreamsAuth"]),
         .library(name: "AWSSDKHTTPAuth", targets: ["AWSSDKHTTPAuth"]),
         .library(name: "AWSSDKIdentity", targets: ["AWSSDKIdentity"]),
+        .library(name: "AWSSDKChecksums", targets: ["AWSSDKChecksums"]),
     ],
     targets: [
         .target(
@@ -96,14 +100,19 @@ let package = Package(
         ),
         .target(
             name: "AWSSDKHTTPAuth",
-            dependencies: [.crt, .smithy, .clientRuntime, .smithyHTTPAuth, "AWSSDKIdentity"],
+            dependencies: [.crt, .smithy, .clientRuntime, .smithyHTTPAuth, "AWSSDKIdentity", "AWSSDKChecksums"],
             path: "./Sources/Core/AWSSDKHTTPAuth"
         ),
         .target(
             name: "AWSSDKIdentity",
             dependencies: [.crt, .smithy, .clientRuntime, .smithyIdentity, .smithyIdentityAPI, .smithyHTTPAPI, .awsSDKCommon],
             path: "./Sources/Core/AWSSDKIdentity"
         ),
+        .target(
+            name: "AWSSDKChecksums",
+            dependencies: [.crt, .smithy, .clientRuntime, .smithyChecksumsAPI, .smithyChecksums, .smithyHTTPAPI],
+            path: "./Sources/Core/AWSSDKChecksums"
+        ),
         .testTarget(
             name: "AWSClientRuntimeTests",
             dependencies: [.awsClientRuntime, .clientRuntime, .smithyTestUtils, .awsSDKCommon],
@@ -112,7 +121,7 @@ let package = Package(
         ),
         .testTarget(
             name: "AWSSDKEventStreamsAuthTests",
-            dependencies: ["AWSClientRuntime", "AWSSDKEventStreamsAuth"],
+            dependencies: ["AWSClientRuntime", "AWSSDKEventStreamsAuth", .smithyStreams],
             path: "./Tests/Core/AWSSDKEventStreamsAuthTests"
         ),
         .testTarget(
@@ -186,11 +195,13 @@ let serviceTargetDependencies: [Target.Dependency] = [
     .smithyEventStreamsAuthAPI,
     .smithyEventStreams,
     .smithyChecksumsAPI,
+    .smithyChecksums,
     .smithyWaitersAPI,
     .awsSDKCommon,
     .awsSDKIdentity,
     .awsSDKHTTPAuth,
     .awsSDKEventStreamsAuth,
+    .awsSDKChecksums,
 ]
 
 func addServiceTarget(_ name: String) {
@@ -325,7 +336,7 @@ func addProtocolTests() {
         )
         let testTarget = protocolTest.buildOnly ? nil : Target.testTarget(
             name: "\(protocolTest.name)Tests",
-            dependencies: [.smithyTestUtils, .smithyWaitersAPI, .byNameItem(name: protocolTest.name, condition: nil)],
+            dependencies: [.smithyTestUtils, .smithyStreams, .smithyWaitersAPI, .byNameItem(name: protocolTest.name, condition: nil)],
             path: "\(protocolTest.testPath ?? protocolTest.sourcePath)/swift-codegen/Tests/\(protocolTest.name)Tests"
         )
         package.targets += [target, testTarget].compactMap { $0 }

IntegrationTests/Services/AWSS3IntegrationTests/S3FlexibleChecksumsTests.swift

@@ -9,6 +9,8 @@ import Smithy
 import XCTest
 import AWSS3
 @testable import ClientRuntime
+import class SmithyStreams.BufferedStream
+import class SmithyChecksums.ValidatingBufferedStream
 
 final class S3FlexibleChecksumsTests: S3XCTestCase {
     var originalData: Data!

IntegrationTests/Services/AWSS3IntegrationTests/S3StreamTests.swift

@@ -9,6 +9,7 @@ import Smithy
 import XCTest
 import AWSS3
 @testable import ClientRuntime
+import class SmithyStreams.FileStream
 
 final class S3StreamTests: S3XCTestCase {
     let objectName = "hello-world"

Sources/Core/AWSClientRuntime/Middlewares/FlexibleChecksumsRequestMiddleware.swift

@@ -0,0 +1,119 @@
+//
+// Copyright Amazon.com Inc. or its affiliates.
+// All Rights Reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+import enum SmithyChecksumsAPI.ChecksumAlgorithm
+import enum SmithyChecksums.ChecksumMismatchException
+import enum Smithy.ClientError
+import class Smithy.Context
+import AwsCommonRuntimeKit
+import AWSSDKChecksums
+import ClientRuntime
+import SmithyHTTPAPI
+
+public struct FlexibleChecksumsRequestMiddleware<OperationStackInput, OperationStackOutput>: Middleware {
+
+    public let id: String = "FlexibleChecksumsRequestMiddleware"
+
+    let checksumAlgorithm: String?
+
+    public init(checksumAlgorithm: String?) {
+        self.checksumAlgorithm = checksumAlgorithm
+    }
+
+    public func handle<H>(context: Context,
+                          input: SerializeStepInput<OperationStackInput>,
+                          next: H) async throws -> OperationOutput<OperationStackOutput>
+    where H: Handler,
+    Self.MInput == H.Input,
+    Self.MOutput == H.Output {
+        try await addHeaders(builder: input.builder, attributes: context)
+        return try await next.handle(context: context, input: input)
+    }
+
+    private func addHeaders(builder: SdkHttpRequestBuilder, attributes: Context) async throws {
+        if case(.stream(let stream)) = builder.body {
+            attributes.isChunkedEligibleStream = stream.isEligibleForChunkedStreaming
+            if stream.isEligibleForChunkedStreaming {
+                try builder.setAwsChunkedHeaders() // x-amz-decoded-content-length
+            }
+        }
+
+        // Initialize logger
+        guard let logger = attributes.getLogger() else {
+            throw ClientError.unknownError("No logger found!")
+        }
+
+        guard let checksumString = checksumAlgorithm else {
+            logger.info("No checksum provided! Skipping flexible checksums workflow...")
+            return
+        }
+
+        guard let checksumHashFunction = ChecksumAlgorithm.from(string: checksumString) else {
+            logger.info("Found no supported checksums! Skipping flexible checksums workflow...")
+            return
+        }
+
+        // Determine the header name
+        let headerName = "x-amz-checksum-\(checksumHashFunction)"
+        logger.debug("Resolved checksum header name: \(headerName)")
+
+        // Check if any checksum header is already provided by the user
+        let checksumHeaderPrefix = "x-amz-checksum-"
+        if builder.headers.headers.contains(where: {
+            $0.name.lowercased().starts(with: checksumHeaderPrefix) &&
+            $0.name.lowercased() != "x-amz-checksum-algorithm"
+        }) {
+            logger.debug("Checksum header already provided by the user. Skipping calculation.")
+            return
+        }
+
+        // Handle body vs handle stream
+        switch builder.body {
+        case .data(let data):
+            guard let data else {
+                throw ClientError.dataNotFound("Cannot calculate checksum of empty body!")
+            }
+
+            if builder.headers.value(for: headerName) == nil {
+                logger.debug("Calculating checksum")
+            }
+
+            // Create checksum instance
+            let checksum = checksumHashFunction.createChecksum()
+
+            // Pass data to hash
+            try checksum.update(chunk: data)
+
+            // Retrieve the hash
+            let hash = try checksum.digest().toBase64String()
+
+            builder.updateHeader(name: headerName, value: [hash])
+        case .stream:
+            // Will handle calculating checksum and setting header later
+            attributes.checksum = checksumHashFunction
+            builder.updateHeader(name: "x-amz-trailer", value: [headerName])
+        case .noStream:
+            throw ClientError.dataNotFound("Cannot calculate the checksum of an empty body!")
+        }
+    }
+
+    public typealias MInput = SerializeStepInput<OperationStackInput>
+    public typealias MOutput = OperationOutput<OperationStackOutput>
+}
+
+extension FlexibleChecksumsRequestMiddleware: HttpInterceptor {
+    public typealias InputType = OperationStackInput
+    public typealias OutputType = OperationStackOutput
+
+    public func modifyBeforeRetryLoop(
+        context: some MutableRequest<InputType, RequestType, AttributesType>
+    ) async throws {
+        let builder = context.getRequest().toBuilder()
+        try await addHeaders(builder: builder, attributes: context.getAttributes())
+        context.updateRequest(updated: builder.build())
+    }
+}

Sources/Core/AWSClientRuntime/Middlewares/FlexibleChecksumsResponseMiddleware.swift

@@ -0,0 +1,145 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0.
+
+import Smithy
+import SmithyHTTPAPI
+import enum SmithyChecksumsAPI.ChecksumAlgorithm
+import enum SmithyChecksums.ChecksumMismatchException
+import ClientRuntime
+
+public struct FlexibleChecksumsResponseMiddleware<OperationStackInput, OperationStackOutput>: Middleware {
+
+    public let id: String = "FlexibleChecksumsResponseMiddleware"
+
+    // The priority to validate response checksums, if multiple are present
+    let CHECKSUM_HEADER_VALIDATION_PRIORITY_LIST: [String] = [
+        ChecksumAlgorithm.crc32c,
+        .crc32,
+        .sha1,
+        .sha256
+    ].sorted().map { $0.toString() }
+
+    let validationMode: Bool
+    let priorityList: [String]
+
+    public init(validationMode: Bool, priorityList: [String] = []) {
+        self.validationMode = validationMode
+        self.priorityList = !priorityList.isEmpty
+            ? withPriority(checksums: priorityList)
+            : CHECKSUM_HEADER_VALIDATION_PRIORITY_LIST
+    }
+
+    public func handle<H>(context: Context,
+                          input: SdkHttpRequest,
+                          next: H) async throws -> OperationOutput<OperationStackOutput>
+    where H: Handler,
+    Self.MInput == H.Input,
+    Self.MOutput == H.Output {
+
+        // The name of the checksum header which was validated. If `null`, validation was not performed.
+        context.attributes.set(key: AttributeKey<String>(name: "ChecksumHeaderValidated"), value: nil)
+
+        // Initialize logger
+        guard let logger = context.getLogger() else { throw ClientError.unknownError("No logger found!") }
+
+        // Get the response
+        let output = try await next.handle(context: context, input: input)
+
+        try await validateChecksum(response: output.httpResponse, logger: logger, attributes: context)
+
+        return output
+    }
+
+    private func validateChecksum(response: HttpResponse, logger: any LogAgent, attributes: Context) async throws {
+        // Exit if validation should not be performed
+        if !validationMode {
+            logger.info("Checksum validation should not be performed! Skipping workflow...")
+            return
+        }
+
+        let checksumHeaderIsPresent = priorityList.first {
+            response.headers.value(for: "x-amz-checksum-\($0)") != nil
+        }
+
+        guard let checksumHeader = checksumHeaderIsPresent else {
+            let message =
+                "User requested checksum validation, but the response headers did not contain any valid checksums"
+            logger.warn(message)
+            return
+        }
+
+        let fullChecksumHeader = "x-amz-checksum-" + checksumHeader
+
+        logger.debug("Validating checksum from \(fullChecksumHeader)")
+        attributes.set(key: AttributeKey<String>(name: "ChecksumHeaderValidated"), value: fullChecksumHeader)
+
+        let checksumString = checksumHeader.removePrefix("x-amz-checksum-")
+        guard let responseChecksum = ChecksumAlgorithm.from(string: checksumString) else {
+            throw ClientError.dataNotFound("Checksum found in header is not supported!")
+        }
+
+        guard let expectedChecksum = response.headers.value(for: fullChecksumHeader) else {
+            throw ClientError.dataNotFound("Could not determine the expected checksum!")
+        }
+
+        // Handle body vs handle stream
+        switch response.body {
+        case .data(let data):
+            guard let data else {
+                throw ClientError.dataNotFound("Cannot calculate checksum of empty body!")
+            }
+
+            let responseChecksumHasher = responseChecksum.createChecksum()
+            try responseChecksumHasher.update(chunk: data)
+            let actualChecksum = try responseChecksumHasher.digest().toBase64String()
+
+            guard expectedChecksum == actualChecksum else {
+                let message = "Checksum mismatch. Expected \(expectedChecksum) but was \(actualChecksum)"
+                throw ChecksumMismatchException.message(message)
+            }
+        case .stream(let stream):
+            let validatingStream = ByteStream.getChecksumValidatingBody(
+                stream: stream,
+                expectedChecksum: expectedChecksum,
+                checksumAlgorithm: responseChecksum
+            )
+
+            // Set the response to a validating stream
+            attributes.httpResponse = response
+            attributes.httpResponse?.body = validatingStream
+        case .noStream:
+            throw ClientError.dataNotFound("Cannot calculate the checksum of an empty body!")
+        }
+    }
+
+    public typealias MInput = SdkHttpRequest
+    public typealias MOutput = OperationOutput<OperationStackOutput>
+}
+
+extension FlexibleChecksumsResponseMiddleware: HttpInterceptor {
+    public typealias InputType = OperationStackInput
+    public typealias OutputType = OperationStackOutput
+
+    public func modifyBeforeRetryLoop(
+        context: some MutableRequest<InputType, RequestType, AttributesType>
+    ) async throws {
+        context.getAttributes().set(key: AttributeKey<String>(name: "ChecksumHeaderValidated"), value: nil)
+    }
+
+    public func modifyBeforeDeserialization(
+        context: some MutableResponse<InputType, RequestType, ResponseType, AttributesType>
+    ) async throws {
+        guard let logger = context.getAttributes().getLogger() else {
+            throw ClientError.unknownError("No logger found!")
+        }
+
+        let response = context.getResponse()
+        try await validateChecksum(response: response, logger: logger, attributes: context.getAttributes())
+        context.updateResponse(updated: response)
+    }
+}
+
+private func withPriority(checksums: [String]) -> [String] {
+    let checksumsMap = checksums.compactMap { ChecksumAlgorithm.from(string: $0) }
+    return checksumsMap.sorted().map { $0.toString() }
+}