-
Notifications
You must be signed in to change notification settings - Fork 133
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
RUMM-1679 Compress HTTP body using
deflate format (ETF RFC 1950)
- Loading branch information
Showing
11 changed files
with
293 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| /* | ||
| * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. | ||
| * This product includes software developed at Datadog (https://www.datadoghq.com/). | ||
| * Copyright 2019-2020 Datadog, Inc. | ||
| */ | ||
|
|
||
| import Foundation | ||
| import Compression | ||
| import zlib | ||
|
|
||
| /// Compresses the data into `ZLIB` data format. | ||
| /// | ||
| /// The `Compression` library implements the zlib encoder at level 5 only. This compression level | ||
| /// provides a good balance between compression speed and compression ratio. | ||
| /// | ||
| /// The encoded format is the ZLIB Compressed Data Format as described in IETF RFC 1950 | ||
| /// https://datatracker.ietf.org/doc/html/rfc1950 | ||
| /// | ||
| /// - Parameter data: Source data to deflate | ||
| /// - Returns: The compressed data format. | ||
| internal func zip(_ data: Data) -> Data? { | ||
| // 2 bytes header - defines the compression mode | ||
| // +---+---+ | ||
| // |CMF|FLG| | ||
| // +---+---+ | ||
| // ref. https://datatracker.ietf.org/doc/html/rfc1950#section-2.2 | ||
| let header = Data([0x78, 0x5e]) | ||
|
|
||
| guard | ||
| let raw = deflate(data), | ||
| let checksum = adler32(data), | ||
| data.count > header.count + raw.count + checksum.count | ||
| else { return nil } | ||
|
|
||
| return header + raw + checksum | ||
| } | ||
|
|
||
| /// Compresses the data using the `ZLIB` compression algorithm. | ||
| /// | ||
| /// The `Compression` library implements the zlib encoder at level 5 only. This compression level | ||
| /// provides a good balance between compression speed and compression ratio. | ||
| /// | ||
| /// The encoded format is the raw DEFLATE format as described in in IETF RFC 1951 | ||
| /// https://datatracker.ietf.org/doc/html/rfc1951 | ||
| /// | ||
| /// This deflate implementation uses `compression_encode_buffer(_:_:_:_:_:_:)` | ||
| /// from the `Compression` framework by allocating a destination buffer of source size and copying | ||
| /// the result into a `Data` structure. In the worst possible case, where the compression expands the | ||
| /// data size, the destination buffer becomes too small and deflation returns `nil`. | ||
| /// | ||
| /// ref. https://developer.apple.com/documentation/compression/1480986-compression_encode_buffer | ||
| /// | ||
| /// - Parameter data: Source data to deflate | ||
| /// - Returns: The compressed data. If the compressed data size is bigger than the source size, | ||
| /// or an error occurs, `nil` is returned. | ||
| internal func deflate(_ data: Data) -> Data? { | ||
| return data.withUnsafeBytes { | ||
| guard let ptr = $0.bindMemory(to: UInt8.self).baseAddress else { | ||
| return nil | ||
| } | ||
|
|
||
| let buffer = UnsafeMutablePointer<UInt8>.allocate(capacity: data.count) | ||
| defer { buffer.deallocate() } | ||
|
|
||
| // The number of bytes written to the destination buffer after compressing | ||
| // the input. If the funtion can't compress the entire input to fit into | ||
| // the provided destination buffer, or an error occurs, 0 is returned. | ||
| let size = compression_encode_buffer(buffer, data.count, ptr, data.count, nil, COMPRESSION_ZLIB) | ||
| guard size > 0 else { | ||
| return nil | ||
| } | ||
|
|
||
| return Data(bytes: buffer, count: size) | ||
| } | ||
| } | ||
|
|
||
| /// Calculates the Adler32 checksum of the given data. | ||
| /// | ||
| /// An Adler-32 checksum is almost as reliable as a CRC-32 but can be computed much faster. | ||
| /// | ||
| /// - Parameter data: Data to compute the checksum. | ||
| /// - Returns: The Adler-32 checksum. | ||
| internal func adler32(_ data: Data) -> Data? { | ||
| let adler: uLong? = data.withUnsafeBytes { | ||
| guard let ptr = $0.bindMemory(to: Bytef.self).baseAddress else { | ||
| return nil | ||
| } | ||
|
|
||
| // The Adler-32 checksum should be initialized to 1 as described in | ||
| // https://datatracker.ietf.org/doc/html/rfc1950#section-8 | ||
| return zlib.adler32(1, ptr, uInt(data.count)) | ||
| } | ||
|
|
||
| guard let adler = adler else { | ||
| return nil | ||
| } | ||
|
|
||
| var checksum = UInt32(adler).bigEndian | ||
| return Data(bytes: &checksum, count: MemoryLayout<UInt32>.size) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
64 changes: 64 additions & 0 deletions
64
Tests/DatadogTests/Datadog/Core/Upload/DataCompressionTests.swift
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| /* | ||
| * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. | ||
| * This product includes software developed at Datadog (https://www.datadoghq.com/). | ||
| * Copyright 2019-2020 Datadog, Inc. | ||
| */ | ||
|
|
||
| import XCTest | ||
| @testable import Datadog | ||
|
|
||
| class DataCompressionTests: XCTestCase { | ||
| func testFuzzy_Adler32() { | ||
| for _ in 1...500 { | ||
| // Given | ||
| let data: Data = .mockRandom(ofSize: Int.mockRandom(min: 1, max: 10_000)) | ||
|
|
||
| // When | ||
| let checksum = adler32(data) | ||
|
|
||
| // Then | ||
| XCTAssertEqual(checksum?.count, 4) | ||
| } | ||
| } | ||
|
|
||
| func testFuzzy_deflate_inflate() throws { | ||
| for _ in 1...500 { | ||
| // Given | ||
| let data: Data = .mockRandom(ofSize: Int.mockRandom(min: 100, max: 10_000)) | ||
|
|
||
| // When | ||
| let compressed = try XCTUnwrap(deflate(data)) | ||
| let decompressed = inflate(compressed) | ||
|
|
||
| // Then | ||
| XCTAssertEqual(decompressed, data) | ||
| } | ||
| } | ||
|
|
||
| func testFuzzy_zip_unzip() throws { | ||
| for _ in 1...500 { | ||
| // Given | ||
| let data: Data = .mockRandom(ofSize: Int.mockRandom(min: 100, max: 10_000)) | ||
|
|
||
| // When | ||
| let compressed = try XCTUnwrap(zip(data)) | ||
| let decompressed = unzip(compressed) | ||
|
|
||
| // Then | ||
| XCTAssertEqual(decompressed, data) | ||
| } | ||
| } | ||
|
|
||
| func test8MB_deflate_inflate() throws { | ||
| // Given | ||
| let size = 1_024 * 1_024 * 8 // 8 MB | ||
| let data: Data = .mockRandom(ofSize: size) | ||
|
|
||
| // When | ||
| let compressed = try XCTUnwrap(deflate(data)) | ||
| let decompressed = inflate(compressed, capacity: size) | ||
|
|
||
| // Then | ||
| XCTAssertEqual(decompressed, data) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.