Skip to content

Commit

Permalink
Merge pull request #620 from DataDog/ncreated/RUMM-1394-add-more-info…
Browse files Browse the repository at this point in the history 
…rmation-to-flaky-test-errors

RUMM-1394 Add more verbosity to flaky test errors
  • Loading branch information
@ncreated
ncreated authored Oct 4, 2021
2 parents 356d4d7 + 0b82734 commit f362f01
Show file tree
Hide file tree
Showing 4 changed files with 130 additions and 26 deletions.
24 changes: 14 additions & 10 deletions Tests/DatadogTests/Datadog/URLSessionAutoInstrumentation/DDURLSessionDelegateTests.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,26 +94,28 @@ class DDURLSessionDelegateTests: XCTestCase {

// When
let taskWithURL = session.dataTask(with: URL.mockAny())
taskWithURL.taskDescription = "taskWithURL"
taskWithURL.resume()

let taskWithURLRequest = session.dataTask(with: URLRequest(url: .mockAny()))
taskWithURLRequest.taskDescription = "taskWithURLRequest"
taskWithURLRequest.resume()

// Then
waitForExpectations(timeout: 0.5, handler: nil)
_ = server.waitAndReturnRequests(count: 1)

let dateAfterAllRequests = Date()
XCTAssertTrue(interceptor.taskMetrics[0].task === taskWithURL)
AssertURLSessionTasksIdentical(interceptor.taskMetrics[0].task, taskWithURL)
XCTAssertGreaterThan(interceptor.taskMetrics[0].metrics.taskInterval.start, dateBeforeAnyRequests)
XCTAssertLessThan(interceptor.taskMetrics[0].metrics.taskInterval.end, dateAfterAllRequests)
XCTAssertTrue(interceptor.tasksCompleted[0].task === taskWithURL)
AssertURLSessionTasksIdentical(interceptor.tasksCompleted[0].task, taskWithURL)
XCTAssertEqual((interceptor.tasksCompleted[0].error! as NSError).localizedDescription, "some error")

XCTAssertTrue(interceptor.taskMetrics[1].task === taskWithURLRequest)
AssertURLSessionTasksIdentical(interceptor.taskMetrics[1].task, taskWithURLRequest)
XCTAssertGreaterThan(interceptor.taskMetrics[1].metrics.taskInterval.start, dateBeforeAnyRequests)
XCTAssertLessThan(interceptor.taskMetrics[1].metrics.taskInterval.end, dateAfterAllRequests)
XCTAssertTrue(interceptor.tasksCompleted[1].task === taskWithURLRequest)
AssertURLSessionTasksIdentical(interceptor.tasksCompleted[1].task, taskWithURLRequest)
XCTAssertEqual((interceptor.tasksCompleted[1].error! as NSError).localizedDescription, "some error")

XCTAssertEqual(interceptor.tasksReceivedData.count, 0, "When tasks complete with failure, they should not receive data")
Expand Down Expand Up @@ -141,30 +143,32 @@ class DDURLSessionDelegateTests: XCTestCase {

// When
let taskWithURL = session.dataTask(with: URL.mockAny())
taskWithURL.taskDescription = "taskWithURL"
taskWithURL.resume()

let taskWithURLRequest = session.dataTask(with: URLRequest(url: .mockAny()))
taskWithURLRequest.taskDescription = "taskWithURLRequest"
taskWithURLRequest.resume()

// Then
waitForExpectations(timeout: 0.5, handler: nil)
_ = server.waitAndReturnRequests(count: 1)

let dateAfterAllRequests = Date()
XCTAssertTrue(interceptor.taskMetrics[0].task === taskWithURL)
AssertURLSessionTasksIdentical(interceptor.taskMetrics[0].task, taskWithURL)
XCTAssertGreaterThan(interceptor.taskMetrics[0].metrics.taskInterval.start, dateBeforeAnyRequests)
XCTAssertLessThan(interceptor.taskMetrics[0].metrics.taskInterval.end, dateAfterAllRequests)
XCTAssertTrue(interceptor.tasksCompleted[0].task === taskWithURL)
AssertURLSessionTasksIdentical(interceptor.tasksCompleted[0].task, taskWithURL)
XCTAssertNil(interceptor.tasksCompleted[0].error)
XCTAssertTrue(interceptor.tasksReceivedData[0].task === taskWithURL)
AssertURLSessionTasksIdentical(interceptor.tasksReceivedData[0].task, taskWithURL)
XCTAssertEqual(interceptor.tasksReceivedData[0].data, randomData)

XCTAssertTrue(interceptor.taskMetrics[1].task === taskWithURLRequest)
AssertURLSessionTasksIdentical(interceptor.taskMetrics[1].task, taskWithURLRequest)
XCTAssertGreaterThan(interceptor.taskMetrics[1].metrics.taskInterval.start, dateBeforeAnyRequests)
XCTAssertLessThan(interceptor.taskMetrics[1].metrics.taskInterval.end, dateAfterAllRequests)
XCTAssertTrue(interceptor.tasksCompleted[1].task === taskWithURLRequest)
AssertURLSessionTasksIdentical(interceptor.tasksCompleted[1].task, taskWithURLRequest)
XCTAssertNil(interceptor.tasksCompleted[1].error)
XCTAssertTrue(interceptor.tasksReceivedData[1].task === taskWithURLRequest)
AssertURLSessionTasksIdentical(interceptor.tasksReceivedData[1].task, taskWithURLRequest)
XCTAssertEqual(interceptor.tasksReceivedData[1].data, randomData)
}
}
40 changes: 24 additions & 16 deletions Tests/DatadogTests/Datadog/URLSessionAutoInstrumentation/URLSessionSwizzlerTests.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,7 @@ class URLSessionSwizzlerTests: XCTestCase {
XCTAssertNil(error)
completionHandlersCalled.fulfill()
}
taskWithURLRequestAndCompletion.taskDescription = "taskWithURLRequestAndCompletion"
taskWithURLRequestAndCompletion.resume()

let taskWithURLAndCompletion = session.dataTask(with: URL.mockAny()) { data, response, error in
Expand All @@ -270,12 +271,15 @@ class URLSessionSwizzlerTests: XCTestCase {
XCTAssertNil(error)
completionHandlersCalled.fulfill()
}
taskWithURLAndCompletion.taskDescription = "taskWithURLAndCompletion"
taskWithURLAndCompletion.resume()

let taskWithURLRequest = session.dataTask(with: URLRequest(url: .mockAny()))
taskWithURLRequest.taskDescription = "taskWithURLRequest"
taskWithURLRequest.resume()

let taskWithURL = session.dataTask(with: URL.mockAny())
taskWithURL.taskDescription = "taskWithURL"
taskWithURL.resume()

// Then
Expand All @@ -285,23 +289,23 @@ class URLSessionSwizzlerTests: XCTestCase {
XCTAssertEqual(interceptor.tasksCreated.count, 4, "Interceptor should record all 4 tasks created.")
XCTAssertEqual(interceptor.tasksCompleted.count, 4, "Interceptor should record all 4 tasks completed.")

XCTAssertTrue(interceptor.tasksCreated[0] === taskWithURLRequestAndCompletion)
XCTAssertTrue(interceptor.tasksCompleted[0].task === taskWithURLRequestAndCompletion)
AssertURLSessionTasksIdentical(interceptor.tasksCreated[0], taskWithURLRequestAndCompletion)
AssertURLSessionTasksIdentical(interceptor.tasksCompleted[0].task, taskWithURLRequestAndCompletion)
XCTAssertNil(interceptor.tasksCompleted[0].error)
XCTAssertEqual(interceptor.tasksReceivedData[0].data, expectedData)

XCTAssertTrue(interceptor.tasksCreated[1] === taskWithURLAndCompletion)
XCTAssertTrue(interceptor.tasksCompleted[1].task === taskWithURLAndCompletion)
AssertURLSessionTasksIdentical(interceptor.tasksCreated[1], taskWithURLAndCompletion)
AssertURLSessionTasksIdentical(interceptor.tasksCompleted[1].task, taskWithURLAndCompletion)
XCTAssertNil(interceptor.tasksCompleted[1].error)
XCTAssertEqual(interceptor.tasksReceivedData[1].data, expectedData)

XCTAssertTrue(interceptor.tasksCreated[2] === taskWithURLRequest)
XCTAssertTrue(interceptor.tasksCompleted[2].task === taskWithURLRequest)
AssertURLSessionTasksIdentical(interceptor.tasksCreated[2], taskWithURLRequest)
AssertURLSessionTasksIdentical(interceptor.tasksCompleted[2].task, taskWithURLRequest)
XCTAssertNil(interceptor.tasksCompleted[2].error)
XCTAssertEqual(interceptor.tasksReceivedData[2].data, expectedData)

XCTAssertTrue(interceptor.tasksCreated[3] === taskWithURL)
XCTAssertTrue(interceptor.tasksCompleted[3].task === taskWithURL)
AssertURLSessionTasksIdentical(interceptor.tasksCreated[3], taskWithURL)
AssertURLSessionTasksIdentical(interceptor.tasksCompleted[3].task, taskWithURL)
XCTAssertNil(interceptor.tasksCompleted[3].error)
XCTAssertEqual(interceptor.tasksReceivedData[3].data, expectedData)
}
Expand Down Expand Up @@ -329,6 +333,7 @@ class URLSessionSwizzlerTests: XCTestCase {
XCTAssertEqual((error! as NSError).localizedDescription, "some error")
completionHandlersCalled.fulfill()
}
taskWithURLRequestAndCompletion.taskDescription = "taskWithURLRequestAndCompletion"
taskWithURLRequestAndCompletion.resume()

let taskWithURLAndCompletion = session.dataTask(with: URL.mockAny()) { data, response, error in
Expand All @@ -337,12 +342,15 @@ class URLSessionSwizzlerTests: XCTestCase {
XCTAssertEqual((error! as NSError).localizedDescription, "some error")
completionHandlersCalled.fulfill()
}
taskWithURLAndCompletion.taskDescription = "taskWithURLAndCompletion"
taskWithURLAndCompletion.resume()

let taskWithURLRequest = session.dataTask(with: URLRequest(url: .mockAny()))
taskWithURLRequest.taskDescription = "taskWithURLRequest"
taskWithURLRequest.resume()

let taskWithURL = session.dataTask(with: URL.mockAny())
taskWithURL.taskDescription = "taskWithURL"
taskWithURL.resume()

// Then
Expand All @@ -352,20 +360,20 @@ class URLSessionSwizzlerTests: XCTestCase {
XCTAssertEqual(interceptor.tasksCreated.count, 4, "Interceptor should record all 4 tasks created.")
XCTAssertEqual(interceptor.tasksCompleted.count, 4, "Interceptor should record all 4 tasks completed.")

XCTAssertTrue(interceptor.tasksCreated[0] === taskWithURLRequestAndCompletion)
XCTAssertTrue(interceptor.tasksCompleted[0].task === taskWithURLRequestAndCompletion)
AssertURLSessionTasksIdentical(interceptor.tasksCreated[0], taskWithURLRequestAndCompletion)
AssertURLSessionTasksIdentical(interceptor.tasksCompleted[0].task, taskWithURLRequestAndCompletion)
XCTAssertEqual((interceptor.tasksCompleted[0].error! as NSError).localizedDescription, "some error")

XCTAssertTrue(interceptor.tasksCreated[1] === taskWithURLAndCompletion)
XCTAssertTrue(interceptor.tasksCompleted[1].task === taskWithURLAndCompletion)
AssertURLSessionTasksIdentical(interceptor.tasksCreated[1], taskWithURLAndCompletion)
AssertURLSessionTasksIdentical(interceptor.tasksCompleted[1].task, taskWithURLAndCompletion)
XCTAssertEqual((interceptor.tasksCompleted[1].error! as NSError).localizedDescription, "some error")

XCTAssertTrue(interceptor.tasksCreated[2] === taskWithURLRequest)
XCTAssertTrue(interceptor.tasksCompleted[2].task === taskWithURLRequest)
AssertURLSessionTasksIdentical(interceptor.tasksCreated[2], taskWithURLRequest)
AssertURLSessionTasksIdentical(interceptor.tasksCompleted[2].task, taskWithURLRequest)
XCTAssertEqual((interceptor.tasksCompleted[2].error! as NSError).localizedDescription, "some error")

XCTAssertTrue(interceptor.tasksCreated[3] === taskWithURL)
XCTAssertTrue(interceptor.tasksCompleted[3].task === taskWithURL)
AssertURLSessionTasksIdentical(interceptor.tasksCreated[3], taskWithURL)
AssertURLSessionTasksIdentical(interceptor.tasksCompleted[3].task, taskWithURL)
XCTAssertEqual((interceptor.tasksCompleted[3].error! as NSError).localizedDescription, "some error")

XCTAssertEqual(interceptor.tasksReceivedData.count, 0, "When tasks complete with failure, they should not receive data")
Expand Down
70 changes: 70 additions & 0 deletions Tests/DatadogTests/Helpers/SwiftExtensions.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,76 @@ extension URLRequest {
request.setValue(nil, forHTTPHeaderField: httpHeaderField)
return request
}

func dump() -> String {
var headersDump: String = ""
var bodyDump: String = ""

if let allHTTPHeaderFields = self.allHTTPHeaderFields {
if allHTTPHeaderFields.isEmpty {
headersDump = "[]"
} else {
headersDump = "\n"
allHTTPHeaderFields.forEach { field, value in
headersDump += "'\(field): \(value)'\n"
}
}
} else {
headersDump = "<nil>"
}

if let httpBody = self.httpBody {
bodyDump = "'''"
bodyDump += String(data: httpBody, encoding: .utf8) ?? "<invalid>"
bodyDump += "\n'''"
} else {
bodyDump = "<nil>"
}

return """
URLRequest:
- url: '\(self.url?.absoluteString ?? "<nil>")'
- headers:
\(headersDump)
- body:
\(bodyDump)
"""
}
}

extension URLSessionTask.State {
func dump() -> String {
switch self {
case .running: return "running"
case .suspended: return "suspended"
case .canceling: return "canceling"
case .completed: return "completed"
@unknown default: return "unknown"
}
}
}

extension URLSessionTask {
func dump() -> String {
func indent(string: String, by prefix: String) -> String {
return string
.split(separator: "\n")
.map { prefix + $0 }
.joined(separator: "\n")
}

return """
URLSessionTask:
- taskIdentifier: '\(self.taskIdentifier)'
- taskDescription: '\(self.taskDescription ?? "<nil>")'
- debugDescription: '\(self.debugDescription)'
- state: '\(self.state.dump())'
- originalRequest:
\(indent(string: self.originalRequest?.dump() ?? "<nil>", by: " "))
- currentRequest:
\(indent(string: self.currentRequest?.dump() ?? "<nil>", by: " "))
"""
}
}

/// Combines two arrays together, e.g. `["a", "b"].combined(with: [1, 2, 3])` gives
Expand Down
22 changes: 22 additions & 0 deletions Tests/DatadogTests/Helpers/XCTestCase.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,4 +141,26 @@ extension XCTestCase {
let value2JSONString = encodedValue2.utf8String
XCTAssertEqual(value1JSONString, value2JSONString, file: file, line: line)
}

func AssertURLSessionTasksIdentical(
_ actualTask: URLSessionTask,
_ expectedTask: URLSessionTask,
file: StaticString = #filePath,
line: UInt = #line
) {
XCTAssertTrue(
actualTask === expectedTask,
"""
Both tasks must be identical ('===').
Actual task:
\(actualTask.dump())
Expected task:
\(expectedTask.dump())
""",
file: file,
line: line
)
}
}

0 comments on commit f362f01

Please sign in to comment.