From 7285fdd2fed8682b8d0ce490413785c9c34a21f8 Mon Sep 17 00:00:00 2001 From: Jia Hao Date: Thu, 27 Aug 2020 08:01:56 +0800 Subject: [PATCH] [integration_test] Show stack trace of widget test errors on the platform side (#2967) * [integration_test] Show stack trace of widget test errors on the platform side We keep things simple for now, because this might change once we have a proper test reporter. --- packages/integration_test/CHANGELOG.md | 4 +++ .../integration_test/FlutterTestRunner.java | 2 +- .../ios/Classes/IntegrationTestIosTest.m | 2 +- packages/integration_test/lib/common.dart | 8 +++-- .../lib/integration_test.dart | 32 +++++++++++-------- packages/integration_test/pubspec.yaml | 2 +- .../test/binding_fail_test.dart | 19 +++++------ 7 files changed, 38 insertions(+), 31 deletions(-) diff --git a/packages/integration_test/CHANGELOG.md b/packages/integration_test/CHANGELOG.md index cbd5282028f4..6e5e4bb3d51b 100644 --- a/packages/integration_test/CHANGELOG.md +++ b/packages/integration_test/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.8.1 + +* Show stack trace of widget test errors on the platform side + ## 0.8.0 * Rename plugin to integration_test. diff --git a/packages/integration_test/android/src/main/java/dev/flutter/plugins/integration_test/FlutterTestRunner.java b/packages/integration_test/android/src/main/java/dev/flutter/plugins/integration_test/FlutterTestRunner.java index 511fc141a917..fc6b99b30f64 100644 --- a/packages/integration_test/android/src/main/java/dev/flutter/plugins/integration_test/FlutterTestRunner.java +++ b/packages/integration_test/android/src/main/java/dev/flutter/plugins/integration_test/FlutterTestRunner.java @@ -77,7 +77,7 @@ public void run(RunNotifier notifier) { Description d = Description.createTestDescription(testClass, name); notifier.fireTestStarted(d); String outcome = results.get(name); - if (outcome.equals("failed")) { + if (!outcome.equals("success")) { Exception dummyException = new Exception(outcome); notifier.fireTestFailure(new Failure(d, dummyException)); } diff --git a/packages/integration_test/ios/Classes/IntegrationTestIosTest.m b/packages/integration_test/ios/Classes/IntegrationTestIosTest.m index 4243cdd86bc0..1397f547e6f6 100644 --- a/packages/integration_test/ios/Classes/IntegrationTestIosTest.m +++ b/packages/integration_test/ios/Classes/IntegrationTestIosTest.m @@ -26,7 +26,7 @@ - (BOOL)testIntegrationTest:(NSString **)testResult { NSLog(@"%@ passed.", test); [passedTests addObject:test]; } else { - NSLog(@"%@ failed.", test); + NSLog(@"%@ failed: %@", test, result); [failedTests addObject:test]; } } diff --git a/packages/integration_test/lib/common.dart b/packages/integration_test/lib/common.dart index cea67d189307..789b1fa54948 100644 --- a/packages/integration_test/lib/common.dart +++ b/packages/integration_test/lib/common.dart @@ -78,7 +78,7 @@ class Response { } _failureDetails.forEach((Failure f) { - list.add(f.toString()); + list.add(f.toJson()); }); return list; @@ -107,14 +107,16 @@ class Failure { Failure(this.methodName, this.details); /// Serializes the object to JSON. - @override - String toString() { + String toJson() { return json.encode({ 'methodName': methodName, 'details': details, }); } + @override + String toString() => toJson(); + /// Decode a JSON string to create a Failure object. static Failure fromJsonString(String jsonString) { Map failure = json.decode(jsonString); diff --git a/packages/integration_test/lib/integration_test.dart b/packages/integration_test/lib/integration_test.dart index 28d0da320486..430b7ee38510 100644 --- a/packages/integration_test/lib/integration_test.dart +++ b/packages/integration_test/lib/integration_test.dart @@ -13,6 +13,8 @@ import 'package:flutter/widgets.dart'; import 'common.dart'; import '_extension_io.dart' if (dart.library.html) '_extension_web.dart'; +const String _success = 'success'; + /// A subclass of [LiveTestWidgetsFlutterBinding] that reports tests results /// on a channel to adapt them to native instrumentation test format. class IntegrationTestWidgetsFlutterBinding @@ -33,7 +35,14 @@ class IntegrationTestWidgetsFlutterBinding } await _channel.invokeMethod( 'allTestsFinished', - {'results': results}, + { + 'results': results.map((name, result) { + if (result is Failure) { + return MapEntry(name, result.details); + } + return MapEntry(name, result); + }) + }, ); } on MissingPluginException { print('Warning: integration_test test plugin was not detected.'); @@ -46,8 +55,7 @@ class IntegrationTestWidgetsFlutterBinding final TestExceptionReporter oldTestExceptionReporter = reportTestException; reportTestException = (FlutterErrorDetails details, String testDescription) { - results[testDescription] = 'failed'; - _failureMethodsDetails.add(Failure(testDescription, details.toString())); + results[testDescription] = Failure(testDescription, details.toString()); if (!_allTestsPassed.isCompleted) { _allTestsPassed.complete(false); } @@ -95,17 +103,11 @@ class IntegrationTestWidgetsFlutterBinding final Completer _allTestsPassed = Completer(); - /// Stores failure details. - /// - /// Failed test method's names used as key. - final List _failureMethodsDetails = List(); - /// Similar to [WidgetsFlutterBinding.ensureInitialized]. /// /// Returns an instance of the [IntegrationTestWidgetsFlutterBinding], creating and /// initializing it if necessary. static WidgetsBinding ensureInitialized() { - print('TESTING123 ensuring init'); if (WidgetsBinding.instance == null) { IntegrationTestWidgetsFlutterBinding(); } @@ -118,10 +120,12 @@ class IntegrationTestWidgetsFlutterBinding /// Test results that will be populated after the tests have completed. /// - /// Keys are the test descriptions, and values are either `success` or - /// `failed`. + /// Keys are the test descriptions, and values are either [_success] or + /// a [Failure]. @visibleForTesting - Map results = {}; + Map results = {}; + + List get _failures => results.values.whereType().toList(); /// The extra data for the reported result. /// @@ -143,7 +147,7 @@ class IntegrationTestWidgetsFlutterBinding 'message': allTestsPassed ? Response.allTestsPassed(data: reportData).toJson() : Response.someTestsFailed( - _failureMethodsDetails, + _failures, data: reportData, ).toJson(), }; @@ -185,6 +189,6 @@ class IntegrationTestWidgetsFlutterBinding description: description, timeout: timeout, ); - results[description] ??= 'success'; + results[description] ??= _success; } } diff --git a/packages/integration_test/pubspec.yaml b/packages/integration_test/pubspec.yaml index b222531044e9..c1514146c869 100644 --- a/packages/integration_test/pubspec.yaml +++ b/packages/integration_test/pubspec.yaml @@ -1,6 +1,6 @@ name: integration_test description: Runs tests that use the flutter_test API as integration tests. -version: 0.8.0 +version: 0.8.1 homepage: https://github.com/flutter/plugins/tree/master/packages/integration_test environment: diff --git a/packages/integration_test/test/binding_fail_test.dart b/packages/integration_test/test/binding_fail_test.dart index 020fb9607608..bb5961b18fc7 100644 --- a/packages/integration_test/test/binding_fail_test.dart +++ b/packages/integration_test/test/binding_fail_test.dart @@ -8,6 +8,7 @@ import 'package:flutter_test/flutter_test.dart'; const String _flutterBin = 'flutter'; const String _integrationResultsPrefix = 'IntegrationTestWidgetsFlutterBinding test results:'; +const String _failureExcerpt = 'Expected: \\n Actual: '; void main() async { group('Integration binding result', () { @@ -27,24 +28,20 @@ void main() async { final Map results = await _runTest('test/data/fail_test_script.dart'); + expect(results, hasLength(2)); expect( - results, - equals({ - 'failing test 1': 'failed', - 'failing test 2': 'failed', - })); + results, containsPair('failing test 1', contains(_failureExcerpt))); + expect( + results, containsPair('failing test 2', contains(_failureExcerpt))); }); test('when one test passes, then another fails', () async { final Map results = await _runTest('test/data/pass_then_fail_test_script.dart'); - expect( - results, - equals({ - 'passing test': 'success', - 'failing test': 'failed', - })); + expect(results, hasLength(2)); + expect(results, containsPair('passing test', equals('success'))); + expect(results, containsPair('failing test', contains(_failureExcerpt))); }); }); }