Run a periodic crash-test (#590)

* Run a periodic crash-test

* Better documentation

.github/workflows/crash_test.yaml

@@ -0,0 +1,20 @@
+# Run against all markdown files in latest version of packages on pub.dev to
+# see if any can provoke a crash
+
+name: Crash Tests
+
+on:
+  schedule:
+    # “At 00:00 (UTC) on Sunday.”
+    - cron: '0 0 * * 0'
+
+jobs:
+  crash-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+      - uses: dart-lang/setup-dart@fedb1266e91cf51be2fdb382869461a434b920a3
+      - name: Install dependencies
+        run: dart pub get
+      - name: Run crash_test.dart
+        run: dart test -P crash_test test/crash_test.dart

test/crash_test.dart

@@ -4,6 +4,7 @@
 
 import 'dart:convert';
 import 'dart:io';
+import 'dart:isolate';
 
 import 'package:http/http.dart' as http;
 import 'package:http/retry.dart' as http;
@@ -14,6 +15,18 @@ import 'package:test/test.dart';
 
 // ignore_for_file: avoid_dynamic_calls
 
+const extensions = [
+  '.md',
+  '.mkd',
+  '.mdwn',
+  '.mdown',
+  '.mdtxt',
+  '.mdtext',
+  '.markdown',
+  'README',
+  'CHANGELOG',
+];
+
 void main() async {
   // This test is a really dumb and very slow crash-test.
   // It downloads the latest package version for each package on pub.dev
@@ -26,6 +39,16 @@ void main() async {
   test(
     'crash test',
     () async {
+      final started = DateTime.now();
+      var lastStatus = DateTime(0);
+      void status(String Function() message) {
+        if (DateTime.now().difference(lastStatus) >
+            const Duration(seconds: 30)) {
+          lastStatus = DateTime.now();
+          print(message());
+        }
+      }
+
       final c = http.RetryClient(http.Client());
       Future<dynamic> getJson(String url) async {
         final u = Uri.tryParse(url);
@@ -50,79 +73,154 @@ void main() async {
           ((await getJson('https://pub.dev/api/package-names'))['packages']
                   as List)
               .cast<String>();
-      print('Found ${packages.length} packages to scan');
+      //.take(3).toList(); // useful when testing
+      print('## Found ${packages.length} packages to scan');
 
-      final errors = <String>[];
-      final pool = Pool(50);
       var count = 0;
-      var skipped = 0;
-      var lastStatus = DateTime.now();
+      final pool = Pool(50);
+      final packageVersions = <PackageVersion>[];
       await Future.wait(packages.map((package) async {
         await pool.withResource(() async {
-          final versionsResponse =
-              await getJson('https://pub.dev/api/packages/$package');
-          final archiveUrl = Uri.tryParse(
-            versionsResponse['latest']?['archive_url'] as String? ?? '',
+          final response = await getJson(
+            'https://pub.dev/api/packages/$package',
           );
+          final entry = response['latest'] as Map?;
+          if (entry != null) {
+            packageVersions.add(PackageVersion(
+              package: package,
+              version: entry['version'] as String,
+              archiveUrl: entry['archive_url'] as String,
+            ));
+          }
+          count++;
+          status(
+            () => 'Listed versions for $count / ${packages.length} packages',
+          );
+        });
+      }));
+
+      print('## Found ${packageVersions.length} package versions to scan');
+
+      count = 0;
+      final errors = <String>[];
+      var skipped = 0;
+      await Future.wait(packageVersions.map((pv) async {
+        await pool.withResource(() async {
+          final archiveUrl = Uri.tryParse(pv.archiveUrl);
           if (archiveUrl == null) {
             skipped++;
             return;
           }
           late List<int> archive;
           try {
-            archive = gzip.decode(await c.readBytes(archiveUrl));
+            archive = await c.readBytes(archiveUrl);
           } on http.ClientException {
             skipped++;
             return;
           } on IOException {
             skipped++;
             return;
           }
-          try {
-            await TarReader.forEach(Stream.value(archive), (entry) async {
-              if (entry.name.endsWith('.md')) {
-                late String contents;
-                try {
-                  final bytes = await http.ByteStream(entry.contents).toBytes();
-                  contents = utf8.decode(bytes);
-                } on FormatException {
-                  return; // ignore invalid utf8
-                }
-                try {
-                  markdownToHtml(
-                    contents,
-                    extensionSet: ExtensionSet.gitHubWeb,
-                  );
-                } catch (err, st) {
-                  errors
-                      .add('package:$package/${entry.name}, throws: $err\n$st');
-                }
-              }
-            });
-          } on FormatException {
+
+          final result = await _findMarkdownIssues(
+            pv.package,
+            pv.version,
+            archive,
+          );
+
+          // If tar decoding fails.
+          if (result == null) {
             skipped++;
             return;
           }
+
+          errors.addAll(result);
+          result.forEach(print);
         });
         count++;
-        if (DateTime.now().difference(lastStatus) >
-            const Duration(seconds: 30)) {
-          lastStatus = DateTime.now();
-          print('Scanned $count / ${packages.length} (skipped $skipped),'
-              ' found ${errors.length} issues');
-        }
+        status(() =>
+            'Scanned $count / ${packageVersions.length} (skipped $skipped),'
+            ' found ${errors.length} issues');
       }));
 
       await pool.close();
       c.close();
 
+      print('## Finished scanning');
+      print('Scanned ${packageVersions.length} package versions in '
+          '${DateTime.now().difference(started)}');
+
       if (errors.isNotEmpty) {
         print('Found issues:');
         errors.forEach(print);
         fail('Found ${errors.length} cases where markdownToHtml threw!');
       }
     },
-    timeout: const Timeout(Duration(hours: 1)),
+    timeout: const Timeout(Duration(hours: 5)),
     tags: 'crash_test', // skipped by default, see: dart_test.yaml
   );
 }
+
+class PackageVersion {
+  final String package;
+  final String version;
+  final String archiveUrl;
+
+  PackageVersion({
+    required this.package,
+    required this.version,
+    required this.archiveUrl,
+  });
+}
+
+/// Scans [gzippedArchive] for markdown files and tries to parse them all.
+///
+/// Creates a list of issues that arose when parsing markdown files. The
+/// [package] and [version] strings are used to construct nice issues.
+/// An issue string may be multi-line, but should be printable.
+///
+/// Returns a list of issues, or `null` if decoding and parsing [gzippedArchive]
+/// failed.
+Future<List<String>?> _findMarkdownIssues(
+  String package,
+  String version,
+  List<int> gzippedArchive,
+) async {
+  return Isolate.run<List<String>?>(() async {
+    try {
+      final archive = gzip.decode(gzippedArchive);
+      final issues = <String>[];
+      await TarReader.forEach(Stream.value(archive), (entry) async {
+        if (extensions.any((ext) => entry.name.endsWith(ext))) {
+          late String contents;
+          try {
+            final bytes = await http.ByteStream(entry.contents).toBytes();
+            contents = utf8.decode(bytes);
+          } on FormatException {
+            return; // ignore invalid utf8
+          }
+          final start = DateTime.now();
+          try {
+            markdownToHtml(
+              contents,
+              extensionSet: ExtensionSet.gitHubWeb,
+            );
+          } catch (err, st) {
+            issues.add(
+                'package:$package-$version/${entry.name}, throws: $err\n$st');
+          }
+          final time = DateTime.now().difference(start);
+          if (time.inSeconds > 30) {
+            issues.add(
+                'package:$package-$version/${entry.name} took $time to process');
+          }
+        }
+      });
+      return issues;
+    } on FormatException {
+      return null;
+    }
+  }).timeout(const Duration(minutes: 2), onTimeout: () {
+    return ['package:$package-$version failed to be processed in 2 minutes'];
+  });
+}