Merge pull request #45 from cloudscale-ch/denis/summary-with-setup-er…

…rors

Include errors in the setup phase in the summary

.github/workflows/run-acceptance-tests.yml

@@ -46,13 +46,15 @@ jobs:
           python -m pip install --upgrade pip
           pip install -r requirements.txt
 
+      - name: Remove GitHub problem matchers
+        run: >
+          echo "::remove-matcher owner=python::";
+          echo "::remove-matcher owner=checkout-git::"
+
       - name: Run tests
         env:
           CLOUDSCALE_API_TOKEN: ${{ secrets.api_token }}
-        # The ::remove-matcher call ensures that automatic annotations are
-        # not applied to the pytest output. We summarize the output ourselves.
         run: >
-          echo "::remove-matcher owner=python::";
           ./pytest "${{ inputs.path || '.' }}"
           --zone="${{ inputs.zone }}"
           -n ${{ inputs.workers }}

conftest.py

@@ -306,20 +306,13 @@ def pytest_runtest_logreport(report):
     if os.environ.get('PYTEST_XDIST_MASTER') == '1':
         return
 
-    if report.when == 'setup':
-        trigger('test.setup', name=report.nodeid, outcome=report.outcome)
-    elif report.when == 'call':
-        trigger(
-            'test.call',
-            name=report.nodeid,
-            outcome=report.outcome,
-            error=report.longreprtext,
-            short_error=extract_short_error(report.longreprtext),
-        )
-    elif report.when == 'teardown':
-        trigger('test.teardown', name=report.nodeid, outcome=report.outcome)
-    else:
-        raise NotImplementedError(f"Unsupported report stage: {report.when}")
+    trigger(
+        f'test.{report.when}',
+        name=report.nodeid,
+        outcome=report.outcome,
+        error=report.longreprtext,
+        short_error=extract_short_error(report.longreprtext),
+    )
 
 
 @pytest.fixture(scope='session')

events.py

@@ -197,22 +197,25 @@ def on_record_event(**attributes):
 
 
 # Keep track of test items, recording their start and their result
+track_in_event_log('test.start')
+
+
 @OBS.on('test.start')
-def on_test_setup(name):
+def on_test_start(name):
     CTX.current_test = name.split('::')[-1]
 
 
 @OBS.on('test.teardown')
-def on_test_teardown(name, outcome):
+def on_test_teardown(name, outcome, error, short_error):
     CTX.current_test = None
 
 
-track_in_event_log('test.start')
-track_in_event_log('test.call', include={
-    'outcome': 'outcome',
-    'error': 'error',
-    'short_error': 'short_error',
-})
+for phase in ('call', 'setup', 'teardown'):
+    track_in_event_log(f'test.{phase}', include={
+        'outcome': 'outcome',
+        'error': 'error',
+        'short_error': 'short_error',
+    })
 
 
 # Keep track of API requests

tasks.py

@@ -138,12 +138,17 @@ def summary(c):
     for log in sorted(Path('events').glob(pattern), reverse=True):
         with log.open('r') as f:
             for line in f:
-                event = json.loads(line)
+                e = json.loads(line)
 
-                if event.get('event') != 'test.call':
+                if e.get('event') not in ('test.setup', 'test.call'):
                     continue
 
-                results.append(event)
+                # There won't be an additional phase if the setup fails
+                if e['event'] == 'test.setup' and e['outcome'] != 'passed':
+                    results.append(e)
+
+                if e['event'] == 'test.call':
+                    results.append(e)
 
         if results and results[-1]['run'] == 1:
             break