Fix read_next_event logic for calibration files not containing eny mc…

… shower/event data

src/eventio/base.py

@@ -62,6 +62,7 @@ def __init__(self, path, zcat=True):
         self.read_process = None
         self.zstd = False
         self.next = None
+        self.peek_error = None
 
         if not is_eventio(path):
             raise ValueError('File {} is not an eventio file'.format(path))
@@ -127,7 +128,8 @@ def peek(self):
         if self.next is None:
             try:
                 self.next = next(self)
-            except (StopIteration, EOFError, IOError):
+            except (StopIteration, EOFError, IOError) as e:
+                self.peek_error = e
                 self.next = None
 
         return self.next

src/eventio/simtel/simtelfile.py

@@ -143,6 +143,7 @@ def __init__(
         self.current_mc_shower_id = None
         self.current_mc_event = None
         self.current_mc_event_id = None
+        self.current_array_event_id = None
         self.current_telescope_data_event_id = None
         self.current_photoelectron_sum = None
         self.current_photoelectron_sum_event_id = None
@@ -205,8 +206,17 @@ def _read_next_event(self):
 
         if isinstance(self._file.peek(), (MCEvent, CalibrationEvent)):
             self._parse_next_object()
+            self._read_until_next_event()
             return self._build_event()
 
+        # extracted calibration events have "naked" ArrayEvents without
+        # a preceding MCEvent or CalibrationEvent wrapper
+        if isinstance(self._file.peek(), ArrayEvent):
+            self._parse_next_object()
+            return self._build_event()
+
+        raise ValueError(f"Unexpected obj type: {self._file.peek()}")
+
     def _check_skip(self, event):
         if event['type'] == 'data':
             return self.skip_non_triggered and not event.get('telescope_events')
@@ -236,6 +246,7 @@ def _parse_next_object(self):
             self.current_mc_shower_id = o.header.id
 
         elif isinstance(o, ArrayEvent):
+            self.current_array_event_id = o.header.id
             self.current_array_event = parse_array_event(
                 o,
                 self.allowed_telescopes
@@ -328,14 +339,16 @@ def _build_event(self):
         '''check if all necessary info for an event was found,
         then make an event and invalidate old data
         '''
-        self._read_until_next_event()
 
         # data by default, might be overriden by calibration
         event = {'type': 'data'}
 
         if self.current_array_event:
             event.update(self.current_array_event)
+            if self.current_array_event_id is not None:
+                event["event_id"] = self.current_array_event_id
             self.current_array_event = {}
+            self.current_array_event_id = None
 
             if 'telescope_events' not in event:
                 return event
@@ -363,16 +376,17 @@ def _build_event(self):
                 # no further info for calib events
                 return event
 
-        # this information should always exist
-        event.update({
-            'event_id': self.current_mc_event_id,
-            'mc_shower': self.current_mc_shower,
-            'mc_event': self.current_mc_event,
-            'photons': self.current_photons,
-            'emitter': self.current_emitter,
-            'photoelectrons': self.current_photoelectrons,
-            'photoelectron_sums': self.current_photoelectron_sum,
-        })
+        # update with mc data if available
+        if self.current_mc_shower:
+            event.update({
+                'event_id': self.current_mc_event_id,
+                'mc_shower': self.current_mc_shower,
+                'mc_event': self.current_mc_event,
+                'photons': self.current_photons,
+                'emitter': self.current_emitter,
+                'photoelectrons': self.current_photoelectrons,
+                'photoelectron_sums': self.current_photoelectron_sum,
+            })
 
         return event
 

tests/simtel/test_simtelfile.py

@@ -1,3 +1,4 @@
+import pytest
 from pytest import importorskip
 from eventio.simtel import SimTelFile
 import numpy as np
@@ -19,89 +20,89 @@
 test_paths = [prod2_path, prod3_path, prod4_path]
 
 
-def test_can_open():
-    for path in test_paths:
-        assert SimTelFile(path)
-
-
-def test_at_least_one_event_found():
-    for path in test_paths:
-        one_found = False
-        for event in SimTelFile(path):
-            one_found = True
-            break
-        assert one_found, path
-
-
-def test_show_we_get_a_dict_with_hower_and_event():
-    for path in test_paths:
-        for event in SimTelFile(path):
-            assert 'mc_shower' in event
-            assert 'telescope_events' in event
-            assert 'mc_event' in event
-            break
-
-
-def test_show_event_is_not_empty_and_has_some_members_for_sure():
-    for path in test_paths:
-        for event in SimTelFile(path):
-            assert event['mc_shower'].keys() == {
-                'shower',
-                'primary_id',
-                'energy',
-                'azimuth',
-                'altitude',
-                'depth_start',
-                'h_first_int',
-                'xmax',
-                'hmax',
-                'emax',
-                'cmax',
-                'n_profiles',
-                'profiles'
+@pytest.mark.parametrize("path", test_paths)
+def test_can_open(path):
+    assert SimTelFile(path)
+
+
+@pytest.mark.parametrize("path", test_paths)
+def test_at_least_one_event_found(path):
+    one_found = False
+    for event in SimTelFile(path):
+        one_found = True
+        break
+    assert one_found, path
+
+
+@pytest.mark.parametrize("path", test_paths)
+def test_show_we_get_a_dict_with_hower_and_event(path):
+    for event in SimTelFile(path):
+        assert 'mc_shower' in event
+        assert 'telescope_events' in event
+        assert 'mc_event' in event
+        break
+
+
+@pytest.mark.parametrize("path", test_paths)
+def test_show_event_is_not_empty_and_has_some_members_for_sure(path):
+    for event in SimTelFile(path):
+        assert event['mc_shower'].keys() == {
+            'shower',
+            'primary_id',
+            'energy',
+            'azimuth',
+            'altitude',
+            'depth_start',
+            'h_first_int',
+            'xmax',
+            'hmax',
+            'emax',
+            'cmax',
+            'n_profiles',
+            'profiles'
+        }
+
+        assert event.keys() == {
+            'type',
+            'event_id',
+            'mc_shower',
+            'mc_event',
+            'telescope_events',
+            'trigger_information',
+            'tracking_positions',
+            'photoelectron_sums',
+            'photoelectrons',
+            'photons',
+            'emitter',
+            'camera_monitorings',
+            'laser_calibrations',
+            'pixel_monitorings',
+        }
+
+        telescope_events = event['telescope_events']
+
+        assert telescope_events  # never empty!
+
+        for telescope_event in telescope_events.values():
+            expected_keys = {
+                'header',
+                'pixel_timing',
+                'pixel_lists',
             }
-
-            assert event.keys() == {
-                'type',
-                'event_id',
-                'mc_shower',
-                'mc_event',
-                'telescope_events',
-                'trigger_information',
-                'tracking_positions',
-                'photoelectron_sums',
-                'photoelectrons',
-                'photons',
-                'emitter',
-                'camera_monitorings',
-                'laser_calibrations',
-                'pixel_monitorings',
+            allowed_keys = {
+                'image_parameters',
+                'adc_sums',
+                'adc_samples'
             }
 
-            telescope_events = event['telescope_events']
+            found_keys = set(telescope_event.keys())
+            assert expected_keys.issubset(found_keys)
 
-            assert telescope_events  # never empty!
+            extra_keys = found_keys.difference(expected_keys)
+            assert extra_keys.issubset(allowed_keys)
+            assert 'adc_sums' in found_keys or 'adc_samples' in found_keys
 
-            for telescope_event in telescope_events.values():
-                expected_keys = {
-                    'header',
-                    'pixel_timing',
-                    'pixel_lists',
-                }
-                allowed_keys = {
-                    'image_parameters',
-                    'adc_sums',
-                    'adc_samples'
-                }
-
-                found_keys = set(telescope_event.keys())
-                assert expected_keys.issubset(found_keys)
-
-                extra_keys = found_keys.difference(expected_keys)
-                assert extra_keys.issubset(allowed_keys)
-                assert 'adc_sums' in found_keys or 'adc_samples' in found_keys
-
-            break
+        break
 
 
 def test_iterate_complete_file():
@@ -276,3 +277,13 @@ def test_type_2033():
                     'flags', 'n_pixels', 'n_gains', 'nsb_rate', 'qe_rel', 'gain_rel',
                     'hv_rel', 'current', 'fadc_amp_hg', 'disabled',
                 }
+
+
+def test_extracted_pedestals():
+    with SimTelFile("tests/resources/extracted_pedestals.simtel.zst") as f:
+        expected_event_id = 0
+        for e in f:
+            expected_event_id += 1
+            assert e["event_id"] == expected_event_id
+
+        assert expected_event_id == 5