Fix composite determinism (#360)

pyproject.toml

@@ -16,7 +16,8 @@ dependencies = [
   "soundfile==0.12.1",
   "mido==1.3.2",
   "jsonschema==4.21.1",
-  "numpy==1.24.4",
+  "numpy==1.24.4; python_version < \"3.12\"",
+  "numpy==1.26.4; python_version >= \"3.12\"",
   "pyjacklib==0.1.1",
   "pyside6-essentials==6.6.2",
   "lupa==2.1",

src/shoopdaloop/lib/q_objects/CompositeLoop.py

@@ -40,11 +40,13 @@ def __init__(self, parent=None):
         self._n_cycles = 0
         self._kind = 'regular'
         self._pending_transitions = []
-        self._pending_cycles = 0
+        self._pending_cycles = []
         self._backend = None
         self._initialized = False
         self._play_after_record = False
         self._sync_mode_active = False
+        self._cycle_nr = 0
+        self._last_handled_source_cycle_nr = -1
 
         self.scheduleChangedUnsafe.connect(self.scheduleChanged, Qt.QueuedConnection)
         self.nCyclesChangedUnsafe.connect(self.nCyclesChanged, Qt.QueuedConnection)
@@ -70,8 +72,8 @@ def __init__(self, parent=None):
         self.backendChanged.connect(lambda: self.maybe_initialize())
         self.backendInitializedChanged.connect(lambda: self.maybe_initialize())
 
-    cycled = ShoopSignal()
-    cycledUnsafe = ShoopSignal()
+    cycled = ShoopSignal(int)
+    cycledUnsafe = ShoopSignal(int)
 
     ######################
     ## PROPERTIES
@@ -445,10 +447,10 @@ def transition_with_immediate_sync_impl(self, mode, sync_cycle):
         # Perform the trigger(s) for the next loop cycle
         self.do_triggers(self.iteration + 1, mode)
 
-    @ShoopSlot(thread_protection=ThreadProtectionType.AnyThread)
-    def handle_sync_loop_trigger(self):
+    @ShoopSlot(int, thread_protection=ThreadProtectionType.AnyThread)
+    def handle_sync_loop_trigger(self, cycle_nr):
         self.logger.trace(lambda: f'queue sync loop trigger')
-        self._pending_cycles += 1
+        self._pending_cycles.append(cycle_nr)
 
     @ShoopSlot('QVariant', 'QVariant', 'QVariant', int)
     def adopt_ringbuffers(self, reverse_start_cycle, cycles_length, go_to_cycle, go_to_mode):
@@ -504,8 +506,27 @@ def adopt_ringbuffers(self, reverse_start_cycle, cycles_length, go_to_cycle, go_
         if go_to_mode != LoopMode.Unknown.value:
             self.transition(go_to_mode, DontWaitForSync, go_to_cycle)
 
-    def handle_sync_loop_trigger_impl(self):
-        self.logger.debug(lambda: 'handle sync cycle')
+    def all_loops(self):
+        loops = set()
+        for elem in self._schedule.values():
+            for e in elem['loops_start']:
+                loops.add(e[0])
+            for e in elem['loops_end']:
+                loops.add(e)
+            for e in elem['loops_ignored']:
+                loops.add(e)
+        return loops
+
+    def handle_sync_loop_trigger_impl(self, cycle_nr):
+        if cycle_nr == self._last_handled_source_cycle_nr:
+            self.logger.trace(lambda: f'already handled sync cycle {cycle_nr}, skipping')
+            return
+        self.logger.debug(lambda: f'handle sync cycle {cycle_nr}')
+
+        # Before we start, give any of the loops in our schedule a chance to handle the cycle
+        # first. This ensures a deterministic ordering of execution.
+        for l in self.all_loops():
+            l.dependent_will_handle_sync_loop_cycle(cycle_nr)
 
         if self._next_transition_delay == 0:
             self.handle_transition(self.next_mode)
@@ -525,8 +546,10 @@ def handle_sync_loop_trigger_impl(self):
             self.do_triggers(self.iteration+1, self.mode)
 
             if cycled:
-                self.cycledUnsafe.emit()
+                self._cycle_nr += 1
+                self.cycledUnsafe.emit(self._cycle_nr)
 
+        self._last_handled_source_cycle_nr = cycle_nr
         self.logger.trace(lambda: 'handle sync cycle done')
 
     def cancel_all(self):
@@ -631,10 +654,16 @@ def maybe_initialize(self):
     @ShoopSlot(thread_protection = ThreadProtectionType.OtherThread)
     def updateOnOtherThread(self):
         if self._backend:
-            for _ in range(self._pending_cycles):
-                self.handle_sync_loop_trigger_impl()
+            for cycle_nr in self._pending_cycles:
+                self.handle_sync_loop_trigger_impl(cycle_nr)
             for transition in self._pending_transitions:
                 self.transition_impl(*transition)
             self._pending_transitions = []
-            self._pending_cycles = 0
+            self._pending_cycles = []
+
+    # Another loop which references this loop (composite) can notify this loop that it is
+    # about to handle a sync loop cycle in advance, to ensure a deterministic ordering.
+    @ShoopSlot(int, thread_protection = ThreadProtectionType.OtherThread)
+    def dependent_will_handle_sync_loop_cycle(self, cycle_nr):
+        self.handle_sync_loop_trigger_impl(cycle_nr)
 

src/shoopdaloop/lib/q_objects/FileIO.py

@@ -106,7 +106,7 @@ def make_tarfile(self, filename, source_dir, compress):
     def extract_tarfile(self, filename, target_dir):
         flags = "r:*"
         with tarfile.open(filename, flags) as tar:
-            tar.extractall(target_dir)
+            tar.extractall(target_dir, filter='fully_trusted')
 
     def save_data_to_soundfile_impl(self, filename, sample_rate, data):
         self.startSavingFile.emit()

src/shoopdaloop/lib/q_objects/Loop.py

@@ -24,8 +24,8 @@
 # Wraps a back-end loop.
 class Loop(FindParentBackend):
     # Other signals
-    cycled = ShoopSignal()
-    cycledUnsafe = ShoopSignal(thread_protection=ThreadProtectionType.OtherThread)
+    cycled = ShoopSignal(int)
+    cycledUnsafe = ShoopSignal(int, thread_protection=ThreadProtectionType.OtherThread)
 
     def __init__(self, parent=None):
         super(Loop, self).__init__(parent)
@@ -44,6 +44,7 @@ def __init__(self, parent=None):
         self.logger.name = "Frontend.Loop"
         self._pending_transitions = []
         self._pending_cycles = 0
+        self._cycle_nr = 0
 
         self.modeChangedUnsafe.connect(self.modeChanged, Qt.QueuedConnection)
         self.nextModeChangedUnsafe.connect(self.nextModeChanged, Qt.QueuedConnection)
@@ -276,8 +277,9 @@ def updateOnOtherThread(self):
         self._pending_transitions = []
 
         if (self.position < prev_position and is_playing_mode(prev_mode) and is_playing_mode(self.mode)):
-            self.logger.debug(lambda: 'cycled')
-            self.cycledUnsafe.emit()
+            self._cycle_nr += 1
+            self.logger.debug(lambda: f'cycled -> nr {self._cycle_nr}')
+            self.cycledUnsafe.emit(self._cycle_nr)
 
     # Update on GUI thread.
     @ShoopSlot()
@@ -379,6 +381,12 @@ def get_backend_loop(self):
     @ShoopSlot(result="QVariant")
     def py_loop(self):
         return self
+
+    # Another loop which references this loop (composite) can notify this loop that it is
+    # about to handle a sync loop cycle in advance, to ensure a deterministic ordering.
+    @ShoopSlot(int, thread_protection = ThreadProtectionType.OtherThread)
+    def dependent_will_handle_sync_loop_cycle(self, cycle_nr):
+        pass
 
     def maybe_initialize(self):
         if self._backend and self._backend.initialized and not self._backend_loop:

src/shoopdaloop/lib/qml/CompositeLoop.qml

@@ -47,7 +47,7 @@ Item {
         play_after_record: registries.state_registry.play_after_record_active
         sync_mode_active: registries.state_registry.sync_active
 
-        onCycled: root.cycled()
+        onCycled: n => root.cycled(n)
         Component.onCompleted: root.recalculate_schedule()
     }
 
@@ -127,7 +127,7 @@ Item {
         instanceIdentifier: obj_id
     }
 
-    signal cycled()
+    signal cycled(int cycle_nr)
 
     function add_loop(loop, delay, n_cycles=undefined, playlist_idx=undefined) {
         root.logger.debug(`Adding loop ${loop.obj_id} to playlist ${playlist_idx} with delay ${delay}, n_cycles override ${n_cycles}`)

src/shoopdaloop/lib/qml/LoopWidget.qml

@@ -226,7 +226,7 @@ Item {
     }
 
     // Signals
-    signal cycled
+    signal cycled(int cycle_nr)
 
     // Methods
     function transition_loops(loops, mode, maybe_delay, maybe_align_to_sync_at) {

src/shoopdaloop/lib/qml/test/tst_CompositeLoop_scheduling.qml

@@ -17,7 +17,7 @@ ShoopTestFile {
 
         property var maybe_loop: this
 
-        signal cycled()
+        signal cycled(int cycle_nr)
 
         RegisterInRegistry {
             object: fakeloop