Use asyncio.Lock for forward_model_ok

This makes sure that we will not run more than 1 internalization job at a time.

src/ert/scheduler/job.py

@@ -223,17 +223,20 @@ async def _verify_checksum(self, timeout: int = 120) -> None:  # noqa: ASYNC109
                 logger.error(f"Disk synchronization failed for {file_path}")
 
     async def _handle_finished_forward_model(self) -> None:
-        callback_status, status_msg = await forward_model_ok(self.real.run_arg)
-        if self._callback_status_msg:
-            self._callback_status_msg = status_msg
-        else:
-            self._callback_status_msg += f"\nstatus from done callback: {status_msg}"
-
-        if callback_status == LoadStatus.LOAD_SUCCESSFUL:
-            await self._send(JobState.COMPLETED)
-        else:
-            assert callback_status == LoadStatus.LOAD_FAILURE
-            await self._send(JobState.FAILED)
+        async with self._scheduler._fmok_lock:
+            callback_status, status_msg = await forward_model_ok(self.real.run_arg)
+            if self._callback_status_msg:
+                self._callback_status_msg = status_msg
+            else:
+                self._callback_status_msg += (
+                    f"\nstatus from done callback: {status_msg}"
+                )
+
+            if callback_status == LoadStatus.LOAD_SUCCESSFUL:
+                await self._send(JobState.COMPLETED)
+            else:
+                assert callback_status == LoadStatus.LOAD_FAILURE
+                await self._send(JobState.FAILED)
 
     async def _handle_failure(self) -> None:
         assert self._requested_max_submit is not None

src/ert/scheduler/scheduler.py

@@ -106,6 +106,10 @@ def __init__(
         self._completed_jobs_num: int = 0
         self.completed_jobs: asyncio.Queue[int] = asyncio.Queue()
 
+        # this lock is to assure that no more than 1 task
+        # does internalization at a time
+        self._fmok_lock: asyncio.Lock = asyncio.Lock()
+
         self._cancelled = False
         if max_submit < 0:
             raise ValueError(