dummy test train job

users/zeyer/experiments/exp2023_04_25_rf/conformer_import_moh_att_2023_06_30.py

@@ -322,6 +322,19 @@ def sis_run_with_prefix(prefix_name: Optional[str] = None):
         num_epochs=500,  # because of multi-GPU, 1 subepoch here is like 4 subepochs in single-GPU
     )
 
+    train_exp(
+        "tmp-dummy-test",
+        config_24gb_v6,
+        config_updates={"startup_callback": _tmp_dummy_startup_callback_exit, "dummy_test": 42},
+        gpu_mem=24,
+        num_epochs=1,
+        time_rqmt=1,
+    )
+
+
+def _tmp_dummy_startup_callback_exit(**_kwargs):
+    raise SystemExit("dummy test exit")
+
 
 _sis_prefix: Optional[str] = None
 
@@ -384,6 +397,7 @@ def train_exp(
     gpu_mem: Optional[int] = 24,
     num_processes: Optional[int] = None,
     fine_tune: Optional[Union[int, List[Tuple[int, Dict[str, Any]]]]] = None,
+    time_rqmt: Optional[int] = None,
 ) -> ModelWithCheckpoints:
     """
     Train experiment
@@ -410,6 +424,7 @@ def train_exp(
         gpu_mem=gpu_mem,
         num_processes=num_processes,
         distributed_launch_cmd="torchrun" if num_processes else "mpirun",
+        time_rqmt=time_rqmt,
     )
     recog_training_exp(prefix, task, model_with_checkpoint, recog_def=model_recog)
 

users/zeyer/experiments/exp2023_04_25_rf/train.py

@@ -147,7 +147,8 @@ def train(
         cpu_rqmt=4 if (not num_processes or num_processes <= 4) else 3,
         horovod_num_processes=num_processes,  # legacy name but also applies for Torch
     ).items():
-        kwargs.setdefault(k, v)
+        if k not in kwargs or kwargs[k] is None:
+            kwargs[k] = v
     returnn_train_job = ReturnnTrainingJob(returnn_train_config, **kwargs)
     returnn_train_job.add_alias(prefix_name + "/train")
     if gpu_mem: