Add trackback log for subprocess env manager

opendilab · Nov 1, 2022 · a0671f4 · a0671f4
1 parent b22589c
commit a0671f4
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/ding/envs/env_manager/subprocess_env_manager.py b/ding/envs/env_manager/subprocess_env_manager.py
@@ -398,6 +398,7 @@ def reset(self, reset_param: Optional[Dict] = None) -> None:
                     self._env_seed[env_id] = None  # seed only use once
                 except BaseException as e:
                     logging.warning("subprocess reset set seed failed, ignore and continue...")
+                    logging.warning("subprocess exception traceback: \n" + traceback.format_exc())
             self._env_states[env_id] = EnvState.RESET
             reset_thread = PropagatingThread(target=self._reset, args=(env_id, ))
             reset_thread.daemon = True
@@ -439,6 +440,7 @@ def reset_fn():
                 reset_fn()
                 return
             except BaseException as e:
+                logging.warning("subprocess exception traceback: \n" + traceback.format_exc())
                 if self._retry_type == 'renew' or isinstance(e, pickle.UnpicklingError):
                     self._pipe_parents[env_id].close()
                     if self._subprocesses[env_id].is_alive():
@@ -616,6 +618,7 @@ def worker_fn(
                 except Exception as e:
                     # when there are some errors in env, worker_fn will send the errors to env manager
                     # directly send error to another process will lose the stack trace, so we create a new Exception
+                    logging.warning("subprocess exception traceback: \n" + traceback.format_exc())
                     c.send(
                         e.__class__(
                             '\nEnv Process Exception:\n' + ''.join(traceback.format_tb(e.__traceback__)) + repr(e)
@@ -671,6 +674,7 @@ def reset_fn(*args, **kwargs):
                     ret = None
                 return ret
             except BaseException as e:
+                logging.warning("subprocess exception traceback: \n" + traceback.format_exc())
                 env.close()
                 raise e
 
@@ -704,6 +708,7 @@ def reset_fn(*args, **kwargs):
                 logging.debug("Sub env '{}' error when executing {}".format(str(env), cmd))
                 # when there are some errors in env, worker_fn will send the errors to env manager
                 # directly send error to another process will lose the stack trace, so we create a new Exception
+                logging.warning("subprocess exception traceback: \n" + traceback.format_exc())
                 child.send(
                     e.__class__('\nEnv Process Exception:\n' + ''.join(traceback.format_tb(e.__traceback__)) + repr(e))
                 )