polish(zjow): add trackback log for subprocess env manager (#534)

* Add trackback log for subprocess env manager * Minor change.
opendilab · Nov 2, 2022 · 806f133 · 806f133
1 parent b22589c
commit 806f133
Showing 1 changed file with 8 additions and 1 deletion.
diff --git a/ding/envs/env_manager/subprocess_env_manager.py b/ding/envs/env_manager/subprocess_env_manager.py
@@ -397,7 +397,10 @@ def reset(self, reset_param: Optional[Dict] = None) -> None:
                     self._check_data({env_id: ret})
                     self._env_seed[env_id] = None  # seed only use once
                 except BaseException as e:
-                    logging.warning("subprocess reset set seed failed, ignore and continue...")
+                    logging.warning(
+                        "subprocess reset set seed failed, ignore and continue... \n subprocess exception traceback: \n"
+                        + traceback.format_exc()
+                    )
             self._env_states[env_id] = EnvState.RESET
             reset_thread = PropagatingThread(target=self._reset, args=(env_id, ))
             reset_thread.daemon = True
@@ -439,6 +442,7 @@ def reset_fn():
                 reset_fn()
                 return
             except BaseException as e:
+                logging.info("subprocess exception traceback: \n" + traceback.format_exc())
                 if self._retry_type == 'renew' or isinstance(e, pickle.UnpicklingError):
                     self._pipe_parents[env_id].close()
                     if self._subprocesses[env_id].is_alive():
@@ -616,6 +620,7 @@ def worker_fn(
                 except Exception as e:
                     # when there are some errors in env, worker_fn will send the errors to env manager
                     # directly send error to another process will lose the stack trace, so we create a new Exception
+                    logging.warning("subprocess exception traceback: \n" + traceback.format_exc())
                     c.send(
                         e.__class__(
                             '\nEnv Process Exception:\n' + ''.join(traceback.format_tb(e.__traceback__)) + repr(e)
@@ -671,6 +676,7 @@ def reset_fn(*args, **kwargs):
                     ret = None
                 return ret
             except BaseException as e:
+                logging.warning("subprocess exception traceback: \n" + traceback.format_exc())
                 env.close()
                 raise e
 
@@ -704,6 +710,7 @@ def reset_fn(*args, **kwargs):
                 logging.debug("Sub env '{}' error when executing {}".format(str(env), cmd))
                 # when there are some errors in env, worker_fn will send the errors to env manager
                 # directly send error to another process will lose the stack trace, so we create a new Exception
+                logging.warning("subprocess exception traceback: \n" + traceback.format_exc())
                 child.send(
                     e.__class__('\nEnv Process Exception:\n' + ''.join(traceback.format_tb(e.__traceback__)) + repr(e))
                 )