Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feature(zjow): add trackback log for subprocess env manager #534

Merged
merged 2 commits into from
Nov 2, 2022
Merged

feature(zjow): add trackback log for subprocess env manager #534

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a0671f4
Add trackback log for subprocess env manager
zjowowen Nov 1, 2022
aca1e06
Minor change.
zjowowen Nov 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion ding/envs/env_manager/subprocess_env_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -397,7 +397,10 @@ def reset(self, reset_param: Optional[Dict] = None) -> None:
self._check_data({env_id: ret})
self._env_seed[env_id] = None # seed only use once
except BaseException as e:
logging.warning("subprocess reset set seed failed, ignore and continue...")
logging.warning(
"subprocess reset set seed failed, ignore and continue... \n subprocess exception traceback: \n"
+ traceback.format_exc()
)
self._env_states[env_id] = EnvState.RESET
reset_thread = PropagatingThread(target=self._reset, args=(env_id, ))
reset_thread.daemon = True
Expand Down Expand Up @@ -439,6 +442,7 @@ def reset_fn():
reset_fn()
return
except BaseException as e:
logging.info("subprocess exception traceback: \n" + traceback.format_exc())
if self._retry_type == 'renew' or isinstance(e, pickle.UnpicklingError):
self._pipe_parents[env_id].close()
if self._subprocesses[env_id].is_alive():
Expand Down Expand Up @@ -616,6 +620,7 @@ def worker_fn(
except Exception as e:
# when there are some errors in env, worker_fn will send the errors to env manager
# directly send error to another process will lose the stack trace, so we create a new Exception
logging.warning("subprocess exception traceback: \n" + traceback.format_exc())
c.send(
e.__class__(
'\nEnv Process Exception:\n' + ''.join(traceback.format_tb(e.__traceback__)) + repr(e)
Expand Down Expand Up @@ -671,6 +676,7 @@ def reset_fn(*args, **kwargs):
ret = None
return ret
except BaseException as e:
logging.warning("subprocess exception traceback: \n" + traceback.format_exc())
env.close()
raise e

Expand Down Expand Up @@ -704,6 +710,7 @@ def reset_fn(*args, **kwargs):
logging.debug("Sub env '{}' error when executing {}".format(str(env), cmd))
# when there are some errors in env, worker_fn will send the errors to env manager
# directly send error to another process will lose the stack trace, so we create a new Exception
logging.warning("subprocess exception traceback: \n" + traceback.format_exc())
child.send(
e.__class__('\nEnv Process Exception:\n' + ''.join(traceback.format_tb(e.__traceback__)) + repr(e))
)
Expand Down