Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[RLlib; PyTorch] - Add kwargs to torch.nn.parallel.DistributedDataParallel. #47276

Merged
merged 3 commits into from
Aug 26, 2024
+22 −3
Merged

[RLlib; PyTorch] - Add kwargs to torch.nn.parallel.DistributedDataParallel. #47276

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a768b63
Added kwargs to torch.nn.parallel.DistributedDataParallel.
simonsays1980 Aug 22, 2024
1b73fb7
Merge branch 'master' into add-ctor-args-kwargs-to-DDPTorchRLModule
simonsays1980 Aug 23, 2024
7e57b8e
Added missing literal string in docs.
simonsays1980 Aug 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions rllib/algorithms/algorithm_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,8 @@ def __init__(self, algo_class: Optional[type] = None):
"aot_eager" if sys.platform == "darwin" else "onnxrt"
)
self.torch_compile_worker_dynamo_mode = None
# Default kwargs for `torch.nn.parallel.DistributedDataParallel`.
self.torch_ddp_kwargs = {}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice! Thanks for making this generic and not hard-code e.g. the find_unused_parameters flag! :)


# `self.api_stack()`
self.enable_rl_module_and_learner = False
Expand Down Expand Up @@ -1378,6 +1380,7 @@ def framework(
torch_compile_worker: Optional[bool] = NotProvided,
torch_compile_worker_dynamo_backend: Optional[str] = NotProvided,
torch_compile_worker_dynamo_mode: Optional[str] = NotProvided,
torch_ddp_kwargs: Optional[Dict[str, Any]] = NotProvided,
) -> "AlgorithmConfig":
"""Sets the config's DL framework settings.

Expand Down Expand Up @@ -1417,6 +1420,12 @@ def framework(
the workers.
torch_compile_worker_dynamo_mode: The torch dynamo mode to use on the
workers.
torch_ddp_kwargs: The kwargs to pass into
`torch.nn.parallel.DistributedDataParallel` when using `num_learners
> 1`. This is specifically helpful when searching for unused parameters
that are not used in the backward pass. This can give hints for errors
in custom models where some parameters do not get touched in the
backward pass although they should.

Returns:
This updated AlgorithmConfig object.
Expand Down Expand Up @@ -1458,6 +1467,8 @@ def framework(
)
if torch_compile_worker_dynamo_mode is not NotProvided:
self.torch_compile_worker_dynamo_mode = torch_compile_worker_dynamo_mode
if torch_ddp_kwargs is not NotProvided:
self.torch_ddp_kwargs = torch_ddp_kwargs

return self

Expand Down
14 changes: 11 additions & 3 deletions rllib/core/learner/torch/torch_learner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,9 @@ def add_module(
"torch compile."
)
self._module.add_module(
module_id, TorchDDPRLModule(module), override=True
module_id,
TorchDDPRLModule(module, **self.config.torch_ddp_kwargs),
override=True,
)

return marl_spec
Expand Down Expand Up @@ -406,7 +408,9 @@ def _make_modules_ddp_if_necessary(self) -> None:
if self._distributed:
# Single agent module: Convert to `TorchDDPRLModule`.
if isinstance(self._module, TorchRLModule):
self._module = TorchDDPRLModule(self._module)
self._module = TorchDDPRLModule(
self._module, **self.config.torch_ddp_kwargs
)
# Multi agent module: Convert each submodule to `TorchDDPRLModule`.
else:
assert isinstance(self._module, MultiRLModule)
Expand All @@ -415,7 +419,11 @@ def _make_modules_ddp_if_necessary(self) -> None:
if isinstance(sub_module, TorchRLModule):
# Wrap and override the module ID key in self._module.
self._module.add_module(
key, TorchDDPRLModule(sub_module), override=True
key,
TorchDDPRLModule(
sub_module, **self.config.torch_ddp_kwargs
),
override=True,
)

def _is_module_compatible_with_learner(self, module: RLModule) -> bool:
Expand Down