-
Notifications
You must be signed in to change notification settings - Fork 423
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add a Builder class which encapsulates a full agent.
This class allows for the consituent components to be broken apart so that it can be used both for distributed and non-distributed variants. For the time-being this is only incorporated into the TF D4PG agent to allow for minimal disruption and experimentation, but should be rolled out for all agents soon. PiperOrigin-RevId: 356975846 Change-Id: I00ead33da40f4f98052ae3beb218c23788ada206
- Loading branch information
Showing
6 changed files
with
395 additions
and
126 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,103 @@ | ||
# python3 | ||
# Copyright 2018 DeepMind Technologies Limited. All rights reserved. | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
"""RL agent Builder interface.""" | ||
|
||
import abc | ||
from typing import Iterator, List, Optional | ||
|
||
from acme import adders | ||
from acme import core | ||
from acme import specs | ||
from acme.utils import counting | ||
from acme.utils import loggers | ||
import reverb | ||
|
||
|
||
class ActorLearnerBuilder(abc.ABC): | ||
"""Defines an interface for defining the components of an RL agent. | ||
Implementations of this interface contain a complete specification of a | ||
concrete RL agent. An instance of this class can be used to build an | ||
RL agent which interacts with the environment either locally or in a | ||
distributed setup. | ||
""" | ||
|
||
@abc.abstractmethod | ||
def make_replay_tables( | ||
self, | ||
environment_spec: specs.EnvironmentSpec, | ||
) -> List[reverb.Table]: | ||
"""Create tables to insert data into.""" | ||
|
||
@abc.abstractmethod | ||
def make_dataset_iterator( | ||
self, | ||
replay_client: reverb.Client, | ||
) -> Iterator[reverb.ReplaySample]: | ||
"""Create a dataset iterator to use for learning/updating the agent.""" | ||
|
||
@abc.abstractmethod | ||
def make_adder( | ||
self, | ||
replay_client: reverb.Client, | ||
) -> Optional[adders.Adder]: | ||
"""Create an adder which records data generated by the actor/environment. | ||
Args: | ||
replay_client: Reverb Client which points to the replay server. | ||
""" | ||
|
||
@abc.abstractmethod | ||
def make_actor( | ||
self, | ||
policy_network, | ||
adder: Optional[adders.Adder] = None, | ||
variable_source: Optional[core.VariableSource] = None, | ||
) -> core.Actor: | ||
"""Create an actor instance. | ||
Args: | ||
policy_network: Instance of a policy network; this should be a callable | ||
which takes as input observations and returns actions. | ||
adder: How data is recorded (e.g. added to replay). | ||
variable_source: A source providing the necessary actor parameters. | ||
""" | ||
|
||
@abc.abstractmethod | ||
def make_learner( | ||
self, | ||
networks, | ||
dataset: Iterator[reverb.ReplaySample], | ||
replay_client: Optional[reverb.Client] = None, | ||
counter: Optional[counting.Counter] = None, | ||
# TODO(mwhoffman): consider eliminating logger and log return values. | ||
# TODO(mwhoffman): eliminate checkpoint and move it outside. | ||
logger: Optional[loggers.Logger] = None, | ||
checkpoint: bool = False, | ||
) -> core.Learner: | ||
"""Creates an instance of the learner. | ||
Args: | ||
networks: struct describing the networks needed by the learner; this can | ||
be specific to the learner in question. | ||
dataset: iterator over samples from replay. | ||
replay_client: client which allows communication with replay, e.g. in | ||
order to update priorities. | ||
counter: a Counter which allows for recording of counts (learner steps, | ||
actor steps, etc.) distributed throughout the agent. | ||
logger: Logger object for logging metadata. | ||
checkpoint: bool controlling whether the learner checkpoints itself. | ||
""" |
Oops, something went wrong.