Add GAIL algorithm

configs/lunarlander_continuous_v2/gail_ppo.yaml

@@ -0,0 +1,59 @@
+type: "GAILPPOAgent"
+hyper_params:
+  gamma: 0.99
+  tau: 0.95
+  batch_size: 128
+  max_epsilon: 0.2
+  min_epsilon: 0.2
+  epsilon_decay_period: 1500
+  w_value: 1.0
+  w_entropy: 0.001
+  gradient_clip_ac: 0.5
+  gradient_clip_cr: 1.0
+  epoch: 10
+  rollout_len: 1024
+  n_workers: 4
+  use_clipped_value_loss: False
+  standardize_advantage: True
+  gail_reward_weight: 1.0
+  demo_path: "data/lunarlander_continuous_demo.pkl"
+
+learner_cfg:
+  type: "GAILPPOLearner"
+  backbone:
+    actor:
+    critic:
+    discriminator: 
+    shared_actor_critic:
+  head:
+    actor:
+      type: "GaussianDist"
+      configs: 
+        hidden_sizes: [256, 256]
+        output_activation: "identity"
+        fixed_logstd: True
+    critic:
+      type: "MLP"
+      configs:
+        hidden_sizes: [256, 256]
+        output_size: 1
+        output_activation: "identity"
+    discriminator:
+      type: "MLP"
+      configs:
+        hidden_sizes: [256, 256]
+        output_size: 1
+        output_activation: "identity"
+    aciton_embedder:
+      type: "MLP"
+      configs:
+        hidden_sizes: []
+        output_size: 16
+        output_activation: "identity"
+
+  optim_cfg:
+    lr_actor: 0.0003
+    lr_critic: 0.001
+    lr_discriminator: 0.0003
+    weight_decay: 0.0
+    discriminator_acc_threshold : 0.8

rl_algorithms/__init__.py

@@ -21,6 +21,8 @@
 from .fd.dqn_learner import DQfDLearner
 from .fd.sac_agent import SACfDAgent
 from .fd.sac_learner import SACfDLearner
+from .gail.agent import GAILPPOAgent
+from .gail.learner import GAILPPOLearner
 from .ppo.agent import PPOAgent
 from .ppo.learner import PPOLearner
 from .recurrent.dqn_agent import R2D1Agent
@@ -45,6 +47,7 @@
     "PPOAgent",
     "SACAgent",
     "TD3Agent",
+    "GAILPPOAgent",
     "A2CLearner",
     "BCDDPGLearner",
     "BCSACLearner",
@@ -56,6 +59,7 @@
     "PPOLearner",
     "SACLearner",
     "TD3Learner",
+    "GAILPPOLearner",
     "R2D1Learner",
     "LunarLanderContinuousHER",
     "ReacherHER",

rl_algorithms/common/buffer/gail_buffer.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+"""Demo buffer for GAIL algorithm."""
+
+import pickle
+from typing import List, Tuple
+
+import numpy as np
+import torch
+
+from rl_algorithms.common.abstract.buffer import BaseBuffer
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+
+class GAILBuffer(BaseBuffer):
+    """Buffer to store expert states and actions.
+
+    Attributes:
+        obs_buf (np.ndarray): observations
+        acts_buf (np.ndarray): actions
+    """
+
+    def __init__(self, dataset_path: str):
+        """Initialize a Buffer.
+
+        Args:
+            dataset_path (str): path of the demo dataset
+        """
+
+        self.obs_buf: np.ndarray = None
+        self.acts_buf: np.ndarray = None
+
+        self.load_demo(dataset_path)
+
+    def load_demo(self, dataset_path: str):
+        """load demo data."""
+        with open(dataset_path, "rb") as f:
+            demo = list(pickle.load(f))
+        demo = np.array(demo)
+        self.obs_buf = np.array(list(map(np.array, demo[:, 0])))
+        self.acts_buf = np.array(list(map(np.array, demo[:, 1])))
+
+    def add(self):
+        pass
+
+    def sample(self, batch_size, indices: List[int] = None) -> Tuple[np.ndarray, ...]:
+        """Randomly sample a batch of experiences from memory."""
+        assert 0 < batch_size < len(self)
+
+        if indices is None:
+            indices = np.random.choice(len(self), size=batch_size)
+
+        states = self.obs_buf[indices]
+        actions = self.acts_buf[indices]
+
+        return torch.Tensor(states).to(device), torch.Tensor(actions).to(device)
+
+    def __len__(self) -> int:
+        """Return the current size of internal memory."""
+        return len(self.obs_buf)

rl_algorithms/common/helper_functions.py

@@ -28,6 +28,11 @@ def identity(x: torch.Tensor) -> torch.Tensor:
     return x
 
 
+def relu(x: torch.Tensor) -> torch.Tensor:
+    """Return torch.relu(x)"""
+    return torch.relu(x)
+
+
 def soft_update(local: nn.Module, target: nn.Module, tau: float):
     """Soft-update: target = tau*local + (1-tau)*target."""
     for t_param, l_param in zip(target.parameters(), local.parameters()):

rl_algorithms/gail/__init__.py

@@ -0,0 +1 @@
+"""Empty."""