env(zjow): add Huggingface model card support for ppof envs

ding/bonus/config.py

@@ -7,24 +7,27 @@
 from ding.policy import PPOFPolicy
 
 
 def get_instance_config(env_id: str, algorithm: str) -> EasyDict:
     if algorithm == 'PPOF':
         cfg = PPOFPolicy.default_config()
         if env_id == 'LunarLander-v2':
             cfg.n_sample = 512
             cfg.value_norm = 'popart'
             cfg.entropy_weight = 1e-3
         elif env_id == 'LunarLanderContinuous-v2':
             cfg.action_space = 'continuous'
             cfg.n_sample = 400
         elif env_id == 'BipedalWalker-v3':
             cfg.learning_rate = 1e-3
             cfg.action_space = 'continuous'
             cfg.n_sample = 1024
+        elif env_id == 'Pendulum-v1':
+            cfg.action_space = 'continuous'
+            cfg.n_sample = 400
         elif env_id == 'acrobot':
             cfg.learning_rate = 1e-4
             cfg.n_sample = 400
         elif env_id == 'rocket_landing':
             cfg.n_sample = 2048
             cfg.adv_norm = False
             cfg.model = dict(
@@ -32,13 +35,13 @@
                 actor_head_hidden_size=128,
                 critic_head_hidden_size=128,
             )
         elif env_id == 'drone_fly':
             cfg.action_space = 'continuous'
             cfg.adv_norm = False
             cfg.epoch_per_collect = 5
             cfg.learning_rate = 5e-5
             cfg.n_sample = 640
         elif env_id == 'hybrid_moving':
             cfg.action_space = 'hybrid'
             cfg.n_sample = 3200
             cfg.entropy_weight = 0.03
@@ -50,13 +53,13 @@
                 fixed_sigma_value=0.3,
                 bound_type='tanh',
             )
         elif env_id == 'evogym_carrier':
             cfg.action_space = 'continuous'
             cfg.n_sample = 2048
             cfg.batch_size = 256
             cfg.epoch_per_collect = 10
             cfg.learning_rate = 3e-3
         elif env_id == 'mario':
             cfg.n_sample = 256
             cfg.batch_size = 64
             cfg.epoch_per_collect = 2
@@ -66,14 +69,14 @@
                 critic_head_hidden_size=128,
                 actor_head_hidden_size=128,
             )
         elif env_id == 'di_sheep':
             cfg.n_sample = 3200
             cfg.batch_size = 320
             cfg.epoch_per_collect = 10
             cfg.learning_rate = 3e-4
             cfg.adv_norm = False
             cfg.entropy_weight = 0.001
         elif env_id == 'procgen_bigfish':
             cfg.n_sample = 16384
             cfg.batch_size = 16384
             cfg.epoch_per_collect = 10
@@ -83,7 +86,7 @@
                 critic_head_hidden_size=256,
                 actor_head_hidden_size=256,
             )
         elif env_id in ['KangarooNoFrameskip-v4', 'BowlingNoFrameskip-v4']:
             cfg.n_sample = 1024
             cfg.batch_size = 128
             cfg.epoch_per_collect = 10
@@ -94,7 +97,7 @@
                 critic_head_hidden_size=128,
                 critic_head_layer_num=2,
             )
         elif env_id == 'PongNoFrameskip-v4':
             cfg.n_sample = 3200
             cfg.batch_size = 320
             cfg.epoch_per_collect = 10
@@ -104,7 +107,7 @@
                 actor_head_hidden_size=128,
                 critic_head_hidden_size=128,
             )
         elif env_id == 'SpaceInvadersNoFrameskip-v4':
             cfg.n_sample = 320
             cfg.batch_size = 320
             cfg.epoch_per_collect = 1
@@ -116,7 +119,7 @@
                 actor_head_hidden_size=128,
                 critic_head_hidden_size=128,
             )
         elif env_id == 'QbertNoFrameskip-v4':
             cfg.n_sample = 3200
             cfg.batch_size = 320
             cfg.epoch_per_collect = 10
@@ -127,13 +130,13 @@
                 actor_head_hidden_size=128,
                 critic_head_hidden_size=128,
             )
         elif env_id == 'minigrid_fourroom':
             cfg.n_sample = 3200
             cfg.batch_size = 320
             cfg.learning_rate = 3e-4
             cfg.epoch_per_collect = 10
             cfg.entropy_weight = 0.001
         elif env_id == 'metadrive':
             cfg.learning_rate = 3e-4
             cfg.action_space = 'continuous'
             cfg.entropy_weight = 0.001
@@ -146,61 +149,61 @@
                 critic_head_hidden_size=128,
                 critic_head_layer_num=2,
             )
         elif env_id == 'Hopper-v3':
             cfg.action_space = "continuous"
             cfg.n_sample = 3200
             cfg.batch_size = 320
             cfg.epoch_per_collect = 10
             cfg.learning_rate = 3e-4
         elif env_id == 'HalfCheetah-v3':
             cfg.action_space = "continuous"
             cfg.n_sample = 3200
             cfg.batch_size = 320
             cfg.epoch_per_collect = 10
             cfg.learning_rate = 3e-4
         elif env_id == 'Walker2d-v3':
             cfg.action_space = "continuous"
             cfg.n_sample = 3200
             cfg.batch_size = 320
             cfg.epoch_per_collect = 10
             cfg.learning_rate = 3e-4
         else:
             raise KeyError("not supported env type: {}".format(env_id))
     else:
         raise KeyError("not supported algorithm type: {}".format(algorithm))
 
     return cfg
 
 
 def get_instance_env(env_id: str) -> BaseEnv:
     if env_id == 'LunarLander-v2':
         return DingEnvWrapper(gym.make('LunarLander-v2'))
     elif env_id == 'LunarLanderContinuous-v2':
         return DingEnvWrapper(gym.make('LunarLanderContinuous-v2', continuous=True))
     elif env_id == 'BipedalWalker-v3':
         return DingEnvWrapper(gym.make('BipedalWalker-v3'), cfg={'act_scale': True, 'rew_clip': True})
     elif env_id == 'Pendulum-v1':
         return DingEnvWrapper(gym.make('Pendulum-v1'), cfg={'act_scale': True})
     elif env_id == 'acrobot':
         return DingEnvWrapper(gym.make('Acrobot-v1'))
     elif env_id == 'rocket_landing':
         from dizoo.rocket.envs import RocketEnv
         cfg = EasyDict({
             'task': 'landing',
             'max_steps': 800,
         })
         return RocketEnv(cfg)
     elif env_id == 'drone_fly':
         from dizoo.gym_pybullet_drones.envs import GymPybulletDronesEnv
         cfg = EasyDict({
             'env_id': 'flythrugate-aviary-v0',
             'action_type': 'VEL',
         })
         return GymPybulletDronesEnv(cfg)
     elif env_id == 'hybrid_moving':
         import gym_hybrid
         return DingEnvWrapper(gym.make('Moving-v0'))
     elif env_id == 'evogym_carrier':
         import evogym.envs
         from evogym import sample_robot, WorldObject
         path = os.path.join(os.path.dirname(__file__), '../../dizoo/evogym/envs/world_data/carry_bot.json')
@@ -215,7 +218,7 @@
                 ]
             }
         )
     elif env_id == 'mario':
         import gym_super_mario_bros
         from nes_py.wrappers import JoypadSpace
         return DingEnvWrapper(
@@ -231,10 +234,10 @@
                 ]
             }
         )
     elif env_id == 'di_sheep':
         from sheep_env import SheepEnv
         return DingEnvWrapper(SheepEnv(level=9))
     elif env_id == 'procgen_bigfish':
         return DingEnvWrapper(
             gym.make('procgen:procgen-bigfish-v0', start_level=0, num_levels=1),
             cfg={
@@ -246,7 +249,7 @@
             },
             seed_api=False,
         )
     elif env_id == 'Hopper-v3':
         cfg = EasyDict(
             env_id='Hopper-v3',
             env_wrapper='mujoco_default',
@@ -254,7 +257,7 @@
             rew_clip=True,
         )
         return DingEnvWrapper(gym.make('Hopper-v3'), cfg=cfg)
     elif env_id == 'HalfCheetah-v3':
         cfg = EasyDict(
             env_id='HalfCheetah-v3',
             env_wrapper='mujoco_default',
@@ -262,7 +265,7 @@
             rew_clip=True,
         )
         return DingEnvWrapper(gym.make('HalfCheetah-v3'), cfg=cfg)
     elif env_id == 'Walker2d-v3':
         cfg = EasyDict(
             env_id='Walker2d-v3',
             env_wrapper='mujoco_default',
@@ -271,7 +274,7 @@
         )
         return DingEnvWrapper(gym.make('Walker2d-v3'), cfg=cfg)
 
     elif env_id in [
             'BowlingNoFrameskip-v4',
             'BreakoutNoFrameskip-v4',
             'GopherNoFrameskip-v4'
@@ -285,9 +288,9 @@
             'env_id': env_id,
             'env_wrapper': 'atari_default',
         })
         ding_env_atari = DingEnvWrapper(gym.make(env_id), cfg=cfg)
         return ding_env_atari
     elif env_id == 'minigrid_fourroom':
         import gymnasium
         return DingEnvWrapper(
             gymnasium.make('MiniGrid-FourRooms-v0'),
@@ -300,7 +303,7 @@
                 ]
             }
         )
     elif env_id == 'metadrive':
         from dizoo.metadrive.env.drive_env import MetaDrivePPOOriginEnv
         from dizoo.metadrive.env.drive_wrapper import DriveEnvWrapper
         cfg = dict(
@@ -313,7 +316,7 @@
         cfg = EasyDict(cfg)
         return DriveEnvWrapper(MetaDrivePPOOriginEnv(cfg))
     else:
         raise KeyError("not supported env type: {}".format(env_id))
 
 
 def get_hybrid_shape(action_space) -> EasyDict:

ding/bonus/ppof.py

@@ -5,7 +5,7 @@
 import os
 import gym
 import gymnasium
 import numpy as np
 import torch
 from ding.framework import task, OnlineRLContext
 from ding.framework.middleware import interaction_evaluator_ttorch, PPOFStepCollector, multistep_trainer, CkptSaver, \
@@ -13,7 +13,7 @@
 from ding.envs import BaseEnv, BaseEnvManagerV2, SubprocessEnvManagerV2
 from ding.policy import PPOFPolicy, single_env_forward_wrapper_ttorch
 from ding.utils import set_pkg_seed
 from ding.utils import get_env_fps, render
 from ding.config import save_config_py
 from .model import PPOFModel
 from .config import get_instance_config, get_instance_env, get_hybrid_shape
@@ -26,6 +26,7 @@
         'LunarLander-v2',
         'LunarLanderContinuous-v2',
         'BipedalWalker-v3',
+        'Pendulum-v1',
         'acrobot',
         # ch2: action
         'rocket_landing',
@@ -63,40 +64,40 @@
             cfg: Optional[Union[EasyDict, dict]] = None,
             policy_state_dict: str = None
     ) -> None:
         assert env_id is not None or cfg is not None, "Please specify env_id or cfg."
 
         if cfg is not None and not isinstance(cfg, EasyDict):
             cfg = EasyDict(cfg)
 
         if env_id is not None:
             assert env_id in PPOF.supported_env_list, "Please use supported envs: {}".format(PPOF.supported_env_list)
             if cfg is None:
                 cfg = get_instance_config(env_id, algorithm="PPOF")
 
             if not hasattr(cfg, "env_id"):
                 cfg.env_id = env_id
             assert cfg.env_id == env_id, "env_id in cfg should be the same as env_id in args."
         else:
             assert hasattr(cfg, "env_id"), "Please specify env_id in cfg."
             assert cfg.env_id in PPOF.supported_env_list, "Please use supported envs: {}".format(
                 PPOF.supported_env_list
             )
 
         if exp_name is not None:
             cfg.exp_name = exp_name
         elif not hasattr(cfg, "exp_name"):
             cfg.exp_name = "{}-{}".format(cfg.env_id, "PPO")
         self.cfg = cfg
         self.exp_name = self.cfg.exp_name
 
         if env is None:
             self.env = get_instance_env(self.cfg.env_id)
         else:
             self.env = env
 
         logging.getLogger().setLevel(logging.INFO)
         self.seed = seed
         set_pkg_seed(self.seed, use_cuda=self.cfg.cuda)
 
         if not os.path.exists(self.exp_name):
             os.makedirs(self.exp_name)
@@ -176,7 +177,7 @@
 
         return TrainingReturn(wandb_url=task.ctx.wandb_url)
 
     def deploy(
             self,
             enable_save_replay: bool = False,
             concatenate_all_replay: bool = False,
@@ -189,23 +190,23 @@
         # define env and policy
         env = self.env.clone(caller='evaluator')
 
         if seed is not None and isinstance(seed, int):
             seeds = [seed]
         elif seed is not None and isinstance(seed, list):
             seeds = seed
         else:
             seeds = [self.seed]
 
         returns = []
         images = []
         if enable_save_replay:
             replay_save_path = os.path.join(self.exp_name, 'videos') if replay_save_path is None else replay_save_path
             env.enable_save_replay(replay_path=replay_save_path)
         else:
             logging.warning('No video would be generated during the deploy.')
             if concatenate_all_replay:
                 logging.warning('concatenate_all_replay is set to False because enable_save_replay is False.')
                 concatenate_all_replay = False
 
         forward_fn = single_env_forward_wrapper_ttorch(self.policy.eval, self.cfg.cuda)
 
@@ -213,31 +214,31 @@
         # env will be reset again in the main loop
         env.reset()
 
         for seed in seeds:
             env.seed(seed, dynamic_seed=False)
             return_ = 0.
             step = 0
             obs = env.reset()
             images.append(render(env)[None]) if concatenate_all_replay else None
             while True:
                 action = forward_fn(obs)
                 obs, rew, done, info = env.step(action)
                 images.append(render(env)[None]) if concatenate_all_replay else None
                 return_ += rew
                 step += 1
                 if done:
                     break
             logging.info(f'DQN deploy is finished, final episode return with {step} steps is: {return_}')
             returns.append(return_)
 
         env.close()
 
         if concatenate_all_replay:
             images = np.concatenate(images, axis=0)
             import imageio
             imageio.mimwrite(os.path.join(replay_save_path, 'deploy.mp4'), images, fps=get_env_fps(env))
 
         return EvalReturn(eval_value=np.mean(returns), eval_value_std=np.std(returns))
 
     def collect_data(
             self,