diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 49497a9909..e90992ca29 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -43,6 +43,7 @@ Guidelines for modifications: * Chenyu Yang * David Yang * Dorsa Rohani +* Felix Yu * Gary Lvov * Giulio Romualdi * HoJin Jeon diff --git a/docs/source/overview/environments.rst b/docs/source/overview/environments.rst index f42f3c34a5..531588a8de 100644 --- a/docs/source/overview/environments.rst +++ b/docs/source/overview/environments.rst @@ -61,6 +61,10 @@ Classic environments that are based on IsaacGymEnvs implementation of MuJoCo-sty | | | | | | |cartpole-depth-direct-link|| | +------------------+-----------------------------+-------------------------------------------------------------------------+ + | |cartpole| | |cartpole-resnet-link| | Move the cart to keep the pole upwards in the classic cartpole control | + | | | based off of features extracted from perceptive inputs with pre-trained | + | | |cartpole-theia-link| | frozen vision encoders | + +------------------+-----------------------------+-------------------------------------------------------------------------+ .. |humanoid| image:: ../_static/tasks/classic/humanoid.jpg .. |ant| image:: ../_static/tasks/classic/ant.jpg @@ -69,8 +73,11 @@ Classic environments that are based on IsaacGymEnvs implementation of MuJoCo-sty .. |humanoid-link| replace:: `Isaac-Humanoid-v0 `__ .. |ant-link| replace:: `Isaac-Ant-v0 `__ .. |cartpole-link| replace:: `Isaac-Cartpole-v0 `__ -.. |cartpole-rgb-link| replace:: `Isaac-Cartpole-RGB-Camera-v0 `__ -.. |cartpole-depth-link| replace:: `Isaac-Cartpole-Depth-Camera-v0 `__ +.. |cartpole-rgb-link| replace:: `Isaac-Cartpole-RGB-v0 `__ +.. |cartpole-depth-link| replace:: `Isaac-Cartpole-Depth-v0 `__ +.. |cartpole-resnet-link| replace:: `Isaac-Cartpole-RGB-ResNet18-v0 `__ +.. |cartpole-theia-link| replace:: `Isaac-Cartpole-RGB-TheiaTiny-v0 `__ + .. |humanoid-direct-link| replace:: `Isaac-Humanoid-Direct-v0 `__ .. |ant-direct-link| replace:: `Isaac-Ant-Direct-v0 `__ diff --git a/pyproject.toml b/pyproject.toml index 51d4375907..63ec9afd2a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,9 @@ extra_standard_library = [ "toml", "trimesh", "tqdm", + "torchvision", + "transformers", + "einops" # Needed for transformers, doesn't always auto-install ] # Imports from Isaac Sim and Omniverse known_third_party = [ diff --git a/source/extensions/omni.isaac.lab/config/extension.toml b/source/extensions/omni.isaac.lab/config/extension.toml index feee940cdb..4a6faf6114 100644 --- a/source/extensions/omni.isaac.lab/config/extension.toml +++ b/source/extensions/omni.isaac.lab/config/extension.toml @@ -1,7 +1,8 @@ [package] # Note: Semantic Versioning is used: https://semver.org/ -version = "0.27.4" + +version = "0.27.7" # Description title = "Isaac Lab framework for Robot Learning" diff --git a/source/extensions/omni.isaac.lab/docs/CHANGELOG.rst b/source/extensions/omni.isaac.lab/docs/CHANGELOG.rst index c0ac6cd460..ecc0471794 100644 --- a/source/extensions/omni.isaac.lab/docs/CHANGELOG.rst +++ b/source/extensions/omni.isaac.lab/docs/CHANGELOG.rst @@ -1,6 +1,35 @@ Changelog --------- + +0.27.7 (2024-10-28) +~~~~~~~~~~~~~~~~~~~ + +Added +^^^^^ + +* Added frozen encoder feature extraction observation space with ResNet and Theia + + +0.27.6 (2024-10-25) +~~~~~~~~~~~~~~~~~~~ + +Fixed +^^^^^ + +* Fixed usage of ``meshes`` property in :class:`omni.isaac.lab.sensors.RayCasterCamera` to use ``self.meshes`` instead of the undefined ``RayCaster.meshes``. +* Fixed issue in :class:`omni.isaac.lab.envs.ui.BaseEnvWindow` where undefined configs were being accessed when creating debug visualization elements in UI. + + +0.27.5 (2024-10-25) +~~~~~~~~~~~~~~~~~~~ + +Added +^^^^^ + +* Added utilities for serializing/deserializing Gymnasium spaces. + + 0.27.4 (2024-10-18) ~~~~~~~~~~~~~~~~~~~ diff --git a/source/extensions/omni.isaac.lab/omni/isaac/lab/envs/mdp/observations.py b/source/extensions/omni.isaac.lab/omni/isaac/lab/envs/mdp/observations.py index fbae1d21cc..aca0f579ce 100644 --- a/source/extensions/omni.isaac.lab/omni/isaac/lab/envs/mdp/observations.py +++ b/source/extensions/omni.isaac.lab/omni/isaac/lab/envs/mdp/observations.py @@ -17,11 +17,14 @@ import omni.isaac.lab.utils.math as math_utils from omni.isaac.lab.assets import Articulation, RigidObject from omni.isaac.lab.managers import SceneEntityCfg +from omni.isaac.lab.managers.manager_base import ManagerTermBase +from omni.isaac.lab.managers.manager_term_cfg import ObservationTermCfg from omni.isaac.lab.sensors import Camera, Imu, RayCaster, RayCasterCamera, TiledCamera if TYPE_CHECKING: from omni.isaac.lab.envs import ManagerBasedEnv, ManagerBasedRLEnv + """ Root state. """ @@ -273,6 +276,134 @@ def image( return images.clone() +class image_features(ManagerTermBase): + """Extracted image features from a pre-trained frozen encoder. + + This method calls the :meth:`image` function to retrieve images, and then performs + inference on those images. + """ + + def __init__(self, cfg: ObservationTermCfg, env: ManagerBasedEnv): + super().__init__(cfg, env) + from torchvision import models + from transformers import AutoModel + + def create_theia_model(model_name): + return { + "model": ( + lambda: AutoModel.from_pretrained(f"theaiinstitute/{model_name}", trust_remote_code=True) + .eval() + .to("cuda:0") + ), + "preprocess": lambda img: (img - torch.amin(img, dim=(1, 2), keepdim=True)) / ( + torch.amax(img, dim=(1, 2), keepdim=True) - torch.amin(img, dim=(1, 2), keepdim=True) + ), + "inference": lambda model, images: model.forward_feature( + images, do_rescale=False, interpolate_pos_encoding=True + ), + } + + def create_resnet_model(resnet_name): + return { + "model": lambda: getattr(models, resnet_name)(pretrained=True).eval().to("cuda:0"), + "preprocess": lambda img: ( + img.permute(0, 3, 1, 2) # Convert [batch, height, width, 3] -> [batch, 3, height, width] + - torch.tensor([0.485, 0.456, 0.406], device=img.device).view(1, 3, 1, 1) + ) / torch.tensor([0.229, 0.224, 0.225], device=img.device).view(1, 3, 1, 1), + "inference": lambda model, images: model(images), + } + + # List of Theia models + theia_models = [ + "theia-tiny-patch16-224-cddsv", + "theia-tiny-patch16-224-cdiv", + "theia-small-patch16-224-cdiv", + "theia-base-patch16-224-cdiv", + "theia-small-patch16-224-cddsv", + "theia-base-patch16-224-cddsv", + ] + + # List of ResNet models + resnet_models = ["resnet18", "resnet34", "resnet50", "resnet101"] + + self.default_model_zoo_cfg = {} + + # Add Theia models to the zoo + for model_name in theia_models: + self.default_model_zoo_cfg[model_name] = create_theia_model(model_name) + + # Add ResNet models to the zoo + for resnet_name in resnet_models: + self.default_model_zoo_cfg[resnet_name] = create_resnet_model(resnet_name) + + self.model_zoo_cfg = self.default_model_zoo_cfg + self.model_zoo = {} + + def __call__( + self, + env: ManagerBasedEnv, + sensor_cfg: SceneEntityCfg = SceneEntityCfg("tiled_camera"), + data_type: str = "rgb", + convert_perspective_to_orthogonal: bool = False, + model_zoo_cfg: dict | None = None, + model_name: str = "ResNet18", + model_device: str | None = "cuda:0", + reset_model: bool = False, + ) -> torch.Tensor: + """Extracted image features from a pre-trained frozen encoder. + + Args: + env: The environment. + sensor_cfg: The sensor configuration to poll. Defaults to SceneEntityCfg("tiled_camera"). + data_type: THe sensor configuration datatype. Defaults to "rgb". + convert_perspective_to_orthogonal: Whether to orthogonalize perspective depth images. + This is used only when the data type is "distance_to_camera". Defaults to False. + model_zoo_cfg: Map from model name to model configuration dictionary. Each model + configuration dictionary should include the following entries: + - "model": A callable that returns the model when invoked without arguments. + - "preprocess": A callable that processes the images and returns the preprocessed results. + - "inference": A callable that, when given the model and preprocessed images, + returns the extracted features. + model_name: The name of the model to use for inference. Defaults to "ResNet18". + model_device: The device to store and infer models on. This can be used help offload + computation from the main environment GPU. Defaults to "cuda:0". + reset_model: Initialize the model even if it already exists. Defaults to False. + + Returns: + torch.Tensor: the image features, on the same device as the image + """ + if model_zoo_cfg is not None: # use other than default + self.model_zoo_cfg.update(model_zoo_cfg) + + if model_name not in self.model_zoo or reset_model: + # The following allows to only load a desired subset of a model zoo into GPU memory + # as it becomes needed, in a "lazy" evaluation. + print(f"[INFO]: Adding {model_name} to the model zoo") + self.model_zoo[model_name] = self.model_zoo_cfg[model_name]["model"]() + + if model_device is not None and self.model_zoo[model_name].device != model_device: + # want to offload vision model inference to another device + self.model_zoo[model_name] = self.model_zoo[model_name].to(model_device) + + images = image( + env=env, + sensor_cfg=sensor_cfg, + data_type=data_type, + convert_perspective_to_orthogonal=convert_perspective_to_orthogonal, + normalize=True, # want this for training stability + ) + + image_device = images.device + + if model_device is not None: + images = images.to(model_device) + + proc_images = self.model_zoo_cfg[model_name]["preprocess"](images) + features = self.model_zoo_cfg[model_name]["inference"](self.model_zoo[model_name], proc_images) + + return features.to(image_device).clone() + + """ Actions. """ diff --git a/source/extensions/omni.isaac.lab/omni/isaac/lab/envs/mdp/rewards.py b/source/extensions/omni.isaac.lab/omni/isaac/lab/envs/mdp/rewards.py index 7f03bf41f0..1a68d321ab 100644 --- a/source/extensions/omni.isaac.lab/omni/isaac/lab/envs/mdp/rewards.py +++ b/source/extensions/omni.isaac.lab/omni/isaac/lab/envs/mdp/rewards.py @@ -159,7 +159,7 @@ def joint_acc_l2(env: ManagerBasedRLEnv, asset_cfg: SceneEntityCfg = SceneEntity return torch.sum(torch.square(asset.data.joint_acc[:, asset_cfg.joint_ids]), dim=1) -def joint_deviation_l1(env, asset_cfg: SceneEntityCfg = SceneEntityCfg("robot")) -> torch.Tensor: +def joint_deviation_l1(env: ManagerBasedRLEnv, asset_cfg: SceneEntityCfg = SceneEntityCfg("robot")) -> torch.Tensor: """Penalize joint positions that deviate from the default one.""" # extract the used quantities (to enable type-hinting) asset: Articulation = env.scene[asset_cfg.name] diff --git a/source/extensions/omni.isaac.lab/omni/isaac/lab/envs/ui/base_env_window.py b/source/extensions/omni.isaac.lab/omni/isaac/lab/envs/ui/base_env_window.py index a37a8723ec..850ad0a355 100644 --- a/source/extensions/omni.isaac.lab/omni/isaac/lab/envs/ui/base_env_window.py +++ b/source/extensions/omni.isaac.lab/omni/isaac/lab/envs/ui/base_env_window.py @@ -382,7 +382,7 @@ def _create_debug_vis_ui_element(self, name: str, elem: object): self.ui_window_elements[f"{name}_cb"] = SimpleCheckBox( model=omni.ui.SimpleBoolModel(), enabled=elem.has_debug_vis_implementation, - checked=elem.cfg.debug_vis, + checked=elem.cfg.debug_vis if elem.cfg else False, on_checked_fn=lambda value, e=weakref.proxy(elem): e.set_debug_vis(value), ) omni.isaac.ui.ui_utils.add_line_rect_flourish() diff --git a/source/extensions/omni.isaac.lab/omni/isaac/lab/envs/utils/spaces.py b/source/extensions/omni.isaac.lab/omni/isaac/lab/envs/utils/spaces.py index 8604392ec6..2a1e30c1ee 100644 --- a/source/extensions/omni.isaac.lab/omni/isaac/lab/envs/utils/spaces.py +++ b/source/extensions/omni.isaac.lab/omni/isaac/lab/envs/utils/spaces.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: BSD-3-Clause import gymnasium as gym +import json import numpy as np import torch from typing import Any @@ -90,3 +91,131 @@ def tensorize(s, x): sample = (gym.vector.utils.batch_space(space, batch_size) if batch_size > 0 else space).sample() return tensorize(space, sample) + + +def serialize_space(space: SpaceType) -> str: + """Serialize a space specification as JSON. + + Args: + space: Space specification. + + Returns: + Serialized JSON representation. + """ + # Gymnasium spaces + if isinstance(space, gym.spaces.Discrete): + return json.dumps({"type": "gymnasium", "space": "Discrete", "n": int(space.n)}) + elif isinstance(space, gym.spaces.Box): + return json.dumps({ + "type": "gymnasium", + "space": "Box", + "low": space.low.tolist(), + "high": space.high.tolist(), + "shape": space.shape, + }) + elif isinstance(space, gym.spaces.MultiDiscrete): + return json.dumps({"type": "gymnasium", "space": "MultiDiscrete", "nvec": space.nvec.tolist()}) + elif isinstance(space, gym.spaces.Tuple): + return json.dumps({"type": "gymnasium", "space": "Tuple", "spaces": tuple(map(serialize_space, space.spaces))}) + elif isinstance(space, gym.spaces.Dict): + return json.dumps( + {"type": "gymnasium", "space": "Dict", "spaces": {k: serialize_space(v) for k, v in space.spaces.items()}} + ) + # Python data types + # Box + elif isinstance(space, int) or (isinstance(space, list) and all(isinstance(x, int) for x in space)): + return json.dumps({"type": "python", "space": "Box", "value": space}) + # Discrete + elif isinstance(space, set) and len(space) == 1: + return json.dumps({"type": "python", "space": "Discrete", "value": next(iter(space))}) + # MultiDiscrete + elif isinstance(space, list) and all(isinstance(x, set) and len(x) == 1 for x in space): + return json.dumps({"type": "python", "space": "MultiDiscrete", "value": [next(iter(x)) for x in space]}) + # composite spaces + # Tuple + elif isinstance(space, tuple): + return json.dumps({"type": "python", "space": "Tuple", "value": [serialize_space(x) for x in space]}) + # Dict + elif isinstance(space, dict): + return json.dumps( + {"type": "python", "space": "Dict", "value": {k: serialize_space(v) for k, v in space.items()}} + ) + raise ValueError(f"Unsupported space ({space})") + + +def deserialize_space(string: str) -> gym.spaces.Space: + """Deserialize a space specification encoded as JSON. + + Args: + string: Serialized JSON representation. + + Returns: + Space specification. + """ + obj = json.loads(string) + # Gymnasium spaces + if obj["type"] == "gymnasium": + if obj["space"] == "Discrete": + return gym.spaces.Discrete(n=obj["n"]) + elif obj["space"] == "Box": + return gym.spaces.Box(low=np.array(obj["low"]), high=np.array(obj["high"]), shape=obj["shape"]) + elif obj["space"] == "MultiDiscrete": + return gym.spaces.MultiDiscrete(nvec=np.array(obj["nvec"])) + elif obj["space"] == "Tuple": + return gym.spaces.Tuple(spaces=tuple(map(deserialize_space, obj["spaces"]))) + elif obj["space"] == "Dict": + return gym.spaces.Dict(spaces={k: deserialize_space(v) for k, v in obj["spaces"].items()}) + else: + raise ValueError(f"Unsupported space ({obj['spaces']})") + # Python data types + elif obj["type"] == "python": + if obj["space"] == "Discrete": + return {obj["value"]} + elif obj["space"] == "Box": + return obj["value"] + elif obj["space"] == "MultiDiscrete": + return [{x} for x in obj["value"]] + elif obj["space"] == "Tuple": + return tuple(map(deserialize_space, obj["value"])) + elif obj["space"] == "Dict": + return {k: deserialize_space(v) for k, v in obj["value"].items()} + else: + raise ValueError(f"Unsupported space ({obj['spaces']})") + else: + raise ValueError(f"Unsupported type ({obj['type']})") + + +def replace_env_cfg_spaces_with_strings(env_cfg: object) -> object: + """Replace spaces objects with their serialized JSON representations in an environment config. + + Args: + env_cfg: Environment config instance. + + Returns: + Environment config instance with spaces replaced if any. + """ + for attr in ["observation_space", "action_space", "state_space"]: + if hasattr(env_cfg, attr): + setattr(env_cfg, attr, serialize_space(getattr(env_cfg, attr))) + for attr in ["observation_spaces", "action_spaces"]: + if hasattr(env_cfg, attr): + setattr(env_cfg, attr, {k: serialize_space(v) for k, v in getattr(env_cfg, attr).items()}) + return env_cfg + + +def replace_strings_with_env_cfg_spaces(env_cfg: object) -> object: + """Replace spaces objects with their serialized JSON representations in an environment config. + + Args: + env_cfg: Environment config instance. + + Returns: + Environment config instance with spaces replaced if any. + """ + for attr in ["observation_space", "action_space", "state_space"]: + if hasattr(env_cfg, attr): + setattr(env_cfg, attr, deserialize_space(getattr(env_cfg, attr))) + for attr in ["observation_spaces", "action_spaces"]: + if hasattr(env_cfg, attr): + setattr(env_cfg, attr, {k: deserialize_space(v) for k, v in getattr(env_cfg, attr).items()}) + return env_cfg diff --git a/source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/ray_caster/ray_caster_camera.py b/source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/ray_caster/ray_caster_camera.py index 03da4ca9b8..91689b03ee 100644 --- a/source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/ray_caster/ray_caster_camera.py +++ b/source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/ray_caster/ray_caster_camera.py @@ -87,7 +87,7 @@ def __str__(self) -> str: f"Ray-Caster-Camera @ '{self.cfg.prim_path}': \n" f"\tview type : {self._view.__class__}\n" f"\tupdate period (s) : {self.cfg.update_period}\n" - f"\tnumber of meshes : {len(RayCaster.meshes)}\n" + f"\tnumber of meshes : {len(self.meshes)}\n" f"\tnumber of sensors : {self._view.count}\n" f"\tnumber of rays/sensor: {self.num_rays}\n" f"\ttotal number of rays : {self.num_rays * self._view.count}\n" diff --git a/source/extensions/omni.isaac.lab/test/envs/test_manager_based_rl_env_ui.py b/source/extensions/omni.isaac.lab/test/envs/test_manager_based_rl_env_ui.py new file mode 100644 index 0000000000..f61af77b8d --- /dev/null +++ b/source/extensions/omni.isaac.lab/test/envs/test_manager_based_rl_env_ui.py @@ -0,0 +1,104 @@ +# Copyright (c) 2022-2024, The Isaac Lab Project Developers. +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# ignore private usage of variables warning +# pyright: reportPrivateUsage=none + +from __future__ import annotations + +"""Launch Isaac Sim Simulator first.""" + +from omni.isaac.lab.app import AppLauncher, run_tests + +# Can set this to False to see the GUI for debugging +HEADLESS = True + +# launch omniverse app +app_launcher = AppLauncher(headless=HEADLESS, enable_cameras=True) +simulation_app = app_launcher.app + +"""Rest everything follows.""" + +import unittest + +import carb +import omni.usd +from omni.isaac.core.utils.extensions import enable_extension + +from omni.isaac.lab.envs import ManagerBasedRLEnv, ManagerBasedRLEnvCfg +from omni.isaac.lab.envs.ui import ManagerBasedRLEnvWindow +from omni.isaac.lab.scene import InteractiveSceneCfg +from omni.isaac.lab.utils import configclass + +enable_extension("omni.isaac.ui") + + +@configclass +class EmptyManagerCfg: + """Empty manager specifications for the environment.""" + + pass + + +@configclass +class EmptySceneCfg(InteractiveSceneCfg): + """Configuration for an empty scene.""" + + pass + + +def get_empty_base_env_cfg(device: str = "cuda:0", num_envs: int = 1, env_spacing: float = 1.0): + """Generate base environment config based on device""" + + @configclass + class EmptyEnvCfg(ManagerBasedRLEnvCfg): + """Configuration for the empty test environment.""" + + # Scene settings + scene: EmptySceneCfg = EmptySceneCfg(num_envs=num_envs, env_spacing=env_spacing) + # Basic settings + actions: EmptyManagerCfg = EmptyManagerCfg() + observations: EmptyManagerCfg = EmptyManagerCfg() + rewards: EmptyManagerCfg = EmptyManagerCfg() + terminations: EmptyManagerCfg = EmptyManagerCfg() + # Define window + ui_window_class_type: ManagerBasedRLEnvWindow = ManagerBasedRLEnvWindow + + def __post_init__(self): + """Post initialization.""" + # step settings + self.decimation = 4 # env step every 4 sim steps: 200Hz / 4 = 50Hz + # simulation settings + self.sim.dt = 0.005 # sim step every 5ms: 200Hz + self.sim.render_interval = self.decimation # render every 4 sim steps + # pass device down from test + self.sim.device = device + # episode length + self.episode_length_s = 5.0 + + return EmptyEnvCfg() + + +class TestManagerBasedRLEnvUI(unittest.TestCase): + """Test for manager-based RL env class UI""" + + """ + Tests + """ + + def test_ui_window(self): + device = "cuda:0" + # override sim setting to enable UI + carb.settings.get_settings().set_bool("/app/window/enabled", True) + # create a new stage + omni.usd.get_context().new_stage() + # create environment + env = ManagerBasedRLEnv(cfg=get_empty_base_env_cfg(device=device)) + # close the environment + env.close() + + +if __name__ == "__main__": + run_tests() diff --git a/source/extensions/omni.isaac.lab/test/envs/test_spaces_utils.py b/source/extensions/omni.isaac.lab/test/envs/test_spaces_utils.py index 274f0de650..791c61380c 100644 --- a/source/extensions/omni.isaac.lab/test/envs/test_spaces_utils.py +++ b/source/extensions/omni.isaac.lab/test/envs/test_spaces_utils.py @@ -26,7 +26,7 @@ import unittest from gymnasium.spaces import Box, Dict, Discrete, MultiDiscrete, Tuple -from omni.isaac.lab.envs.utils.spaces import sample_space, spec_to_gym_space +from omni.isaac.lab.envs.utils.spaces import deserialize_space, sample_space, serialize_space, spec_to_gym_space class TestSpacesUtils(unittest.TestCase): @@ -104,6 +104,59 @@ def test_sample_space(self): self.assertIsInstance(sample, dict) self._check_tensorized(sample, batch_size=5) + def test_space_serialization_deserialization(self): + # fundamental spaces + # Box + space = 1 + output = deserialize_space(serialize_space(space)) + self.assertEqual(space, output) + space = [1, 2, 3, 4, 5] + output = deserialize_space(serialize_space(space)) + self.assertEqual(space, output) + space = Box(low=-1.0, high=1.0, shape=(1, 2)) + output = deserialize_space(serialize_space(space)) + self.assertIsInstance(output, Box) + self.assertTrue((space.low == output.low).all()) + self.assertTrue((space.high == output.high).all()) + self.assertEqual(space.shape, output.shape) + # Discrete + space = {2} + output = deserialize_space(serialize_space(space)) + self.assertEqual(space, output) + space = Discrete(2) + output = deserialize_space(serialize_space(space)) + self.assertIsInstance(output, Discrete) + self.assertEqual(space.n, output.n) + # MultiDiscrete + space = [{1}, {2}, {3}] + output = deserialize_space(serialize_space(space)) + self.assertEqual(space, output) + space = MultiDiscrete(np.array([1, 2, 3])) + output = deserialize_space(serialize_space(space)) + self.assertIsInstance(output, MultiDiscrete) + self.assertTrue((space.nvec == output.nvec).all()) + # composite spaces + # Tuple + space = ([1, 2, 3, 4, 5], {2}, [{1}, {2}, {3}]) + output = deserialize_space(serialize_space(space)) + self.assertEqual(space, output) + space = Tuple((Box(-1, 1, shape=(1,)), Discrete(2))) + output = deserialize_space(serialize_space(space)) + self.assertIsInstance(output, Tuple) + self.assertEqual(len(output), 2) + self.assertIsInstance(output[0], Box) + self.assertIsInstance(output[1], Discrete) + # Dict + space = {"box": [1, 2, 3, 4, 5], "discrete": {2}, "multi_discrete": [{1}, {2}, {3}]} + output = deserialize_space(serialize_space(space)) + self.assertEqual(space, output) + space = Dict({"box": Box(-1, 1, shape=(1,)), "discrete": Discrete(2)}) + output = deserialize_space(serialize_space(space)) + self.assertIsInstance(output, Dict) + self.assertEqual(len(output), 2) + self.assertIsInstance(output["box"], Box) + self.assertIsInstance(output["discrete"], Discrete) + """ Helper functions. """ diff --git a/source/extensions/omni.isaac.lab/test/sensors/test_camera.py b/source/extensions/omni.isaac.lab/test/sensors/test_camera.py index fc6953e166..bc8f8722df 100644 --- a/source/extensions/omni.isaac.lab/test/sensors/test_camera.py +++ b/source/extensions/omni.isaac.lab/test/sensors/test_camera.py @@ -719,6 +719,15 @@ def test_throughput(self): for im_data in camera.data.output.values(): self.assertEqual(im_data.shape, (1, camera_cfg.height, camera_cfg.width, 1)) + def test_sensor_print(self): + """Test sensor print is working correctly.""" + # Create sensor + sensor = Camera(cfg=self.camera_cfg) + # Play sim + self.sim.reset() + # print info + print(sensor) + """ Helper functions. """ diff --git a/source/extensions/omni.isaac.lab/test/sensors/test_contact_sensor.py b/source/extensions/omni.isaac.lab/test/sensors/test_contact_sensor.py index 8390ee2269..6bba29b858 100644 --- a/source/extensions/omni.isaac.lab/test/sensors/test_contact_sensor.py +++ b/source/extensions/omni.isaac.lab/test/sensors/test_contact_sensor.py @@ -294,6 +294,27 @@ def test_cube_stack_contact_filtering(self): contact_sensor_2.data.force_matrix_w[:, :, 0], contact_sensor.data.force_matrix_w[:, :, 0] ) + def test_sensor_print(self): + """Test sensor print is working correctly.""" + with build_simulation_context(device="cuda:0", dt=self.sim_dt, add_lighting=False) as sim: + # Spawn things into stage + scene_cfg = ContactSensorSceneCfg(num_envs=1, env_spacing=1.0, lazy_sensor_update=False) + scene_cfg.terrain = FLAT_TERRAIN_CFG.replace(prim_path="/World/ground") + scene_cfg.shape = CUBE_CFG + scene_cfg.contact_sensor = ContactSensorCfg( + prim_path=scene_cfg.shape.prim_path, + track_pose=True, + debug_vis=False, + update_period=0.0, + track_air_time=True, + history_length=3, + ) + scene = InteractiveScene(scene_cfg) + # Play the simulator + sim.reset() + # print info + print(scene.sensors["contact_sensor"]) + """ Internal helpers. """ diff --git a/source/extensions/omni.isaac.lab/test/sensors/test_frame_transformer.py b/source/extensions/omni.isaac.lab/test/sensors/test_frame_transformer.py index aeab3d9e72..34321e76da 100644 --- a/source/extensions/omni.isaac.lab/test/sensors/test_frame_transformer.py +++ b/source/extensions/omni.isaac.lab/test/sensors/test_frame_transformer.py @@ -579,6 +579,25 @@ def test_frame_transformer_all_bodies(self): torch.testing.assert_close(bodies_pos_source_tf[:, index], body_pos_b) torch.testing.assert_close(bodies_quat_source_tf[:, index], body_quat_b) + def test_sensor_print(self): + """Test sensor print is working correctly.""" + # Spawn things into stage + scene_cfg = MySceneCfg(num_envs=2, env_spacing=5.0, lazy_sensor_update=False) + scene_cfg.frame_transformer = FrameTransformerCfg( + prim_path="{ENV_REGEX_NS}/Robot/base", + target_frames=[ + FrameTransformerCfg.FrameCfg( + prim_path="{ENV_REGEX_NS}/Robot/.*", + ), + ], + ) + scene = InteractiveScene(scene_cfg) + + # Play the simulator + self.sim.reset() + # print info + print(scene.sensors["frame_transformer"]) + if __name__ == "__main__": run_tests() diff --git a/source/extensions/omni.isaac.lab/test/sensors/test_imu.py b/source/extensions/omni.isaac.lab/test/sensors/test_imu.py index ee0db71bf7..e04211d1a9 100644 --- a/source/extensions/omni.isaac.lab/test/sensors/test_imu.py +++ b/source/extensions/omni.isaac.lab/test/sensors/test_imu.py @@ -498,34 +498,40 @@ def test_offset_calculation(self): atol=1e-4, ) + def test_env_ids_propogation(self): + """Test that env_ids argument propagates through update and reset methods""" + self.scene.reset() -def test_env_ids_propogation(self): - """Test that env_ids argument propagates through update and reset methods""" - self.scene.reset() - - for idx in range(10): - # set acceleration - self.scene.articulations["robot"].write_root_velocity_to_sim( - torch.tensor([[0.5, 0.0, 0.0, 0.0, 0.0, 0.0]], dtype=torch.float32, device=self.scene.device).repeat( - self.scene.num_envs, 1 + for idx in range(10): + # set acceleration + self.scene.articulations["robot"].write_root_velocity_to_sim( + torch.tensor([[0.5, 0.0, 0.0, 0.0, 0.0, 0.0]], dtype=torch.float32, device=self.scene.device).repeat( + self.scene.num_envs, 1 + ) + * (idx + 1) ) - * (idx + 1) - ) - # write data to sim - self.scene.write_data_to_sim() + # write data to sim + self.scene.write_data_to_sim() + # perform step + self.sim.step() + # read data from sim + self.scene.update(self.sim.get_physics_dt()) + + # reset scene for env 1 + self.scene.reset(env_ids=[1]) + # read data from sim + self.scene.update(self.sim.get_physics_dt()) # perform step self.sim.step() # read data from sim self.scene.update(self.sim.get_physics_dt()) - # reset scene for env 1 - self.scene.reset(env_ids=[1]) - # read data from sim - self.scene.update(self.sim.get_physics_dt()) - # perform step - self.sim.step() - # read data from sim - self.scene.update(self.sim.get_physics_dt()) + def test_sensor_print(self): + """Test sensor print is working correctly.""" + # Create sensor + sensor = self.scene.sensors["imu_ball"] + # print info + print(sensor) if __name__ == "__main__": diff --git a/source/extensions/omni.isaac.lab/test/sensors/test_ray_caster_camera.py b/source/extensions/omni.isaac.lab/test/sensors/test_ray_caster_camera.py index b28d023c85..ec932429ba 100644 --- a/source/extensions/omni.isaac.lab/test/sensors/test_ray_caster_camera.py +++ b/source/extensions/omni.isaac.lab/test/sensors/test_ray_caster_camera.py @@ -817,6 +817,15 @@ def test_output_equal_to_usd_camera_when_intrinsics_set(self): atol=1e-4, ) + def test_sensor_print(self): + """Test sensor print is working correctly.""" + # Create sensor + sensor = RayCasterCamera(cfg=self.camera_cfg) + # Play sim + self.sim.reset() + # print info + print(sensor) + if __name__ == "__main__": run_tests() diff --git a/source/extensions/omni.isaac.lab/test/sensors/test_tiled_camera.py b/source/extensions/omni.isaac.lab/test/sensors/test_tiled_camera.py index 269cb4bd81..28446a52ef 100644 --- a/source/extensions/omni.isaac.lab/test/sensors/test_tiled_camera.py +++ b/source/extensions/omni.isaac.lab/test/sensors/test_tiled_camera.py @@ -1304,6 +1304,15 @@ def test_output_equal_to_usd_camera_intrinsics(self): del camera_tiled del camera_usd + def test_sensor_print(self): + """Test sensor print is working correctly.""" + # Create sensor + sensor = TiledCamera(cfg=self.camera_cfg) + # Play sim + self.sim.reset() + # print info + print(sensor) + """ Helper functions. """ diff --git a/source/extensions/omni.isaac.lab_tasks/config/extension.toml b/source/extensions/omni.isaac.lab_tasks/config/extension.toml index a739dc74a0..544cd97377 100644 --- a/source/extensions/omni.isaac.lab_tasks/config/extension.toml +++ b/source/extensions/omni.isaac.lab_tasks/config/extension.toml @@ -1,7 +1,7 @@ [package] # Note: Semantic Versioning is used: https://semver.org/ -version = "0.10.9" +version = "0.10.12" # Description title = "Isaac Lab Environments" diff --git a/source/extensions/omni.isaac.lab_tasks/docs/CHANGELOG.rst b/source/extensions/omni.isaac.lab_tasks/docs/CHANGELOG.rst index 5bf5d9eeea..c194591492 100644 --- a/source/extensions/omni.isaac.lab_tasks/docs/CHANGELOG.rst +++ b/source/extensions/omni.isaac.lab_tasks/docs/CHANGELOG.rst @@ -1,6 +1,32 @@ Changelog --------- +0.10.12 (2024-10-28) +~~~~~~~~~~~~~~~~~~~~ + +Changed +^^^^^^^ + +* Changed manager-based vision cartpole environment names from Isaac-Cartpole-RGB-Camera-v0 + and Isaac-Cartpole-Depth-Camera-v0 to Isaac-Cartpole-RGB-v0 and Isaac-Cartpole-Depth-v0 + +0.10.11 (2024-10-28) +~~~~~~~~~~~~~~~~~~~~ + +Added +^^^^^ + +* Added feature extracted observation cartpole examples. + +0.10.10 (2024-10-25) +~~~~~~~~~~~~~~~~~~~~ + +Fixed +^^^^^ + +* Fixed issues with defining Gymnasium spaces in Direct workflows due to Hydra/OmegaConf limitations with non-primitive types. + + 0.10.9 (2024-10-22) ~~~~~~~~~~~~~~~~~~~ diff --git a/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/direct/shadow_hand/feature_extractor.py b/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/direct/shadow_hand/feature_extractor.py index 1dbfb39b1a..fc92bbfb1b 100644 --- a/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/direct/shadow_hand/feature_extractor.py +++ b/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/direct/shadow_hand/feature_extractor.py @@ -7,7 +7,6 @@ import os import torch import torch.nn as nn - import torchvision from omni.isaac.lab.sensors import save_images_to_file diff --git a/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/manager_based/classic/cartpole/__init__.py b/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/manager_based/classic/cartpole/__init__.py index 7a3070d775..43040be70a 100644 --- a/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/manager_based/classic/cartpole/__init__.py +++ b/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/manager_based/classic/cartpole/__init__.py @@ -10,7 +10,12 @@ import gymnasium as gym from . import agents -from .cartpole_camera_env_cfg import CartpoleDepthCameraEnvCfg, CartpoleRGBCameraEnvCfg +from .cartpole_camera_env_cfg import ( + CartpoleDepthCameraEnvCfg, + CartpoleResNet18CameraEnvCfg, + CartpoleRGBCameraEnvCfg, + CartpoleTheiaTinyCameraEnvCfg, +) from .cartpole_env_cfg import CartpoleEnvCfg ## @@ -31,7 +36,7 @@ ) gym.register( - id="Isaac-Cartpole-RGB-Camera-v0", + id="Isaac-Cartpole-RGB-v0", entry_point="omni.isaac.lab.envs:ManagerBasedRLEnv", disable_env_checker=True, kwargs={ @@ -41,7 +46,7 @@ ) gym.register( - id="Isaac-Cartpole-Depth-Camera-v0", + id="Isaac-Cartpole-Depth-v0", entry_point="omni.isaac.lab.envs:ManagerBasedRLEnv", disable_env_checker=True, kwargs={ @@ -49,3 +54,23 @@ "rl_games_cfg_entry_point": f"{agents.__name__}:rl_games_camera_ppo_cfg.yaml", }, ) + +gym.register( + id="Isaac-Cartpole-RGB-ResNet18-v0", + entry_point="omni.isaac.lab.envs:ManagerBasedRLEnv", + disable_env_checker=True, + kwargs={ + "env_cfg_entry_point": CartpoleResNet18CameraEnvCfg, + "rl_games_cfg_entry_point": f"{agents.__name__}:rl_games_feature_ppo_cfg.yaml", + }, +) + +gym.register( + id="Isaac-Cartpole-RGB-TheiaTiny-v0", + entry_point="omni.isaac.lab.envs:ManagerBasedRLEnv", + disable_env_checker=True, + kwargs={ + "env_cfg_entry_point": CartpoleTheiaTinyCameraEnvCfg, + "rl_games_cfg_entry_point": f"{agents.__name__}:rl_games_feature_ppo_cfg.yaml", + }, +) diff --git a/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/manager_based/classic/cartpole/agents/rl_games_feature_ppo_cfg.yaml b/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/manager_based/classic/cartpole/agents/rl_games_feature_ppo_cfg.yaml new file mode 100644 index 0000000000..18e0ffd022 --- /dev/null +++ b/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/manager_based/classic/cartpole/agents/rl_games_feature_ppo_cfg.yaml @@ -0,0 +1,79 @@ +params: + seed: 42 + + # environment wrapper clipping + env: + # added to the wrapper + clip_observations: 5.0 + # can make custom wrapper? + clip_actions: 1.0 + + algo: + name: a2c_continuous + + model: + name: continuous_a2c_logstd + + # doesn't have this fine grained control but made it close + network: + name: actor_critic + separate: False + space: + continuous: + mu_activation: None + sigma_activation: None + + mu_init: + name: default + sigma_init: + name: const_initializer + val: 0 + fixed_sigma: True + mlp: + units: [256] + activation: elu + d2rl: False + + initializer: + name: default + regularizer: + name: None + + load_checkpoint: False # flag which sets whether to load the checkpoint + load_path: '' # path to the checkpoint to load + + config: + name: cartpole_features + env_name: rlgpu + device: 'cuda:0' + device_name: 'cuda:0' + multi_gpu: False + ppo: True + mixed_precision: False + normalize_input: True + normalize_value: True + value_bootstraop: True + num_actors: -1 # configured from the script (based on num_envs) + reward_shaper: + scale_value: 1.0 + normalize_advantage: True + gamma: 0.99 + tau : 0.95 + learning_rate: 3e-4 + lr_schedule: adaptive + kl_threshold: 0.008 + score_to_win: 20000 + max_epochs: 5000 + save_best_after: 50 + save_frequency: 25 + grad_norm: 1.0 + entropy_coef: 0.0 + truncate_grads: True + e_clip: 0.2 + horizon_length: 16 + minibatch_size: 2048 + mini_epochs: 8 + critic_coef: 4 + clip_value: True + seq_length: 4 + bounds_loss_coef: 0.0001 diff --git a/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/manager_based/classic/cartpole/cartpole_camera_env_cfg.py b/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/manager_based/classic/cartpole/cartpole_camera_env_cfg.py index ce5a6c90b8..f767a21962 100644 --- a/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/manager_based/classic/cartpole/cartpole_camera_env_cfg.py +++ b/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/manager_based/classic/cartpole/cartpole_camera_env_cfg.py @@ -78,7 +78,7 @@ class DepthObservationsCfg: """Observation specifications for the MDP.""" @configclass - class DepthCameraPolicyCfg(RGBObservationsCfg.RGBCameraPolicyCfg): + class DepthCameraPolicyCfg(ObsGroup): """Observations for policy group with depth images.""" image = ObsTerm( @@ -88,6 +88,43 @@ class DepthCameraPolicyCfg(RGBObservationsCfg.RGBCameraPolicyCfg): policy: ObsGroup = DepthCameraPolicyCfg() +@configclass +class ResNet18ObservationCfg: + """Observation specifications for the MDP.""" + + @configclass + class ResNet18FeaturesCameraPolicyCfg(ObsGroup): + """Observations for policy group with features extracted from RGB images with a frozen ResNet18.""" + + image = ObsTerm( + func=mdp.image_features, + params={"sensor_cfg": SceneEntityCfg("tiled_camera"), "data_type": "rgb", "model_name": "resnet18"}, + ) + + policy: ObsGroup = ResNet18FeaturesCameraPolicyCfg() + + +@configclass +class TheiaTinyObservationCfg: + """Observation specifications for the MDP.""" + + @configclass + class TheiaTinyFeaturesCameraPolicyCfg(ObsGroup): + """Observations for policy group with features extracted from RGB images with a frozen Theia-Tiny Transformer""" + + image = ObsTerm( + func=mdp.image_features, + params={ + "sensor_cfg": SceneEntityCfg("tiled_camera"), + "data_type": "rgb", + "model_name": "theia-tiny-patch16-224-cddsv", + "model_device": "cuda:0", + }, + ) + + policy: ObsGroup = TheiaTinyFeaturesCameraPolicyCfg() + + ## # Environment configuration ## @@ -107,3 +144,20 @@ class CartpoleDepthCameraEnvCfg(CartpoleEnvCfg): scene: CartpoleSceneCfg = CartpoleDepthCameraSceneCfg(num_envs=1024, env_spacing=20) observations: DepthObservationsCfg = DepthObservationsCfg() + + +@configclass +class CartpoleResNet18CameraEnvCfg(CartpoleRGBCameraEnvCfg): + observations: ResNet18ObservationCfg = ResNet18ObservationCfg() + + +@configclass +class CartpoleTheiaTinyCameraEnvCfg(CartpoleRGBCameraEnvCfg): + """ + Due to TheiaTiny's size in GPU memory, we reduce the number of environments by default. + This helps reduce the possibility of crashing on more modest hardware. + The following configuration uses ~12gb VRAM at peak. + """ + + scene: CartpoleSceneCfg = CartpoleRGBCameraSceneCfg(num_envs=128, env_spacing=20) + observations: TheiaTinyObservationCfg = TheiaTinyObservationCfg() diff --git a/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/manager_based/manipulation/lift/config/franka/__init__.py b/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/manager_based/manipulation/lift/config/franka/__init__.py index 14657e4f92..651d84ed63 100644 --- a/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/manager_based/manipulation/lift/config/franka/__init__.py +++ b/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/manager_based/manipulation/lift/config/franka/__init__.py @@ -23,6 +23,7 @@ "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:LiftCubePPORunnerCfg", "skrl_cfg_entry_point": f"{agents.__name__}:skrl_ppo_cfg.yaml", "rl_games_cfg_entry_point": f"{agents.__name__}:rl_games_ppo_cfg.yaml", + "sb3_cfg_entry_point": f"{agents.__name__}:sb3_ppo_cfg.yaml", }, disable_env_checker=True, ) @@ -35,6 +36,7 @@ "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:LiftCubePPORunnerCfg", "skrl_cfg_entry_point": f"{agents.__name__}:skrl_ppo_cfg.yaml", "rl_games_cfg_entry_point": f"{agents.__name__}:rl_games_ppo_cfg.yaml", + "sb3_cfg_entry_point": f"{agents.__name__}:sb3_ppo_cfg.yaml", }, disable_env_checker=True, ) diff --git a/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/manager_based/manipulation/lift/config/franka/agents/sb3_ppo_cfg.yaml b/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/manager_based/manipulation/lift/config/franka/agents/sb3_ppo_cfg.yaml index 57960b8822..6d6f15781a 100644 --- a/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/manager_based/manipulation/lift/config/franka/agents/sb3_ppo_cfg.yaml +++ b/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/manager_based/manipulation/lift/config/franka/agents/sb3_ppo_cfg.yaml @@ -16,7 +16,7 @@ learning_rate: !!float 3e-4 clip_range: 0.2 policy_kwargs: "dict( activation_fn=nn.ELU, - net_arch=[32, 32, dict(pi=[256, 128, 64], vf=[256, 128, 64])] + net_arch=dict(pi=[256, 128, 64], vf=[256, 128, 64]) )" target_kl: 0.01 max_grad_norm: 1.0 diff --git a/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/utils/hydra.py b/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/utils/hydra.py index 1522e9f25e..5d6a8b7c61 100644 --- a/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/utils/hydra.py +++ b/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/utils/hydra.py @@ -17,6 +17,7 @@ raise ImportError("Hydra is not installed. Please install it by running 'pip install hydra-core'.") from omni.isaac.lab.envs import DirectRLEnvCfg, ManagerBasedRLEnvCfg +from omni.isaac.lab.envs.utils.spaces import replace_env_cfg_spaces_with_strings, replace_strings_with_env_cfg_spaces from omni.isaac.lab.utils import replace_slices_with_strings, replace_strings_with_slices from omni.isaac.lab_tasks.utils.parse_cfg import load_cfg_from_registry @@ -40,6 +41,9 @@ def register_task_to_hydra( # load the configurations env_cfg = load_cfg_from_registry(task_name, "env_cfg_entry_point") agent_cfg = load_cfg_from_registry(task_name, agent_cfg_entry_point) + # replace gymnasium spaces with strings because OmegaConf does not support them. + # this must be done before converting the env configs to dictionary to avoid internal reinterpretations + replace_env_cfg_spaces_with_strings(env_cfg) # convert the configs to dictionary env_cfg_dict = env_cfg.to_dict() if isinstance(agent_cfg, dict): @@ -83,6 +87,10 @@ def hydra_main(hydra_env_cfg: DictConfig, env_cfg=env_cfg, agent_cfg=agent_cfg): hydra_env_cfg = replace_strings_with_slices(hydra_env_cfg) # update the configs with the Hydra command line arguments env_cfg.from_dict(hydra_env_cfg["env"]) + # replace strings that represent gymnasium spaces because OmegaConf does not support them. + # this must be done after converting the env configs from dictionary to avoid internal reinterpretations + replace_strings_with_env_cfg_spaces(env_cfg) + # get agent configs if isinstance(agent_cfg, dict): agent_cfg = hydra_env_cfg["agent"] else: