Integrate dmcontrol updated (#55)

* Add dm_control to requirements

* Add/transform dm_control/Mujoco into CARLEnv

We could also use the Mujoco envs from gym, but they don't have every env and this way we have more options to modify the envs (it is at least more obvious).
There are still some open todos.

* adapt observations and render rgb

* change dmc load, simplify wrapper

* dmc initializations

* refactor, load context into dmc cartpole

* remove print

* update cartpole context

* general dmc env

* add walker

* fix load dmc

* update walker context

* camera id in render

* dmc test function

* add more context tasks to walker

* add quadruped

* add dmc fish

* walker context defaults, adapt context utils

* update adapt context function

* quadruped adapt context utils

* update dmc context parameters

* rename utils->loader

* add masking to dmc envs

* remove cartpole env

* formatting

* dmc env less duplicate code

* Rename/move file

* Fix tests

* Change default tasks

* Adjust episode lengths to 1000

According to dm control settings.

* add docstring

* update readme

* flake8 formatting

* dmc dict observation space

* import list

* formatting

* black and isort formatting

Co-authored-by: Carolin Benjamins <benjamins@tnt.uni-hannover.de>

README.md

@@ -24,6 +24,8 @@ Benchmarks include:
 - [RNADesign](https://github.com/automl/learna/), an environment for RNA design given structure
   constraints with structures from different datasets to choose from
 
+- [dm_control](https://github.com/deepmind/dm_control), environments based on the MuJoCo physics engine. The environments are extended with different context features.
+
 ![Screenshot of each environment included in CARL.](./docs/source/figures/envs_overview.png)
 
 For more information, check out our [documentation](https://carl.readthedocs.io/en/latest/)!
@@ -42,7 +44,7 @@ pip install .
 
 This will only install the basic classic control environments, which should run on most operating systems. For the full set of environments, use the install options:
 ```bash
-pip install -e .[box2d, brax, rna, mario]
+pip install -e .[box2d, brax, rna, mario, dm_control]
 ```
 
 These may not be compatible with Windows systems. Box2D environment may need to be installed via conda on MacOS systems:
@@ -95,6 +97,8 @@ Awiszus et al., AIIDE 2020](https://arxiv.org/pdf/2008.01531.pdf)
 
 [Learning to Design RNA, Runge et al., ICRL 2019](https://arxiv.org/pdf/1812.11951.pdf)
 
+[dm_control: Software and Tasks for Continuous Control](https://arxiv.org/pdf/2006.12983.pdf)
+
 ## License
 CARL falls under the Apache License 2.0 (see file 'LICENSE') as is permitted by all
 work that we use. This includes CARLMario, which is not based on the Nintendo Game, but on

carl/__init__.py

@@ -1,6 +1,4 @@
 __copyright__ = "Copyright 2021, AutoML.org Freiburg-Hannover"
 __license__ = "Apache-2.0 License"
 __version__ = "0.1"
-__author__ = (
-    "Carolin Benjamins, Theresa Eimer, Frederik Schubert, André Biedenkapp, Aditya Mohan"
-)
+__author__ = "Carolin Benjamins, Theresa Eimer, Frederik Schubert, André Biedenkapp, Aditya Mohan"

carl/context/selection.py

@@ -1,7 +1,9 @@
 from abc import abstractmethod
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
 import numpy as np
+
 from carl.utils.types import Context
-from typing import Dict, Any, Optional, Tuple, List, Callable
 
 
 class AbstractSelector(object):

carl/envs/__init__.py

@@ -20,6 +20,7 @@
 found = brax_spec is not None
 if found:
     from carl.envs.brax import *
+
     pass
 else:
     warnings.warn(

carl/envs/box2d/carl_bipedal_walker.py

@@ -1,16 +1,13 @@
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Union
 
-import Box2D
 import numpy as np
 from Box2D.b2 import edgeShape, fixtureDef, polygonShape
-from gym import spaces
 from gym.envs.box2d import bipedal_walker
 from gym.envs.box2d import bipedal_walker as bpw
-from gym.utils import EzPickle
 
+from carl.context.selection import AbstractSelector
 from carl.envs.carl_env import CARLEnv
 from carl.utils.trial_logger import TrialLogger
-from carl.context.selection import AbstractSelector
 
 DEFAULT_CONTEXT = {
     "FPS": 50,
@@ -91,7 +88,9 @@ def __init__(
         state_context_features: Optional[List[str]] = None,
         context_mask: Optional[List[str]] = None,
         dict_observation_space: bool = False,
-        context_selector: Optional[Union[AbstractSelector, type(AbstractSelector)]] = None,
+        context_selector: Optional[
+            Union[AbstractSelector, type(AbstractSelector)]
+        ] = None,
         context_selector_kwargs: Optional[Dict] = None,
     ):
         """
@@ -122,7 +121,6 @@ def __init__(
             context_selector=context_selector,
             context_selector_kwargs=context_selector_kwargs,
             context_mask=context_mask,
-
         )
         self.whitelist_gaussian_noise = list(
             DEFAULT_CONTEXT.keys()
@@ -197,7 +195,9 @@ def _update_context(self):
         self.env.world.gravity = gravity
 
 
-def demo_heuristic(env: Union[CARLBipedalWalkerEnv, bipedal_walker.BipedalWalker]) -> None:
+def demo_heuristic(
+    env: Union[CARLBipedalWalkerEnv, bipedal_walker.BipedalWalker]
+) -> None:
     env.reset()
     steps = 0
     total_reward = 0

carl/envs/box2d/carl_lunarlander.py

@@ -1,17 +1,12 @@
-from typing import Any, Dict, List, Optional, Tuple, Union, TypeVar
+from typing import Any, Dict, List, Optional, Tuple, TypeVar, Union
 
-import Box2D
-import numpy as np
-from gym import spaces
 from gym import Wrapper
-
 from gym.envs.box2d import lunar_lander
 from gym.envs.box2d.lunar_lander import heuristic
-from gym.utils import EzPickle, seeding
 
+from carl.context.selection import AbstractSelector
 from carl.envs.carl_env import CARLEnv
 from carl.utils.trial_logger import TrialLogger
-from carl.context.selection import AbstractSelector
 
 ObsType = TypeVar("ObsType")
 ActType = TypeVar("ActType")
@@ -78,9 +73,9 @@
 
 class LunarLanderEnv(Wrapper):
     def __init__(
-            self,
-            env: Optional[lunar_lander.LunarLander] = None,
-            high_gameover_penalty: bool = False,
+        self,
+        env: Optional[lunar_lander.LunarLander] = None,
+        high_gameover_penalty: bool = False,
     ):
         if env is None:
             env = lunar_lander.LunarLander()
@@ -117,7 +112,9 @@ def __init__(
         max_episode_length: int = 1000,
         high_gameover_penalty: bool = False,
         dict_observation_space: bool = False,
-        context_selector: Optional[Union[AbstractSelector, type(AbstractSelector)]] = None,
+        context_selector: Optional[
+            Union[AbstractSelector, type(AbstractSelector)]
+        ] = None,
         context_selector_kwargs: Optional[Dict] = None,
     ):
         """
@@ -182,7 +179,9 @@ def _update_context(self) -> None:
 
 
 def demo_heuristic_lander(
-    env: Union[CARLLunarLanderEnv, lunar_lander.LunarLander, lunar_lander.LunarLanderContinuous],
+    env: Union[
+        CARLLunarLanderEnv, lunar_lander.LunarLander, lunar_lander.LunarLanderContinuous
+    ],
     seed: Optional[int] = None,
     render: bool = False,
 ) -> float:

carl/envs/box2d/carl_vehicle_racing.py

@@ -6,6 +6,7 @@
 from gym.envs.box2d.car_dynamics import Car
 from pyglet import gl
 
+from carl.context.selection import AbstractSelector
 from carl.envs.box2d.parking_garage.bus import AWDBus  # as Car
 from carl.envs.box2d.parking_garage.bus import AWDBusLargeTrailer  # as Car
 from carl.envs.box2d.parking_garage.bus import AWDBusSmallTrailer  # as Car
@@ -37,7 +38,6 @@
 from carl.envs.box2d.parking_garage.trike import TukTukSmallTrailer  # as Car
 from carl.envs.carl_env import CARLEnv
 from carl.utils.trial_logger import TrialLogger
-from carl.context.selection import AbstractSelector
 
 PARKING_GARAGE_DICT = {
     # Racing car
@@ -196,7 +196,9 @@ def __init__(
         state_context_features: Optional[List[str]] = None,
         context_mask: Optional[List[str]] = None,
         dict_observation_space: bool = False,
-        context_selector: Optional[Union[AbstractSelector, type(AbstractSelector)]] = None,
+        context_selector: Optional[
+            Union[AbstractSelector, type(AbstractSelector)]
+        ] = None,
         context_selector_kwargs: Optional[Dict] = None,
     ):
         """

carl/envs/brax/carl_ant.py

@@ -6,14 +6,14 @@
 import brax
 import numpy as np
 from brax.envs.ant import _SYSTEM_CONFIG, Ant
-from brax.envs.wrappers import GymWrapper, VectorWrapper, VectorGymWrapper
+from brax.envs.wrappers import GymWrapper, VectorGymWrapper, VectorWrapper
 from google.protobuf import json_format, text_format
 from google.protobuf.json_format import MessageToDict
 from numpyencoder import NumpyEncoder
 
+from carl.context.selection import AbstractSelector
 from carl.envs.carl_env import CARLEnv
 from carl.utils.trial_logger import TrialLogger
-from carl.context.selection import AbstractSelector
 
 DEFAULT_CONTEXT = {
     "joint_stiffness": 5000,
@@ -38,22 +38,23 @@
 
 class CARLAnt(CARLEnv):
     def __init__(
-            self,
-            env: Ant = Ant(),
-            n_envs: int = 1,
-            contexts: Dict[str, Dict] = {},
-            hide_context=False,
-            add_gaussian_noise_to_context: bool = False,
-            gaussian_noise_std_percentage: float = 0.01,
-            logger: Optional[TrialLogger] = None,
-            scale_context_features: str = "no",
-            default_context: Optional[Dict] = DEFAULT_CONTEXT,
-            state_context_features: Optional[List[str]] = None,
-            context_mask: Optional[List[str]] = None,
-            dict_observation_space: bool = False,
-            context_selector: Optional[Union[AbstractSelector, type(AbstractSelector)]] = None,
-            context_selector_kwargs: Optional[Dict] = None,
-
+        self,
+        env: Ant = Ant(),
+        n_envs: int = 1,
+        contexts: Dict[str, Dict] = {},
+        hide_context=False,
+        add_gaussian_noise_to_context: bool = False,
+        gaussian_noise_std_percentage: float = 0.01,
+        logger: Optional[TrialLogger] = None,
+        scale_context_features: str = "no",
+        default_context: Optional[Dict] = DEFAULT_CONTEXT,
+        state_context_features: Optional[List[str]] = None,
+        context_mask: Optional[List[str]] = None,
+        dict_observation_space: bool = False,
+        context_selector: Optional[
+            Union[AbstractSelector, type(AbstractSelector)]
+        ] = None,
+        context_selector_kwargs: Optional[Dict] = None,
     ):
         if n_envs == 1:
             env = GymWrapper(env)

carl/envs/brax/carl_fetch.py

@@ -6,14 +6,14 @@
 import brax
 import numpy as np
 from brax.envs.fetch import _SYSTEM_CONFIG, Fetch
-from brax.envs.wrappers import GymWrapper, VectorWrapper, VectorGymWrapper
+from brax.envs.wrappers import GymWrapper, VectorGymWrapper, VectorWrapper
 from google.protobuf import json_format, text_format
 from google.protobuf.json_format import MessageToDict
 from numpyencoder import NumpyEncoder
 
+from carl.context.selection import AbstractSelector
 from carl.envs.carl_env import CARLEnv
 from carl.utils.trial_logger import TrialLogger
-from carl.context.selection import AbstractSelector
 
 DEFAULT_CONTEXT = {
     "joint_stiffness": 5000,
@@ -55,7 +55,9 @@ def __init__(
         state_context_features: Optional[List[str]] = None,
         context_mask: Optional[List[str]] = None,
         dict_observation_space: bool = False,
-        context_selector: Optional[Union[AbstractSelector, type(AbstractSelector)]] = None,
+        context_selector: Optional[
+            Union[AbstractSelector, type(AbstractSelector)]
+        ] = None,
         context_selector_kwargs: Optional[Dict] = None,
     ):
         if n_envs == 1:

carl/envs/brax/carl_grasp.py

@@ -6,14 +6,14 @@
 import brax
 import numpy as np
 from brax.envs.grasp import _SYSTEM_CONFIG, Grasp
-from brax.envs.wrappers import GymWrapper, VectorWrapper, VectorGymWrapper
+from brax.envs.wrappers import GymWrapper, VectorGymWrapper, VectorWrapper
 from google.protobuf import json_format, text_format
 from google.protobuf.json_format import MessageToDict
 from numpyencoder import NumpyEncoder
 
+from carl.context.selection import AbstractSelector
 from carl.envs.carl_env import CARLEnv
 from carl.utils.trial_logger import TrialLogger
-from carl.context.selection import AbstractSelector
 
 DEFAULT_CONTEXT = {
     "joint_stiffness": 5000,
@@ -55,7 +55,9 @@ def __init__(
         state_context_features: Optional[List[str]] = None,
         context_mask: Optional[List[str]] = None,
         dict_observation_space: bool = False,
-        context_selector: Optional[Union[AbstractSelector, type(AbstractSelector)]] = None,
+        context_selector: Optional[
+            Union[AbstractSelector, type(AbstractSelector)]
+        ] = None,
         context_selector_kwargs: Optional[Dict] = None,
     ):
         if n_envs == 1:

carl/envs/brax/carl_halfcheetah.py

@@ -6,14 +6,14 @@
 import brax
 import numpy as np
 from brax.envs.halfcheetah import _SYSTEM_CONFIG, Halfcheetah
-from brax.envs.wrappers import GymWrapper, VectorWrapper, VectorGymWrapper
+from brax.envs.wrappers import GymWrapper, VectorGymWrapper, VectorWrapper
 from google.protobuf import json_format, text_format
 from google.protobuf.json_format import MessageToDict
 from numpyencoder import NumpyEncoder
 
+from carl.context.selection import AbstractSelector
 from carl.envs.carl_env import CARLEnv
 from carl.utils.trial_logger import TrialLogger
-from carl.context.selection import AbstractSelector
 
 DEFAULT_CONTEXT = {
     "joint_stiffness": 15000.0,
@@ -49,7 +49,9 @@ def __init__(
         state_context_features: Optional[List[str]] = None,
         context_mask: Optional[List[str]] = None,
         dict_observation_space: bool = False,
-        context_selector: Optional[Union[AbstractSelector, type(AbstractSelector)]] = None,
+        context_selector: Optional[
+            Union[AbstractSelector, type(AbstractSelector)]
+        ] = None,
         context_selector_kwargs: Optional[Dict] = None,
     ):
         if n_envs == 1:

carl/envs/brax/carl_humanoid.py

@@ -3,20 +3,19 @@
 import copy
 import json
 
-import numpy as np
 import brax
+import numpy as np
 from brax import jumpy as jp
-from brax.envs.wrappers import GymWrapper, VectorWrapper, VectorGymWrapper
-from brax.envs.humanoid import Humanoid, _SYSTEM_CONFIG
+from brax.envs.humanoid import _SYSTEM_CONFIG, Humanoid
+from brax.envs.wrappers import GymWrapper, VectorGymWrapper, VectorWrapper
 from brax.physics import bodies
-
 from google.protobuf import json_format, text_format
 from google.protobuf.json_format import MessageToDict
 from numpyencoder import NumpyEncoder
 
+from carl.context.selection import AbstractSelector
 from carl.envs.carl_env import CARLEnv
 from carl.utils.trial_logger import TrialLogger
-from carl.context.selection import AbstractSelector
 
 DEFAULT_CONTEXT = {
     "gravity": -9.8,
@@ -50,7 +49,9 @@ def __init__(
         state_context_features: Optional[List[str]] = None,
         context_mask: Optional[List[str]] = None,
         dict_observation_space: bool = False,
-        context_selector: Optional[Union[AbstractSelector, type(AbstractSelector)]] = None,
+        context_selector: Optional[
+            Union[AbstractSelector, type(AbstractSelector)]
+        ] = None,
         context_selector_kwargs: Optional[Dict] = None,
     ):
         if n_envs == 1: