Merge branch 'development' into doc_update

.flake8

@@ -10,3 +10,6 @@ extend-ignore =
     E203
     # No lambdas — too strict
     E731
+    E722
+    F405
+    F403

.gitignore

@@ -16,16 +16,23 @@ CARL.egg-info
 carl.egg-info
 .mypy_cache
 .pytest_cache
-.coverage
+.coverage*
 exp_sweep
 multirun
 outputs
+testvenv
+*.egg-info
 runs
-*.tex
 *.png
 *.pdf
 *.csv
-*.json
 *.pickle
-*.egg-info
-*code-workspace
+*.ipynb_checkpoints
+*optgap*
+*smac3*
+*.json
+generated
+core
+*.tex
+build
+target

.gitmodules

@@ -1,9 +1,6 @@
-[submodule "src/envs/rna/learna"]
-	path = src/envs/rna/learna
-	url = https://github.com/automl/learna.git
-[submodule "src/envs/mario/TOAD-GUI"]
-	path = src/envs/mario/TOAD-GUI
+[submodule "carl/envs/mario/TOAD-GUI"]
+	path = carl/envs/mario/TOAD-GUI
 	url = https://github.com/Mawiszus/TOAD-GUI
-[submodule "src/envs/mario/Mario-AI-Framework"]
-	path = src/envs/mario/Mario-AI-Framework
+[submodule "carl/envs/mario/Mario-AI-Framework"]
+	path = carl/envs/mario/Mario-AI-Framework
 	url = https://github.com/frederikschubert/Mario-AI-Framework

.pre-commit-config.yaml

@@ -43,13 +43,6 @@ repos:
         always_run: false
         additional_dependencies: ["toml"] # Needed to parse pyproject.toml
 
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.930
-    hooks:
-      - id: mypy
-        name: mypy carl
-        files: carl/.*
-
   - repo: https://github.com/pycqa/flake8
     rev: 6.0.0
     hooks:

Makefile

@@ -24,7 +24,7 @@ PYTEST ?= python -m pytest
 CTAGS ?= ctags
 PIP ?= python -m pip
 MAKE ?= make
-BLACK ?= black
+BLACK ?= python -m black
 ISORT ?= isort
 PYDOCSTYLE ?= pydocstyle
 MYPY ?= mypy

README.md

@@ -41,7 +41,7 @@ pip install .
 
 This will only install the basic classic control environments, which should run on most operating systems. For the full set of environments, use the install options:
 ```bash
-pip install -e .[box2d,brax,mario,dm_control]
+pip install -e .[box2d,brax,dm_control,mario,rna]
 ```
 
 These may not be compatible with Windows systems. Box2D environment may need to be installed via conda on MacOS systems:
@@ -50,13 +50,26 @@ conda install -c conda-forge gym-box2d
 ```
 
 In general, we test on Linux systems, but aim to keep the benchmark compatible with MacOS as much as possible.
-Mario at this point, however, will not run on any operation system besides Linux
+RNA and Mario at this point, however, will not run on any operation system besides Linux.
 
-To install the additional requirements for ToadGAN:
+To install ToadGAN for the Mario environment:
 ```bash
-javac carl/envs/mario/Mario-AI-Framework/**/*.java
+git submodule update --init --recursive
+
+# if this does not work, clone manually
+git clone https://github.com/frederikschubert/Mario-AI-Framework carl/envs/mario/Mario-AI-Framework
+git clone https://github.com/Mawiszus/TOAD-GUI  carl/envs/mario/TOAD-GUI
+
+# System requirements
+sudo apt install libfreetype6-dev xvfb
+
+# Compile java source files
+cd carl/envs/mario/Mario-AI-Framework/src
+javac *.java
 ```
 
+If you want to use RNA, please take a look at the associated [ReadME](carl/envs/rna/readme.md).
+
 ## CARL's Contextual Extension
 CARL contextually extends the environment by making the context visible and configurable.
 During training we therefore can encounter different contexts and train for generalization.
@@ -68,16 +81,22 @@ Different instiations can be achieved by setting the context features to differe
 ## Cite Us
 If you use CARL in your research, please cite our paper on the benchmark:
 ```bibtex
-@inproceedings{BenEim2021a,
-    title     = {CARL: A Benchmark for Contextual and Adaptive Reinforcement Learning},
-    author    = {Carolin Benjamins and Theresa Eimer and Frederik Schubert and André Biedenkapp and Bodo Rosenhahn and Frank Hutter and Marius Lindauer},
-    booktitle = {NeurIPS 2021 Workshop on Ecological Theory of Reinforcement Learning},
-    year      = {2021},
-    month     = dec
+@inproceedings { BenEim2023a,
+  author       = {Carolin Benjamins and
+                  Theresa Eimer and
+                  Frederik Schubert and
+                  Aditya Mohan and
+                  Sebastian Döhler and
+                  André Biedenkapp and
+                  Bodo Rosenhahn and
+                  Frank Hutter and
+                  Marius Lindauer},
+  title        = {Contextualize Me - The Case for Context in Reinforcement Learning},
+  journal      = {Transactions on Machine Learning Research},
+  year         = {2023},
 }
-```
 
-You can find the code and experiments for this paper in the `neurips_ecorl_workshop_2021` branch.
+```
 
 ## References
 [OpenAI gym, Brockman et al., 2016. arXiv preprint arXiv:1606.01540](https://arxiv.org/pdf/1606.01540.pdf)

carl/__init__.py

@@ -13,7 +13,7 @@
 description = "CARL- Contextually Adaptive Reinforcement Learning"
 url = "https://www.automl.org/"
 project_urls = {
-    "Documentation": "https://carl.readthedocs.io/en/latest/",
+    "Documentation": "https://automl.github.io/CARL",
     "Source Code": "https://github.com/https://github.com/automl/CARL",
 }
 copyright = f"""

carl/context/context_space.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+from typing import List
+
+import gymnasium.spaces as spaces
+import numpy as np
+from ConfigSpace.hyperparameters import (
+    CategoricalHyperparameter,
+    Hyperparameter,
+    NormalFloatHyperparameter,
+    NumericalHyperparameter,
+    UniformFloatHyperparameter,
+    UniformIntegerHyperparameter,
+)
+from typing_extensions import TypeAlias
+
+from carl.utils.types import Context, Contexts
+
+ContextFeature: TypeAlias = Hyperparameter
+NumericalContextFeature: TypeAlias = NumericalHyperparameter
+NormalFloatContextFeature: TypeAlias = NormalFloatHyperparameter
+UniformFloatContextFeature: TypeAlias = UniformFloatHyperparameter
+UniformIntegerContextFeature: TypeAlias = UniformIntegerHyperparameter
+CategoricalContextFeature: TypeAlias = CategoricalHyperparameter
+
+
+class ContextSpace(object):
+    def __init__(self, context_space: dict[str, ContextFeature]) -> None:
+        self.context_space = context_space
+
+    @property
+    def context_feature_names(self) -> list[str]:
+        """
+        Context feature names.
+
+        Returns
+        -------
+        list[str]
+            Context features names.
+        """
+        return list(self.context_space.keys())
+
+    def insert_defaults(
+        self, context: Context, context_keys: List[str] | None = None
+    ) -> Context:
+        context_with_defaults = self.get_default_context()
+
+        # insert defaults only for certain keys
+        if context_keys:
+            context_with_defaults = {
+                key: context_with_defaults[key] for key in context_keys
+            }
+
+        context_with_defaults.update(context)
+        return context_with_defaults
+
+    def verify_context(self, context: Context) -> bool:
+        is_valid = True
+        cfs = self.context_feature_names
+        for cfname, v in context.items():
+            # Check if context feature exists in space
+            # by checking name
+            if cfname not in cfs:
+                is_valid = False
+                break
+
+            # Check if context feature value is in bounds
+            cf = self.context_space[cfname]
+            if isinstance(cf, NumericalContextFeature):
+                if not (cf.lower <= v <= cf.upper):
+                    is_valid = False
+                    break
+        return is_valid
+
+    def get_default_context(self) -> Context:
+        context = {cf.name: cf.default_value for cf in self.context_space.values()}
+        return context
+
+    def get_lower_and_upper_bound(
+        self, context_feature_name: str
+    ) -> tuple[float, float]:
+        cf = self.context_space[context_feature_name]
+        bounds = (cf.lower, cf.upper)
+        return bounds
+
+    def to_gymnasium_space(
+        self, context_feature_names: List[str] | None = None, as_dict: bool = False
+    ) -> spaces.Space:
+        if context_feature_names is None:
+            context_feature_names = self.context_feature_names
+        if as_dict:
+            context_space = {}
+
+            for cf_name in context_feature_names:
+                context_feature = self.context_space[cf_name]
+                if isinstance(context_feature, NumericalContextFeature):
+                    context_space[context_feature.name] = spaces.Box(
+                        low=context_feature.lower, high=context_feature.upper
+                    )
+                else:
+                    context_space[context_feature.name] = spaces.Discrete(
+                        len(context_feature.choices)
+                    )
+            return spaces.Dict(context_space)
+        else:
+            low = np.array(
+                [self.context_space[cf].lower for cf in context_feature_names]
+            )
+            high = np.array(
+                [self.context_space[cf].upper for cf in context_feature_names]
+            )
+
+            return spaces.Box(low=low, high=high, dtype=np.float32)
+
+    def sample_contexts(
+        self, context_keys: List[str] | None = None, size: int = 1
+    ) -> Context | List[Contexts]:
+        if context_keys is None:
+            context_keys = self.context_space.keys()
+        else:
+            for key in context_keys:
+                if key not in self.context_space.keys():
+                    raise ValueError(f"Invalid context feature name: {key}")
+
+        contexts = []
+        for _ in range(size):
+            context = {cf.name: cf.sample() for cf in self.context_space.values()}
+            context = self.insert_defaults(context, context_keys)
+            contexts += [context]
+
+        if size == 1:
+            return contexts[0]
+        else:
+            return contexts

carl/context/sampler.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from ConfigSpace import ConfigurationSpace
+from omegaconf import DictConfig
+
+from carl.context.context_space import ContextFeature, ContextSpace
+from carl.context.search_space_encoding import search_space_to_config_space
+from carl.utils.types import Context, Contexts
+
+
+class ContextSampler(ConfigurationSpace):
+    def __init__(
+        self,
+        context_distributions: list[ContextFeature]
+        | dict[str, ContextFeature]
+        | str
+        | DictConfig,
+        context_space: ContextSpace,
+        seed: int,
+        name: str | None = None,
+    ):
+        self.context_distributions = context_distributions
+        super().__init__(name=name, seed=seed)
+
+        if isinstance(context_distributions, list):
+            self.add_context_features(context_distributions)
+        elif isinstance(context_distributions, dict):
+            self.add_context_features(context_distributions.values())
+        elif type(context_distributions) in [str, DictConfig]:
+            cs = search_space_to_config_space(context_distributions)
+            self.add_context_features(cs.get_hyperparameters())
+        else:
+            raise ValueError(
+                f"Unknown type `{type(context_distributions)}` for `context_distributions`."
+            )
+
+        self.context_feature_names = [cf.name for cf in self.get_context_features()]
+        self.context_space = context_space
+
+    def add_context_features(self, context_features: list[ContextFeature]) -> None:
+        self.add_hyperparameters(context_features)
+
+    def get_context_features(self) -> list[ContextFeature]:
+        return list(self.values())
+
+    def sample_contexts(self, n_contexts: int) -> Contexts:
+        contexts = self._sample_contexts(size=n_contexts)
+
+        # Convert to dict
+        contexts = {i: C for i, C in enumerate(contexts)}
+
+        return contexts
+
+    def _sample_contexts(self, size: int = 1) -> list[Context]:
+        contexts = self.sample_configuration(size=size)
+        default_context = self.context_space.get_default_context()
+
+        if size == 1:
+            contexts = [contexts]
+        contexts = [dict(default_context | dict(C)) for C in contexts]
+
+        return contexts