Benchmarks revamp

benchmarks/README.md

@@ -1,17 +1,43 @@
 # Benchmarks
 
 ## Installation
+
+The recommended way to run the benchmarks is to create a dedicated environment for river and its contenders.
+
+An easy way to achieve that is through [Anaconda](https://docs.conda.io/projects/miniconda/en/latest/). Here is an example of creating an environment for the benchmarks:
+
+```sh
+conda create --name river-benchmark python=3.10
+```
+
+The next step is to clone river if you have not done that already:
+
+```sh
+git clone https://github.com/online-ml/river
+cd river
+```
+
+From the river folder you can run the following command to install the needed dependencies:
+
 ```sh
 pip install ".[benchmarks]"
 ```
 
 ## Usage
-The `run.py` executes the benchmarks and creates the necessary .csv files for rendering the plots.
+
+The `run.py` script executes the benchmarks and creates the necessary .csv files for rendering the plots.
+
 ```sh
 cd benchmarks
 python run.py
 ```
+
 The `render.py` renders the plots from the .csv files and moves them to the `docs/benchmarks` folder.
+
 ```sh
 python render.py
 ```
+
+## Notes: VolpalWabbit
+
+Installing Volpal Wabbit (VW) can be tricky sometimes. That is especially true when using apple silicon. If cannot make the pip install guidelines from VW work a workaround is the following. When using anaconda, you can install the recommended dependencies utilized for building VW with conda. You can get more info [here](https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Building#conda) about such dependencies. After that, `pip install volpalwabbit` should work just fine.

benchmarks/config.py

@@ -1,16 +1,6 @@
-from model_zoo.torch import (
-    TorchLinearRegression,
-    TorchLogisticRegression,
-    TorchLSTMClassifier,
-    TorchLSTMRegressor,
-    TorchMLPClassifier,
-    TorchMLPRegressor,
-)
-from model_zoo.vw import VW2RiverClassifier
-from river_torch.classification import Classifier as TorchClassifier
-from river_torch.classification import RollingClassifier as TorchRollingClassifier
-from river_torch.regression import Regressor as TorchRegressor
-from river_torch.regression import RollingRegressor as TorchRollingRegressor
+from __future__ import annotations
+
+from model_adapters.vw import VW2RiverClassifier
 from sklearn.linear_model import SGDClassifier
 
 from river import (
@@ -39,7 +29,6 @@
     evaluate.MultiClassClassificationTrack(),
     evaluate.RegressionTrack(),
 ]
-import river
 
 MODELS = {
     "Binary classification": {
@@ -53,7 +42,7 @@
             preprocessing.StandardScaler()
             | compat.SKL2RiverClassifier(
                 SGDClassifier(
-                    loss="log", learning_rate="constant", eta0=LEARNING_RATE, penalty="none"
+                    loss="log_loss", learning_rate="constant", eta0=LEARNING_RATE, penalty=None
                 ),
                 classes=[False, True],
             )
@@ -105,38 +94,6 @@
                 preprocessing.StandardScaler() | neighbors.KNNClassifier(),
             ]
         ),
-        "Torch Logistic Regression": (
-            preprocessing.StandardScaler()
-            | TorchClassifier(
-                module=TorchLogisticRegression,
-                loss_fn="binary_cross_entropy",
-                optimizer_fn="adam",
-                is_class_incremental=True,
-                lr=LEARNING_RATE,
-            )
-        ),
-        "Torch MLP": (
-            preprocessing.StandardScaler()
-            | TorchClassifier(
-                module=TorchMLPClassifier,
-                loss_fn="binary_cross_entropy",
-                optimizer_fn="adam",
-                is_class_incremental=True,
-                lr=LEARNING_RATE,
-            )
-        ),
-        "Torch LSTM": (
-            preprocessing.StandardScaler()
-            | TorchRollingClassifier(
-                module=TorchLSTMClassifier,
-                loss_fn="binary_cross_entropy",
-                optimizer_fn="adam",
-                is_class_incremental=True,
-                lr=LEARNING_RATE,
-                window_size=20,
-                append_predict=False,
-            )
-        ),
         # Baseline
         "[baseline] Last Class": dummy.NoChangeClassifier(),
     },
@@ -172,24 +129,6 @@
                 rules.AMRules(),
             ],
         ),
-        "Torch Linear Regression": (
-            preprocessing.StandardScaler()
-            | TorchRegressor(
-                module=TorchLinearRegression,
-                loss_fn="mse",
-                optimizer_fn="adam",
-                learning_rate=LEARNING_RATE,
-            )
-        ),
-        "Torch MLP": (
-            preprocessing.StandardScaler()
-            | TorchRegressor(
-                module=TorchMLPRegressor,
-                loss_fn="mse",
-                optimizer_fn="adam",
-                learning_rate=LEARNING_RATE,
-            )
-        ),
         "River MLP": preprocessing.StandardScaler()
         | neural_net.MLPRegressor(
             hidden_dims=(5,),
@@ -201,17 +140,6 @@
             optimizer=optim.SGD(1e-3),
             seed=42,
         ),
-        "Torch LSTM": (
-            preprocessing.StandardScaler()
-            | TorchRollingRegressor(
-                module=TorchLSTMRegressor,
-                loss_fn="mse",
-                optimizer_fn="adam",
-                learning_rate=LEARNING_RATE,
-                window_size=20,
-                append_predict=False,
-            )
-        ),
         # Baseline
         "[baseline] Mean predictor": dummy.StatisticRegressor(stats.Mean()),
     },