Prepare Release 0.4.0 (#429)

.github/workflows/run_renate.yml

@@ -35,7 +35,7 @@ jobs:
     steps:
       - name: Configure AWS Credentials (if required)
         if: ${{ inputs.requires-aws-credentials == true }}
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: ${{ secrets.PROD_AWS_END_TO_END_TEST_ROLE_ARN }}
           role-session-name: integtestsession

.github/workflows/sagemaker_tests.yml

@@ -38,7 +38,7 @@ jobs:
       run: |
         python test/integration_tests/generate_requirements.py
     - name: Get Credentials
-      uses: aws-actions/configure-aws-credentials@v2
+      uses: aws-actions/configure-aws-credentials@v4
       with:
         role-to-assume: ${{ secrets.PROD_AWS_END_TO_END_TEST_ROLE_ARN }}
         role-session-name: integtestsession

README.rst

@@ -2,7 +2,7 @@
     :target: #
     :alt: PyPI - Status
 .. image:: https://img.shields.io/github/v/release/awslabs/Renate
-    :target: https://github.com/awslabs/Renate/releases/tag/v0.3.1
+    :target: https://github.com/awslabs/Renate/releases/tag/v0.4.0
     :alt: Latest Release
 .. image:: https://img.shields.io/pypi/dm/Renate
     :target: https://pypistats.org/packages/renate
@@ -116,14 +116,14 @@ If you did not find what you were looking for, open an `issue <https://github.co
 we will do our best to improve the documentation.
 
 
-.. [#] To create this plot, we simulated class-incremental learning with CIFAR-10.
-    The training data was divided into 5 partitions, and we trained sequentially on them.
+.. [#] To create this plot, we simulated domain-incremental learning with `CLEAR-100 <https://clear-benchmark.github.io/>`_.
+    The training data was divided by year, and we trained sequentially on them.
     Fine-tuning refers to the strategy to learn on the first partition from scratch, and
     train on each of the subsequent partitions for few epochs only.
-    We compare to Experience Replay with a memory size of 500.
-    For both methods we use the same number of epochs and choose the best checkpoint
+    We compare to Experience Replay with an infinite memory size.
+    For both methods we use the same amount of training time and choose the best checkpoint
     using a validation set.
     Results reported are on the test set.
 
-.. [#] The setup is the same as in the last experiment. However, this time we compare
+.. [#] In this experiment, we consider class-incremental learning on CIFAR-10. We compare
     Experience Replay against a version in which its hyperparameters were tuned.

benchmarks/experiment_configs/datasets/multitext.json

@@ -0,0 +1,4 @@
+{
+  "dataset_name": "MultiText",
+  "num_outputs": 33
+}

benchmarks/experiment_configs/datasets/yearbook.json

@@ -0,0 +1,7 @@
+{
+  "dataset_name": "yearbook",
+  "src_bucket": "my_bucket",
+  "src_object_name": "dataset/wildtime/yearbook.hdf5",
+  "num_inputs": 1024,
+  "num_outputs": 2
+}

benchmarks/experiment_configs/fine-tuning-clear10-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "clear10-vit-b16-10updates.json",
+  "model": "vit-b16.json",
+  "updater": "fine-tuning-clear-vit-b16.json",
+  "dataset": "clear10.json"
+}

benchmarks/experiment_configs/fine-tuning-clear100-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "clear100-vit-b16-11updates.json",
+  "model": "vit-b16.json",
+  "updater": "fine-tuning-clear-vit-b16.json",
+  "dataset": "clear100.json"
+}

benchmarks/experiment_configs/fine-tuning-domainnet-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "domainnet-vit-b16-6updates.json",
+  "model": "vit-b16.json",
+  "updater": "fine-tuning-domainnet.json",
+  "dataset": "domainnet.json"
+}

benchmarks/experiment_configs/fine-tuning-fmow-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "fmow-vit-b16-16updates.json",
+  "model": "vit-b16.json",
+  "updater": "fine-tuning-fmow.json",
+  "dataset": "fmow.json"
+}

benchmarks/experiment_configs/fine-tuning-multitext.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "multitext-4updates.json",
+  "model": "bert.json",
+  "updater": "fine-tuning-multitext.json",
+  "dataset": "multitext.json"
+}

benchmarks/experiment_configs/fine-tuning-yearbook-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "yearbook-vit-b16-17updates.json",
+  "model": "vit-b16.json",
+  "updater": "fine-tuning-yearbook.json",
+  "dataset": "yearbook.json"
+}

benchmarks/experiment_configs/fine-tuning-yearbook.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "yearbook-17updates.json",
+  "model": "resnet18-cifar.json",
+  "updater": "fine-tuning-yearbook.json",
+  "dataset": "yearbook.json"
+}

benchmarks/experiment_configs/joint-cifar100-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "cifar100-ci-10updates.json",
+  "model": "vit-b16.json",
+  "updater": "joint-cifar100.json",
+  "dataset": "cifar100.json"
+}

benchmarks/experiment_configs/joint-clear10-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "clear10-vit-b16-10updates.json",
+  "model": "vit-b16.json",
+  "updater": "joint-clear-vit-b16.json",
+  "dataset": "clear10.json"
+}

benchmarks/experiment_configs/joint-clear100-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "clear100-vit-b16-11updates.json",
+  "model": "vit-b16.json",
+  "updater": "joint-clear-vit-b16.json",
+  "dataset": "clear100.json"
+}

benchmarks/experiment_configs/joint-domainnet-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "domainnet-vit-b16-6updates.json",
+  "model": "vit-b16.json",
+  "updater": "joint-domainnet.json",
+  "dataset": "domainnet.json"
+}

benchmarks/experiment_configs/joint-fmow-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "fmow-vit-b16-16updates.json",
+  "model": "vit-b16.json",
+  "updater": "joint-fmow.json",
+  "dataset": "fmow.json"
+}

benchmarks/experiment_configs/joint-multitext.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "multitext-4updates.json",
+  "model": "bert.json",
+  "updater": "joint-multitext.json",
+  "dataset": "multitext.json"
+}

benchmarks/experiment_configs/joint-yearbook-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "yearbook-vit-b16-17updates.json",
+  "model": "vit-b16.json",
+  "updater": "joint-yearbook.json",
+  "dataset": "yearbook.json"
+}

benchmarks/experiment_configs/joint-yearbook.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "yearbook-17updates.json",
+  "model": "resnet18-cifar.json",
+  "updater": "joint-yearbook.json",
+  "dataset": "yearbook.json"
+}

benchmarks/experiment_configs/models/vit-b16.json

@@ -0,0 +1,3 @@
+{
+  "model_name": "VisionTransformerB16"
+}

benchmarks/experiment_configs/offline-er-arxiv.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "arxiv-16updates.json",
+  "model": "bert.json",
+  "updater": "offline-er-arxiv.json",
+  "dataset": "arxiv.json"
+}

benchmarks/experiment_configs/offline-er-clear10-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "clear10-vit-b16-10updates.json",
+  "model": "vit-b16.json",
+  "updater": "offline-er-clear10-vit-b16.json",
+  "dataset": "clear10.json"
+}

benchmarks/experiment_configs/offline-er-clear100-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "clear100-vit-b16-11updates.json",
+  "model": "vit-b16.json",
+  "updater": "offline-er-clear100-vit-b16.json",
+  "dataset": "clear100.json"
+}

benchmarks/experiment_configs/offline-er-domainnet-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "domainnet-vit-b16-6updates.json",
+  "model": "vit-b16.json",
+  "updater": "offline-er-domainnet.json",
+  "dataset": "domainnet.json"
+}

benchmarks/experiment_configs/offline-er-fmow-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "fmow-vit-b16-16updates.json",
+  "model": "vit-b16.json",
+  "updater": "offline-er-fmow.json",
+  "dataset": "fmow.json"
+}

benchmarks/experiment_configs/offline-er-fmow.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "fmow-16updates.json",
+  "model": "resnet18.json",
+  "updater": "offline-er-fmow.json",
+  "dataset": "fmow.json"
+}

benchmarks/experiment_configs/offline-er-huffpost.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "huffpost-7updates.json",
+  "model": "bert.json",
+  "updater": "offline-er-huffpost.json",
+  "dataset": "huffpost.json"
+}

benchmarks/experiment_configs/offline-er-multitext.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "multitext-4updates.json",
+  "model": "bert.json",
+  "updater": "offline-er-multitext.json",
+  "dataset": "multitext.json"
+}

benchmarks/experiment_configs/offline-er-yearbook-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "yearbook-vit-b16-17updates.json",
+  "model": "vit-b16.json",
+  "updater": "offline-er-yearbook.json",
+  "dataset": "yearbook.json"
+}

benchmarks/experiment_configs/offline-er-yearbook.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "yearbook-17updates.json",
+  "model": "resnet18-cifar.json",
+  "updater": "offline-er-yearbook.json",
+  "dataset": "yearbook.json"
+}

benchmarks/experiment_configs/scenarios/clear10-vit-b16-10updates.json

@@ -0,0 +1,14 @@
+{
+  "val_size": 0.1,
+  "scenario_name": "DataIncrementalScenario",
+  "num_tasks": 10,
+  "max_epochs": 10,
+  "optimizer": "SGD",
+  "learning_rate": 0.1,
+  "learning_rate_scheduler": "CosineAnnealingLR",
+  "learning_rate_scheduler_t_max": 10,
+  "learning_rate_scheduler_eta_min": 0.0001,
+  "learning_rate_scheduler_interval": "step",
+  "momentum": 0.0,
+  "weight_decay": 0.0
+}

benchmarks/experiment_configs/scenarios/clear100-vit-b16-11updates.json

@@ -0,0 +1,14 @@
+{
+  "val_size": 0.1,
+  "scenario_name": "DataIncrementalScenario",
+  "num_tasks": 11,
+  "max_epochs": 10,
+  "optimizer": "SGD",
+  "learning_rate": 0.1,
+  "learning_rate_scheduler": "CosineAnnealingLR",
+  "learning_rate_scheduler_t_max": 10,
+  "learning_rate_scheduler_eta_min": 0.0001,
+  "learning_rate_scheduler_interval": "step",
+  "momentum": 0.0,
+  "weight_decay": 0.0
+}

benchmarks/experiment_configs/scenarios/domainnet-vit-b16-6updates.json

@@ -0,0 +1,15 @@
+{
+  "val_size": 0.3,
+  "scenario_name": "DataIncrementalScenario",
+  "num_tasks": 6,
+  "max_epochs": 10,
+  "optimizer": "SGD",
+  "learning_rate": 0.1,
+  "learning_rate_scheduler": "CosineAnnealingLR",
+  "learning_rate_scheduler_t_max": 10,
+  "learning_rate_scheduler_eta_min": 0.0001,
+  "learning_rate_scheduler_interval": "step",
+  "momentum": 0.0,
+  "weight_decay": 0.0,
+  "data_ids": ["clipart", "infograph", "painting", "quickdraw", "real", "sketch"]
+}

benchmarks/experiment_configs/scenarios/fmow-vit-b16-16updates.json

@@ -0,0 +1,14 @@
+{
+  "val_size": 0.1,
+  "scenario_name": "DataIncrementalScenario",
+  "num_tasks": 16,
+  "max_epochs": 10,
+  "optimizer": "SGD",
+  "learning_rate": 0.1,
+  "learning_rate_scheduler": "CosineAnnealingLR",
+  "learning_rate_scheduler_t_max": 10,
+  "learning_rate_scheduler_eta_min": 0.0001,
+  "learning_rate_scheduler_interval": "step",
+  "momentum": 0.0,
+  "weight_decay": 0.0
+}

benchmarks/experiment_configs/scenarios/multitext-4updates.json

@@ -0,0 +1,10 @@
+{
+  "val_size": 0.1,
+  "scenario_name": "DataIncrementalScenario",
+  "num_tasks": 4,
+  "max_epochs": 2,
+  "optimizer": "AdamW",
+  "learning_rate": 0.00002,
+  "weight_decay": 0.01,
+  "data_ids": ["ag_news", "yelp_review_full", "dbpedia_14", "yahoo_answers_topics"]
+}

benchmarks/experiment_configs/scenarios/yearbook-17updates.json

@@ -0,0 +1,33 @@
+{
+  "val_size": 0.1,
+  "scenario_name": "DataIncrementalScenario",
+  "groupings": [
+    [0, 1, 2, 3, 4],
+    [5, 6, 7, 8, 9],
+    [10, 11, 12, 13, 14],
+    [15, 16, 17, 18, 19],
+    [20, 21, 22, 23, 24],
+    [25, 26, 27, 28, 29],
+    [30, 31, 32, 33, 34],
+    [35, 36, 37, 38, 39],
+    [40, 41, 42, 43, 44],
+    [45, 46, 47, 48, 49],
+    [50, 51, 52, 53, 54],
+    [55, 56, 57, 58, 59],
+    [60, 61, 62, 63, 64],
+    [65, 66, 67, 68, 69],
+    [70, 71, 72, 73, 74],
+    [75, 76, 77, 78, 79],
+    [80, 81, 82, 83]
+  ],
+  "num_tasks": 17,
+  "max_epochs": 50,
+  "optimizer": "SGD",
+  "learning_rate": 0.1,
+  "learning_rate_scheduler": "CosineAnnealingLR",
+  "learning_rate_scheduler_t_max": 50,
+  "learning_rate_scheduler_eta_min": 0.0001,
+  "learning_rate_scheduler_interval": "step",
+  "momentum": 0.0,
+  "weight_decay": 0.0
+}

benchmarks/experiment_configs/scenarios/yearbook-vit-b16-17updates.json

@@ -0,0 +1,28 @@
+{
+  "val_size": 0.1,
+  "scenario_name": "DataIncrementalScenario",
+  "groupings": [
+    [0, 1, 2, 3, 4],
+    [5, 6, 7, 8, 9],
+    [10, 11, 12, 13, 14],
+    [15, 16, 17, 18, 19],
+    [20, 21, 22, 23, 24],
+    [25, 26, 27, 28, 29],
+    [30, 31, 32, 33, 34],
+    [35, 36, 37, 38, 39],
+    [40, 41, 42, 43, 44],
+    [45, 46, 47, 48, 49],
+    [50, 51, 52, 53, 54],
+    [55, 56, 57, 58, 59],
+    [60, 61, 62, 63, 64],
+    [65, 66, 67, 68, 69],
+    [70, 71, 72, 73, 74],
+    [75, 76, 77, 78, 79],
+    [80, 81, 82, 83]
+  ],
+  "num_tasks": 17,
+  "max_epochs": 4,
+  "optimizer": "AdamW",
+  "learning_rate": 0.00002,
+  "weight_decay": 0.01
+}

benchmarks/experiment_configs/updaters/fine-tuning-clear-vit-b16.json

@@ -0,0 +1,4 @@
+{
+  "updater": "FineTuning",
+  "batch_size": 64
+}

benchmarks/experiment_configs/updaters/fine-tuning-multitext.json

@@ -0,0 +1,4 @@
+{
+    "updater": "FineTuning",
+    "batch_size": 64
+}

benchmarks/experiment_configs/updaters/fine-tuning-yearbook.json

@@ -0,0 +1,4 @@
+{
+    "updater": "FineTuning",
+    "batch_size": 64
+}

benchmarks/experiment_configs/updaters/joint-clear-vit-b16.json

@@ -0,0 +1,4 @@
+{
+  "updater": "Joint",
+  "batch_size": 64
+}

benchmarks/experiment_configs/updaters/joint-multitext.json

@@ -0,0 +1,4 @@
+{
+    "updater": "Joint",
+    "batch_size": 64
+}