Merge pull request #438 from awslabs/dev

Release 0.4.0

.github/workflows/codeql.yml

@@ -23,7 +23,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/integration_tests.yml

@@ -21,7 +21,7 @@ jobs:
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/setup-python@v4
       with:
         python-version: 3.x

.github/workflows/pypi_publish.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Set up Python 3.10

.github/workflows/run_renate.yml

@@ -35,12 +35,12 @@ jobs:
     steps:
       - name: Configure AWS Credentials (if required)
         if: ${{ inputs.requires-aws-credentials == true }}
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: ${{ secrets.PROD_AWS_END_TO_END_TEST_ROLE_ARN }}
           role-session-name: integtestsession
           aws-region: ${{ env.AWS_DEFAULT_REGION }}
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python 3.9
         uses: actions/setup-python@v4
         with:

.github/workflows/run_unit_tests.yml

@@ -15,7 +15,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Switch to Current Branch
         run: git checkout ${{ env.BRANCH }}
@@ -42,7 +42,7 @@ jobs:
       - name: Black
         uses: psf/black@stable
         with:
-          version: "~= 22.12.0"
+          options: "--check --verbose"
 
       - name: Run unit tests
         run: PYTHONPATH=test python -m pytest test/
@@ -60,7 +60,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: test
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - uses: actions/download-artifact@v3
         id: download

.github/workflows/sagemaker_tests.yml

@@ -0,0 +1,69 @@
+name: SageMaker Tests
+
+# All tests in this file:
+# 1. Run when launched manually
+# 2. Require AWS credentials and launch training jobs
+
+on:
+  workflow_dispatch:
+#  pull_request: # UNCOMMENT FOR TESTING
+#    branches:   # UNCOMMENT FOR TESTING
+#      - main    # UNCOMMENT FOR TESTING
+#      - dev     # UNCOMMENT FOR TESTING
+
+env:
+  AWS_DEFAULT_REGION: us-west-2
+  AWS_ROLE: ${{ secrets.PROD_AWS_END_TO_END_TEST_ROLE_ARN }}
+
+permissions:
+  id-token: write # This is required for requesting the JWT
+  contents: read  # This is required for actions/checkout
+
+jobs:
+  launch-sagemaker-jobs:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+        cache: 'pip'
+    - name: Install Renate
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install -e '.[dev]'
+    - name: Install toml library
+      run: pip install toml
+    - name: Write requirements.txt for SageMaker training jobs
+      run: |
+        python test/integration_tests/generate_requirements.py
+    - name: Get Credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        role-to-assume: ${{ secrets.PROD_AWS_END_TO_END_TEST_ROLE_ARN }}
+        role-session-name: integtestsession
+        aws-region: ${{ env.AWS_DEFAULT_REGION }}
+    - name: Launch SageMaker Jobs
+      run: |
+        import os
+        import subprocess
+        
+        target_directory = 'test/integration_tests/configs/suites/main'
+        files = [f for f in os.listdir(target_directory) if os.path.isfile(os.path.join(target_directory, f))]
+        
+        for file in files:
+          process = subprocess.Popen(
+            [
+              "python",
+              "test/integration_tests/run_test.py",
+              "--test-file",
+              file,
+              "--seed",
+              "0",
+              "--requirements-file",
+              "test/integration_tests/requirements.txt",
+            ]
+          )
+        process.wait()
+
+      shell: python

.github/workflows/test_docs.yml

@@ -13,7 +13,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Switch to Current Branch
         run: git checkout ${{ env.BRANCH }}

README.rst

@@ -2,7 +2,7 @@
     :target: #
     :alt: PyPI - Status
 .. image:: https://img.shields.io/github/v/release/awslabs/Renate
-    :target: https://github.com/awslabs/Renate/releases/tag/v0.3.1
+    :target: https://github.com/awslabs/Renate/releases/tag/v0.4.0
     :alt: Latest Release
 .. image:: https://img.shields.io/pypi/dm/Renate
     :target: https://pypistats.org/packages/renate
@@ -116,14 +116,14 @@ If you did not find what you were looking for, open an `issue <https://github.co
 we will do our best to improve the documentation.
 
 
-.. [#] To create this plot, we simulated class-incremental learning with CIFAR-10.
-    The training data was divided into 5 partitions, and we trained sequentially on them.
+.. [#] To create this plot, we simulated domain-incremental learning with `CLEAR-100 <https://clear-benchmark.github.io/>`_.
+    The training data was divided by year, and we trained sequentially on them.
     Fine-tuning refers to the strategy to learn on the first partition from scratch, and
     train on each of the subsequent partitions for few epochs only.
-    We compare to Experience Replay with a memory size of 500.
-    For both methods we use the same number of epochs and choose the best checkpoint
+    We compare to Experience Replay with an infinite memory size.
+    For both methods we use the same amount of training time and choose the best checkpoint
     using a validation set.
     Results reported are on the test set.
 
-.. [#] The setup is the same as in the last experiment. However, this time we compare
+.. [#] In this experiment, we consider class-incremental learning on CIFAR-10. We compare
     Experience Replay against a version in which its hyperparameters were tuned.

benchmarks/README.rst

@@ -0,0 +1,35 @@
+Running Standard Benchmarks
+******************************************************************************
+
+This folder contains a couple of files to run experiments with Renate.
+Add new experimentation configuration files to add new experiments.
+Edit the ``requirements.txt`` to add additional requirements for an experiment (SageMaker only).
+
+Instructions
+============
+1. Clone the repository
+2. ``cd Renate`` and install Renate
+3. Run a benchmark via
+
+    .. code-block:: bash
+
+        python benchmarks/run_benchmark.py --benchmark-file fine-tuning-clear10.json \
+        --backend sagemaker  --budget-factor 1 --job-name clear10-finetuning-1 --num-repetitions 1
+
+    This is an example command to run an experiment with ClEAR10 on SageMaker using a Fine-Tuning
+    updater.
+
+    Quick explanation of the arguments:
+
+    - ``benchmark-file``: Any filename of files in ``experiment_configs``. This file specifies all
+      properties of the experiment, i.e., dataset, scenario, updater, and hyperparameters settings.
+      Modify or add more to run own experiments.
+    - ``backend``: Run the experiment on SageMaker (``sagemaker``) or locally (``local``).
+    - ``budget-factor``: Each update run will make ``budget_factor * max_epochs`` many passes over
+      the new data during training. ``max_epochs`` is typically defines as part of the scenario
+      .json file. Default: ``1``.
+    - ``job-name``: Defines the name of the output folder and the name of the SageMaker training
+      job.
+    - ``num-repetitions``: The number of times the experiment will be repeated. Only the seed
+      differs between repetitions.
+    - ``max-time``: The wall clock time spent per update. Default: 5 days.

benchmarks/experiment_configs/datasets/arxiv.json

@@ -0,0 +1,6 @@
+{
+  "dataset_name": "arxiv",
+  "src_bucket": "my_bucket",
+  "src_object_name": "dataset/wildtime/arxiv.hdf5",
+  "num_outputs": 172
+}

benchmarks/experiment_configs/datasets/clear10.json

@@ -0,0 +1,5 @@
+{
+  "dataset_name": "CLEAR10",
+  "num_inputs": 50176,
+  "num_outputs": 11
+}

benchmarks/experiment_configs/datasets/clear100.json

@@ -0,0 +1,5 @@
+{
+  "dataset_name": "CLEAR100",
+  "num_inputs": 50176,
+  "num_outputs": 100
+}

benchmarks/experiment_configs/datasets/domainnet.json

@@ -0,0 +1,4 @@
+{
+  "dataset_name": "DomainNet",
+  "num_outputs": 345
+}

benchmarks/experiment_configs/datasets/fmow.json

@@ -0,0 +1,7 @@
+{
+  "dataset_name": "fmow",
+  "src_bucket": "my_bucket",
+  "src_object_name": "dataset/wildtime/fmow.hdf5",
+  "num_inputs": 150528,
+  "num_outputs": 62
+}

benchmarks/experiment_configs/datasets/huffpost.json

@@ -0,0 +1,6 @@
+{
+  "dataset_name": "huffpost",
+  "src_bucket": "my_bucket",
+  "src_object_name": "dataset/wildtime/huffpost.hdf5",
+  "num_outputs": 11
+}

benchmarks/experiment_configs/datasets/multitext.json

@@ -0,0 +1,4 @@
+{
+  "dataset_name": "MultiText",
+  "num_outputs": 33
+}

benchmarks/experiment_configs/datasets/yearbook.json

@@ -0,0 +1,7 @@
+{
+  "dataset_name": "yearbook",
+  "src_bucket": "my_bucket",
+  "src_object_name": "dataset/wildtime/yearbook.hdf5",
+  "num_inputs": 1024,
+  "num_outputs": 2
+}

benchmarks/experiment_configs/fine-tuning-arxiv.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "arxiv-16updates.json",
+  "model": "bert.json",
+  "updater": "fine-tuning-arxiv.json",
+  "dataset": "arxiv.json"
+}

benchmarks/experiment_configs/fine-tuning-clear10-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "clear10-vit-b16-10updates.json",
+  "model": "vit-b16.json",
+  "updater": "fine-tuning-clear-vit-b16.json",
+  "dataset": "clear10.json"
+}

benchmarks/experiment_configs/fine-tuning-clear10.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "clear10-10updates.json",
+  "model": "resnet18.json",
+  "updater": "fine-tuning-clear.json",
+  "dataset": "clear10.json"
+}

benchmarks/experiment_configs/fine-tuning-clear100-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "clear100-vit-b16-11updates.json",
+  "model": "vit-b16.json",
+  "updater": "fine-tuning-clear-vit-b16.json",
+  "dataset": "clear100.json"
+}

benchmarks/experiment_configs/fine-tuning-clear100.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "clear100-11updates.json",
+  "model": "resnet18.json",
+  "updater": "fine-tuning-clear.json",
+  "dataset": "clear100.json"
+}

benchmarks/experiment_configs/fine-tuning-domainnet-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "domainnet-vit-b16-6updates.json",
+  "model": "vit-b16.json",
+  "updater": "fine-tuning-domainnet.json",
+  "dataset": "domainnet.json"
+}

benchmarks/experiment_configs/fine-tuning-domainnet.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "domainnet-6updates.json",
+  "model": "resnet18.json",
+  "updater": "fine-tuning-domainnet.json",
+  "dataset": "domainnet.json"
+}

benchmarks/experiment_configs/fine-tuning-fmow-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "fmow-vit-b16-16updates.json",
+  "model": "vit-b16.json",
+  "updater": "fine-tuning-fmow.json",
+  "dataset": "fmow.json"
+}

benchmarks/experiment_configs/fine-tuning-fmow.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "fmow-16updates.json",
+  "model": "resnet18.json",
+  "updater": "fine-tuning-fmow.json",
+  "dataset": "fmow.json"
+}

benchmarks/experiment_configs/fine-tuning-huffpost.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "huffpost-7updates.json",
+  "model": "bert.json",
+  "updater": "fine-tuning-huffpost.json",
+  "dataset": "huffpost.json"
+}

benchmarks/experiment_configs/fine-tuning-multitext.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "multitext-4updates.json",
+  "model": "bert.json",
+  "updater": "fine-tuning-multitext.json",
+  "dataset": "multitext.json"
+}

benchmarks/experiment_configs/fine-tuning-yearbook-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "yearbook-vit-b16-17updates.json",
+  "model": "vit-b16.json",
+  "updater": "fine-tuning-yearbook.json",
+  "dataset": "yearbook.json"
+}

benchmarks/experiment_configs/fine-tuning-yearbook.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "yearbook-17updates.json",
+  "model": "resnet18-cifar.json",
+  "updater": "fine-tuning-yearbook.json",
+  "dataset": "yearbook.json"
+}

benchmarks/experiment_configs/joint-arxiv.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "arxiv-16updates.json",
+  "model": "bert.json",
+  "updater": "joint-arxiv.json",
+  "dataset": "arxiv.json"
+}

benchmarks/experiment_configs/joint-cifar100-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "cifar100-ci-10updates.json",
+  "model": "vit-b16.json",
+  "updater": "joint-cifar100.json",
+  "dataset": "cifar100.json"
+}

benchmarks/experiment_configs/joint-clear10-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "clear10-vit-b16-10updates.json",
+  "model": "vit-b16.json",
+  "updater": "joint-clear-vit-b16.json",
+  "dataset": "clear10.json"
+}

benchmarks/experiment_configs/joint-clear10.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "clear10-10updates.json",
+  "model": "resnet18.json",
+  "updater": "joint-clear.json",
+  "dataset": "clear10.json"
+}

benchmarks/experiment_configs/joint-clear100-vitb16.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "clear100-vit-b16-11updates.json",
+  "model": "vit-b16.json",
+  "updater": "joint-clear-vit-b16.json",
+  "dataset": "clear100.json"
+}

benchmarks/experiment_configs/joint-clear100.json

@@ -0,0 +1,6 @@
+{
+  "scenario": "clear100-11updates.json",
+  "model": "resnet18.json",
+  "updater": "joint-clear.json",
+  "dataset": "clear100.json"
+}