Rename --run-specs to --run-entries (#2404)

.github/workflows/test.yml

@@ -27,7 +27,7 @@ jobs:
       - run: python3 -m pip install --upgrade build
       - run: python3 -m build
       - run: python3 -m pip install dist/crfm_helm-*.whl
-      - run: helm-run --run-specs simple1:model=simple/model1 --max-eval-instances 10 --suite test
+      - run: helm-run --run-entries simple1:model=simple/model1 --max-eval-instances 10 --suite test
       - run: helm-summarize --suite test
       - run: helm-server --help
 

.github/workflows/update-requirements.yml

@@ -61,7 +61,7 @@ jobs:
       - name: Run tests
         run: pytest
       - name: Run helm-run
-        run: helm-run --suite test --run-specs simple1:model=simple/model1 --max-eval-instances 10 --exit-on-error
+        run: helm-run --suite test --run-entries simple1:model=simple/model1 --max-eval-instances 10 --exit-on-error
       - name: Create pull request
         uses: peter-evans/create-pull-request@v6
         with:

docs/adding_new_models.md

@@ -19,7 +19,7 @@ In the first case, you should create the files `model_deployments.yaml`, `model_
 In the second case, if you want to add a model to HELM, you can directly do it in `src/helm/config`. You can then open a Pull Request on Github to share the model. When you do, make sure to:
 * Include any link justifying the metadata used in `ModelMetadata` such as the release data, number of parameters, capabilities and so on (you should not infer anything).
 * Check that you are respecting the format used in those files (`ModelMetadata` should be named as `<CREATOR-ORGANIZATION>/<MODEL-NAME>` and the `ModelDeployment` should be named as `<HOST-ORGANIZATION>/<MODEL-NAME>`, for example `ModelMetadata`: `openai/gpt2` and `ModelDeployment`: `huggingface/gpt2`). Add the appropriate comments and so on.
-* Run `helm-run --run-specs "mmlu:subject=anatomy,model_deployment=<YOUR-DEPLOYMENT>" --suite v1 --max-eval-instances 10` and make sure that everything works. Include the logs from the terminal in your PR.
+* Run `helm-run --run-entries "mmlu:subject=anatomy,model_deployment=<YOUR-DEPLOYMENT>" --suite v1 --max-eval-instances 10` and make sure that everything works. Include the logs from the terminal in your PR.
 * Not create unnecessary objects (`Client` `TokenizerCOnfig`, `WindowService`) and if you have to create one of these objects, document in your PR why you had to. Make them general enough so that they could be re-used by other models (especially the `Client`).
 
 
@@ -71,6 +71,6 @@ model_deployments:
 
 We won't be adding any `TokenizerConfig` here as we are reusing `simple/model1`. This shows a good practice when adding a new model, always check if the correct tokenizer does not already exists.
 
-You should now be able to run `helm-run --run-specs "mmlu:subject=anatomy,model_deployment=simple/tutorial" --suite v1 --max-eval-instances 10` without any error.
+You should now be able to run `helm-run --run-entries "mmlu:subject=anatomy,model_deployment=simple/tutorial" --suite v1 --max-eval-instances 10` without any error.
 
 

docs/huggingface_models.md

@@ -17,14 +17,14 @@ Examples:
 ```bash
 # Run boolq on stanford-crfm/BioMedLM at the default main revision
 helm-run \
-    --run-specs boolq:model=stanford-crfm/BioMedLM \
+    --run-entries boolq:model=stanford-crfm/BioMedLM \
     --enable-huggingface-models stanford-crfm/BioMedLM \
     --suite v1 \
     --max-eval-instances 10
 
 # Run boolq on stanford-crfm/BioMedLM at revision main
 helm-run \
-    --run-specs boolq:model=stanford-crfm/BioMedLM@main \
+    --run-entries boolq:model=stanford-crfm/BioMedLM@main \
     --enable-huggingface-models stanford-crfm/BioMedLM@main \
     --suite v1 \
     --max-eval-instances 10

src/helm/benchmark/run.py

@@ -172,13 +172,6 @@ def add_run_args(parser: argparse.ArgumentParser):
         help="Name of the suite this run belongs to (default is today's date).",
         required=True,
     )
-    parser.add_argument(
-        "--local",
-        action="store_true",
-        help="DEPRECATED: Does nothing. Do not use. Previously enabled local mode. "
-        "Now does nothing and will be removed in the next released version. "
-        "Local mode is enabled by default, and only disabled if the --server_url flag is set.",
-    )
     parser.add_argument(
         "--local-path",
         type=str,
@@ -244,7 +237,14 @@ def main():
         help="Run RunSpecs with priority less than or equal to this number. "
         "If a value for --priority is not specified, run on everything",
     )
-    parser.add_argument("-r", "--run-specs", nargs="*", help="Specifies what to run", default=[])
+    parser.add_argument(
+        "--run-specs",
+        nargs="*",
+        help="DEPRECATED: Use --run-entries instead. Will be removed in a future release. "
+        "Specifies run entries to run.",
+        default=[],
+    )
+    parser.add_argument("-r", "--run-entries", nargs="*", help="Specifies run entries to run", default=[])
     parser.add_argument(
         "--enable-huggingface-models",
         nargs="+",
@@ -285,6 +285,11 @@ def main():
     run_entries: List[RunEntry] = []
     if args.conf_paths:
         run_entries.extend(read_run_entries(args.conf_paths).entries)
+    if args.run_entries:
+        run_entries.extend(
+            [RunEntry(description=description, priority=1, groups=None) for description in args.run_entries]
+        )
+    # TODO: Remove this eventually.
     if args.run_specs:
         run_entries.extend(
             [RunEntry(description=description, priority=1, groups=None) for description in args.run_specs]
@@ -332,11 +337,10 @@ def main():
         disable_cache=args.disable_cache,
     )
 
-    if args.local:
+    if args.run_specs:
         hlog(
-            "WARNING: The --local flag is deprecated. It now does nothing and will be removed in "
-            "the next released version. Local mode is enabled by default, and only disabled if the "
-            "--server_url flag is set. Please remove --local from your command."
+            "WARNING: The --run-specs flag is deprecated and will be removed in a future release. "
+            "Use --run-entries instead."
         )
 
     hlog("Done.")

src/helm/benchmark/slurm_runner.py

@@ -325,9 +325,9 @@ def main():
     This entry point should only be used by SlurmRunner. Users should use `helm-run` instead.
     SlurmRunner has to use this entry point instead of helm-run because there is no way to
     specify the worker Slurm job parameters through `helm-run`. In particular, there is no way
-    to run a specific `RunSpec` using the `--run-specs` parameter of `helm-run`, because the
-    `run-specs` argument is a `RunSpec` description (not a `RunSpec`), and there is no way to
-    convert a `RunSpec` into a `RunSpec` description."""
+    to run a specific `RunSpec` using the `--run-entries` parameter of `helm-run`, because the
+    `run-entries` argument contains `RunEntry` description (not `RunSpec`s), and there is no way to
+    convert a `RunSpec` into a `RunEntry` description."""
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--slurm-runner-spec-path",