Add private configs

src/helm/benchmark/config_registry.py

@@ -5,10 +5,10 @@
 HELM_REGISTERED: bool = False
 
 
-def register_helm_configurations():
+def register_helm_configurations(base_path: str = "prod_env"):
     global HELM_REGISTERED
     if not HELM_REGISTERED:
-        register_metadatas_if_not_already_registered()
-        register_tokenizers_if_not_already_registered()
-        register_deployments_if_not_already_registered()
+        register_metadatas_if_not_already_registered(base_path)
+        register_tokenizers_if_not_already_registered(base_path)
+        register_deployments_if_not_already_registered(base_path)
         HELM_REGISTERED = True

src/helm/benchmark/model_deployment_registry.py

@@ -177,9 +177,11 @@ def get_model_names_with_tokenizer(tokenizer_name: str) -> List[str]:
     return [deployment.model_name or deployment.name for deployment in deployments]
 
 
-def register_deployments_if_not_already_registered() -> None:
+def register_deployments_if_not_already_registered(base_path: str = "prod_env") -> None:
     global DEPLOYMENTS_REGISTERED
     if not DEPLOYMENTS_REGISTERED:
         path: str = resources.files(CONFIG_PACKAGE).joinpath(MODEL_DEPLOYMENTS_FILE)
+        private_path: str = os.path.join(base_path, MODEL_DEPLOYMENTS_FILE)
         maybe_register_model_deployments_from_base_path(path)
+        maybe_register_model_deployments_from_base_path(private_path)
         DEPLOYMENTS_REGISTERED = True

src/helm/benchmark/model_metadata_registry.py

@@ -200,11 +200,13 @@ def get_all_instruction_following_models() -> List[str]:
     return get_model_names_with_tag(INSTRUCTION_FOLLOWING_MODEL_TAG)
 
 
-def register_metadatas_if_not_already_registered() -> None:
+def register_metadatas_if_not_already_registered(base_path: str = "prod_env") -> None:
     global METADATAS_REGISTERED
     if not METADATAS_REGISTERED:
         path: str = resources.files(CONFIG_PACKAGE).joinpath(MODEL_METADATA_FILE)
+        private_path: str = os.path.join(base_path, MODEL_METADATA_FILE)
         maybe_register_model_metadata_from_base_path(path)
+        maybe_register_model_metadata_from_base_path(private_path)
         METADATAS_REGISTERED = True
 
 

src/helm/benchmark/presentation/summarize.py

@@ -1349,6 +1349,12 @@ def main():
         help="Number of instance ids we're using; only for annotating scenario spec instance ids file",
         default=1000,
     )
+    parser.add_argument(
+        "--local-path",
+        type=str,
+        help="If running locally, the path for `ServerService`.",
+        default="prod_env",
+    )
     parser.add_argument(
         "--allow-unknown-models",
         type=bool,
@@ -1378,7 +1384,7 @@ def main():
     else:
         raise ValueError("Exactly one of --release or --suite must be specified.")
 
-    register_helm_configurations()
+    register_helm_configurations(base_path=args.local_path)
 
     # Output JSON files summarizing the benchmark results which will be loaded in the web interface
     summarizer = Summarizer(

src/helm/benchmark/run.py

@@ -13,8 +13,6 @@
 from helm.common.object_spec import parse_object_spec, get_class_by_name
 from helm.proxy.services.remote_service import create_authentication, add_service_args
 
-from helm.benchmark.model_metadata_registry import register_model_metadata_from_path
-from helm.benchmark.model_deployment_registry import register_model_deployments_from_path
 from helm.benchmark.config_registry import register_helm_configurations
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark import vlm_run_specs  # noqa
@@ -246,18 +244,6 @@ def main():
         default=None,
         help="Full class name of the Runner class to use. If unset, uses the default Runner.",
     )
-    parser.add_argument(
-        "--model-metadata-paths",
-        nargs="+",
-        help="Experimental: Where to read model metadata from",
-        default=[],
-    )
-    parser.add_argument(
-        "--model-deployment-paths",
-        nargs="+",
-        help="Experimental: Where to read model deployments from",
-        default=[],
-    )
     add_run_args(parser)
     args = parser.parse_args()
     validate_args(args)
@@ -266,10 +252,6 @@ def main():
         register_huggingface_hub_model_from_flag_value(huggingface_model_name)
     for huggingface_model_path in args.enable_local_huggingface_models:
         register_huggingface_local_model_from_flag_value(huggingface_model_path)
-    for model_metadata_path in args.model_metadata_paths:
-        register_model_metadata_from_path(model_metadata_path)
-    for model_deployment_paths in args.model_deployment_paths:
-        register_model_deployments_from_path(model_deployment_paths)
 
     run_entries: List[RunEntry] = []
     if args.conf_paths:
@@ -284,7 +266,7 @@ def main():
     ensure_directory_exists(args.output_path)
     set_benchmark_output_path(args.output_path)
 
-    register_helm_configurations()
+    register_helm_configurations(base_path=args.local_path)
 
     run_specs = run_entries_to_run_specs(
         run_entries=run_entries,

src/helm/benchmark/tokenizer_config_registry.py

@@ -70,9 +70,11 @@ def get_tokenizer_config(name: str) -> Optional[TokenizerConfig]:
     return TOKENIZER_NAME_TO_CONFIG.get(name)
 
 
-def register_tokenizers_if_not_already_registered() -> None:
+def register_tokenizers_if_not_already_registered(base_path: str = "prod_env") -> None:
     global TOKENIZERS_REGISTERED
     if not TOKENIZERS_REGISTERED:
         path: str = resources.files(CONFIG_PACKAGE).joinpath(TOKENIZER_CONFIGS_FILE)
+        private_path: str = os.path.join(base_path, TOKENIZER_CONFIGS_FILE)
         maybe_register_tokenizer_configs_from_base_path(path)
+        maybe_register_tokenizer_configs_from_base_path(private_path)
         TOKENIZERS_REGISTERED = True

src/helm/config/model_deployments.yaml

@@ -2,7 +2,12 @@
 # Some models have several deployments, each with different parameters.
 
 # If you want to add a new deployment, you can technically do it here but we recommend
-# you to do it in private/model_deployments.yaml instead.
+# you to do it in prod_env/model_deployments.yaml instead.
+
+# Follow the template of this file to add a new deployment. You can copy paste this to get started:
+#    # This file defines all the model deployments that you do not want to be public.
+#    model_deployments: [] # Leave empty to disable private model deployments
+
 
 model_deployments:
 
@@ -506,8 +511,6 @@ model_deployments:
       class_name: "helm.benchmark.window_services.gpt2_window_service.GPT2WindowService"
       args: {}
 
-
-
   # HuggingFaceM4
   - name: HuggingFaceM4/idefics-9b
     model_name: HuggingFaceM4/idefics-9b

src/helm/config/model_metadata.yaml

@@ -2,7 +2,12 @@
 # The model names here should match the model names in model_deployments.yaml.
 
 # If you want to add a new model, you can technically do it here but we recommend
-# you to do it in private/model_metadata.yaml instead.
+# you to do it in prod_env/model_metadata.yaml instead.
+
+# Follow the template of this file to add a new model. You can copy paste this to get started:
+#    # This file contains the metadata for private models
+#    models: [] # Leave empty to disable private models
+
 
 models:
 

src/helm/config/tokenizer_configs.yaml

@@ -1,3 +1,13 @@
+# This file defines all the tokenizers that are supported by the Helm API.
+
+# If you want to add a new tokenizer, you can technically do it here but we recommend
+# you to do it in prod_env/tokenizer_configs.yaml instead.
+
+# Follow the template of this file to add a new tokenizer. You can copy paste this to get started:
+#    # This file contains the tokenizer configs for the private tokenizers
+#    tokenizer_configs: [] # Leave empty to disable private tokenizers
+
+
 tokenizer_configs:
 
   - name: simple/model1