diff --git a/src/helm/benchmark/config_registry.py b/src/helm/benchmark/config_registry.py index 0fab062949..498c63b364 100644 --- a/src/helm/benchmark/config_registry.py +++ b/src/helm/benchmark/config_registry.py @@ -5,10 +5,10 @@ HELM_REGISTERED: bool = False -def register_helm_configurations(): +def register_helm_configurations(base_path: str = "prod_env"): global HELM_REGISTERED if not HELM_REGISTERED: - register_metadatas_if_not_already_registered() - register_tokenizers_if_not_already_registered() - register_deployments_if_not_already_registered() + register_metadatas_if_not_already_registered(base_path) + register_tokenizers_if_not_already_registered(base_path) + register_deployments_if_not_already_registered(base_path) HELM_REGISTERED = True diff --git a/src/helm/benchmark/model_deployment_registry.py b/src/helm/benchmark/model_deployment_registry.py index c3f9d36147..c0d9b6b428 100644 --- a/src/helm/benchmark/model_deployment_registry.py +++ b/src/helm/benchmark/model_deployment_registry.py @@ -177,9 +177,11 @@ def get_model_names_with_tokenizer(tokenizer_name: str) -> List[str]: return [deployment.model_name or deployment.name for deployment in deployments] -def register_deployments_if_not_already_registered() -> None: +def register_deployments_if_not_already_registered(base_path: str = "prod_env") -> None: global DEPLOYMENTS_REGISTERED if not DEPLOYMENTS_REGISTERED: path: str = resources.files(CONFIG_PACKAGE).joinpath(MODEL_DEPLOYMENTS_FILE) + private_path: str = os.path.join(base_path, MODEL_DEPLOYMENTS_FILE) maybe_register_model_deployments_from_base_path(path) + maybe_register_model_deployments_from_base_path(private_path) DEPLOYMENTS_REGISTERED = True diff --git a/src/helm/benchmark/model_metadata_registry.py b/src/helm/benchmark/model_metadata_registry.py index 6d114cf7ca..3e6e8713b7 100644 --- a/src/helm/benchmark/model_metadata_registry.py +++ b/src/helm/benchmark/model_metadata_registry.py @@ -200,11 +200,13 @@ def get_all_instruction_following_models() -> List[str]: return get_model_names_with_tag(INSTRUCTION_FOLLOWING_MODEL_TAG) -def register_metadatas_if_not_already_registered() -> None: +def register_metadatas_if_not_already_registered(base_path: str = "prod_env") -> None: global METADATAS_REGISTERED if not METADATAS_REGISTERED: path: str = resources.files(CONFIG_PACKAGE).joinpath(MODEL_METADATA_FILE) + private_path: str = os.path.join(base_path, MODEL_METADATA_FILE) maybe_register_model_metadata_from_base_path(path) + maybe_register_model_metadata_from_base_path(private_path) METADATAS_REGISTERED = True diff --git a/src/helm/benchmark/presentation/summarize.py b/src/helm/benchmark/presentation/summarize.py index 0ba7af28e4..63ef3d7af1 100644 --- a/src/helm/benchmark/presentation/summarize.py +++ b/src/helm/benchmark/presentation/summarize.py @@ -1349,6 +1349,12 @@ def main(): help="Number of instance ids we're using; only for annotating scenario spec instance ids file", default=1000, ) + parser.add_argument( + "--local-path", + type=str, + help="If running locally, the path for `ServerService`.", + default="prod_env", + ) parser.add_argument( "--allow-unknown-models", type=bool, @@ -1378,7 +1384,7 @@ def main(): else: raise ValueError("Exactly one of --release or --suite must be specified.") - register_helm_configurations() + register_helm_configurations(base_path=args.local_path) # Output JSON files summarizing the benchmark results which will be loaded in the web interface summarizer = Summarizer( diff --git a/src/helm/benchmark/run.py b/src/helm/benchmark/run.py index 84f5ae8446..b1541f55b0 100644 --- a/src/helm/benchmark/run.py +++ b/src/helm/benchmark/run.py @@ -13,8 +13,6 @@ from helm.common.object_spec import parse_object_spec, get_class_by_name from helm.proxy.services.remote_service import create_authentication, add_service_args -from helm.benchmark.model_metadata_registry import register_model_metadata_from_path -from helm.benchmark.model_deployment_registry import register_model_deployments_from_path from helm.benchmark.config_registry import register_helm_configurations from helm.benchmark.adaptation.adapter_spec import AdapterSpec from helm.benchmark import vlm_run_specs # noqa @@ -246,18 +244,6 @@ def main(): default=None, help="Full class name of the Runner class to use. If unset, uses the default Runner.", ) - parser.add_argument( - "--model-metadata-paths", - nargs="+", - help="Experimental: Where to read model metadata from", - default=[], - ) - parser.add_argument( - "--model-deployment-paths", - nargs="+", - help="Experimental: Where to read model deployments from", - default=[], - ) add_run_args(parser) args = parser.parse_args() validate_args(args) @@ -266,10 +252,6 @@ def main(): register_huggingface_hub_model_from_flag_value(huggingface_model_name) for huggingface_model_path in args.enable_local_huggingface_models: register_huggingface_local_model_from_flag_value(huggingface_model_path) - for model_metadata_path in args.model_metadata_paths: - register_model_metadata_from_path(model_metadata_path) - for model_deployment_paths in args.model_deployment_paths: - register_model_deployments_from_path(model_deployment_paths) run_entries: List[RunEntry] = [] if args.conf_paths: @@ -284,7 +266,7 @@ def main(): ensure_directory_exists(args.output_path) set_benchmark_output_path(args.output_path) - register_helm_configurations() + register_helm_configurations(base_path=args.local_path) run_specs = run_entries_to_run_specs( run_entries=run_entries, diff --git a/src/helm/benchmark/tokenizer_config_registry.py b/src/helm/benchmark/tokenizer_config_registry.py index 732cd38bd1..aa202621fb 100644 --- a/src/helm/benchmark/tokenizer_config_registry.py +++ b/src/helm/benchmark/tokenizer_config_registry.py @@ -70,9 +70,11 @@ def get_tokenizer_config(name: str) -> Optional[TokenizerConfig]: return TOKENIZER_NAME_TO_CONFIG.get(name) -def register_tokenizers_if_not_already_registered() -> None: +def register_tokenizers_if_not_already_registered(base_path: str = "prod_env") -> None: global TOKENIZERS_REGISTERED if not TOKENIZERS_REGISTERED: path: str = resources.files(CONFIG_PACKAGE).joinpath(TOKENIZER_CONFIGS_FILE) + private_path: str = os.path.join(base_path, TOKENIZER_CONFIGS_FILE) maybe_register_tokenizer_configs_from_base_path(path) + maybe_register_tokenizer_configs_from_base_path(private_path) TOKENIZERS_REGISTERED = True diff --git a/src/helm/config/model_deployments.yaml b/src/helm/config/model_deployments.yaml index 50d2c9a78a..9e6f1be232 100644 --- a/src/helm/config/model_deployments.yaml +++ b/src/helm/config/model_deployments.yaml @@ -2,7 +2,12 @@ # Some models have several deployments, each with different parameters. # If you want to add a new deployment, you can technically do it here but we recommend -# you to do it in private/model_deployments.yaml instead. +# you to do it in prod_env/model_deployments.yaml instead. + +# Follow the template of this file to add a new deployment. You can copy paste this to get started: +# # This file defines all the model deployments that you do not want to be public. +# model_deployments: [] # Leave empty to disable private model deployments + model_deployments: @@ -506,8 +511,6 @@ model_deployments: class_name: "helm.benchmark.window_services.gpt2_window_service.GPT2WindowService" args: {} - - # HuggingFaceM4 - name: HuggingFaceM4/idefics-9b model_name: HuggingFaceM4/idefics-9b diff --git a/src/helm/config/model_metadata.yaml b/src/helm/config/model_metadata.yaml index 2e7da80791..09fca2f97f 100644 --- a/src/helm/config/model_metadata.yaml +++ b/src/helm/config/model_metadata.yaml @@ -2,7 +2,12 @@ # The model names here should match the model names in model_deployments.yaml. # If you want to add a new model, you can technically do it here but we recommend -# you to do it in private/model_metadata.yaml instead. +# you to do it in prod_env/model_metadata.yaml instead. + +# Follow the template of this file to add a new model. You can copy paste this to get started: +# # This file contains the metadata for private models +# models: [] # Leave empty to disable private models + models: diff --git a/src/helm/config/tokenizer_configs.yaml b/src/helm/config/tokenizer_configs.yaml index 3b115d9f33..7020fbb1f2 100644 --- a/src/helm/config/tokenizer_configs.yaml +++ b/src/helm/config/tokenizer_configs.yaml @@ -1,3 +1,13 @@ +# This file defines all the tokenizers that are supported by the Helm API. + +# If you want to add a new tokenizer, you can technically do it here but we recommend +# you to do it in prod_env/tokenizer_configs.yaml instead. + +# Follow the template of this file to add a new tokenizer. You can copy paste this to get started: +# # This file contains the tokenizer configs for the private tokenizers +# tokenizer_configs: [] # Leave empty to disable private tokenizers + + tokenizer_configs: - name: simple/model1