From 9232c59642c50bb11bec9856f22c8114d58a18c6 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Thu, 5 Sep 2024 15:15:44 -0700 Subject: [PATCH 1/3] [BugFix] Fix Granite model configuration The recently added granite model doesn't load properly since it's not yet in the latest transformers release. This changes the model config loading logic to not require a registered AutoConfig if the model type has a custom config class in our own config registry. --- vllm/transformers_utils/config.py | 59 ++++++++++++--------- vllm/transformers_utils/configs/__init__.py | 4 ++ 2 files changed, 39 insertions(+), 24 deletions(-) diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index dfe83ddb731d4..3866fecda7c68 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -12,10 +12,11 @@ from vllm.logger import init_logger from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig, EAGLEConfig, ExaoneConfig, - InternVLChatConfig, JAISConfig, - MedusaConfig, MLPSpeculatorConfig, - MPTConfig, NemotronConfig, - RWConfig, UltravoxConfig) + GraniteConfig, InternVLChatConfig, + JAISConfig, MedusaConfig, + MLPSpeculatorConfig, MPTConfig, + NemotronConfig, RWConfig, + UltravoxConfig) from vllm.transformers_utils.utils import check_gguf_file if VLLM_USE_MODELSCOPE: @@ -39,6 +40,9 @@ "internvl_chat": InternVLChatConfig, "nemotron": NemotronConfig, "ultravox": UltravoxConfig, + # Granite can be removed from here once we have upgraded to + # transformers 4.45+ + "granite": GraniteConfig, } for name, cls in _CONFIG_REGISTRY.items(): @@ -62,29 +66,36 @@ def get_config( kwargs["gguf_file"] = Path(model).name model = Path(model).parent - try: - config = AutoConfig.from_pretrained( - model, - trust_remote_code=trust_remote_code, - revision=revision, - code_revision=code_revision, - **kwargs) - except ValueError as e: - if (not trust_remote_code and - "requires you to execute the configuration file" in str(e)): - err_msg = ( - "Failed to load the model config. If the model is a custom " - "model not yet available in the HuggingFace transformers " - "library, consider setting `trust_remote_code=True` in LLM " - "or using the `--trust-remote-code` flag in the CLI.") - raise RuntimeError(err_msg) from e - else: - raise e - if config.model_type in _CONFIG_REGISTRY: - config_class = _CONFIG_REGISTRY[config.model_type] + config_dict, _ = PretrainedConfig.get_config_dict( + model, revision=revision, code_revision=code_revision, **kwargs) + + # Use custom model class if it's in our registry + model_type = config_dict.get("model_type") + if model_type in _CONFIG_REGISTRY: + config_class = _CONFIG_REGISTRY[model_type] config = config_class.from_pretrained(model, revision=revision, code_revision=code_revision) + else: + try: + config = AutoConfig.from_pretrained( + model, + trust_remote_code=trust_remote_code, + revision=revision, + code_revision=code_revision, + **kwargs) + except ValueError as e: + if (not trust_remote_code + and "requires you to execute the configuration file" + in str(e)): + err_msg = ( + "Failed to load the model config. If the model is a custom " + "model not yet available in the HuggingFace transformers " + "library, consider setting `trust_remote_code=True` in LLM " + "or using the `--trust-remote-code` flag in the CLI.") + raise RuntimeError(err_msg) from e + else: + raise e # Special architecture mapping check for GGUF models if is_gguf: diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index 736878b35ad49..8381c5227584e 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -6,6 +6,7 @@ # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the # `FalconConfig` class from the official HuggingFace transformers library. from vllm.transformers_utils.configs.falcon import RWConfig +from vllm.transformers_utils.configs.granite import GraniteConfig from vllm.transformers_utils.configs.internvl import InternVLChatConfig from vllm.transformers_utils.configs.jais import JAISConfig from vllm.transformers_utils.configs.medusa import MedusaConfig @@ -27,4 +28,7 @@ "MLPSpeculatorConfig", "NemotronConfig", "UltravoxConfig", + # Granite can be removed from here once we have upgraded to + # transformers 4.45+ + "GraniteConfig", ] From f0c6068fad4103284d7f35f2cb505edf0c800e18 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Thu, 5 Sep 2024 15:23:58 -0700 Subject: [PATCH 2/3] yapf --- vllm/transformers_utils/config.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 3866fecda7c68..12c5fbfea2391 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -10,13 +10,10 @@ from vllm.envs import VLLM_USE_MODELSCOPE from vllm.logger import init_logger -from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig, - EAGLEConfig, ExaoneConfig, - GraniteConfig, InternVLChatConfig, - JAISConfig, MedusaConfig, - MLPSpeculatorConfig, MPTConfig, - NemotronConfig, RWConfig, - UltravoxConfig) +from vllm.transformers_utils.configs import ( + ChatGLMConfig, DbrxConfig, EAGLEConfig, ExaoneConfig, GraniteConfig, + InternVLChatConfig, JAISConfig, MedusaConfig, MLPSpeculatorConfig, + MPTConfig, NemotronConfig, RWConfig, UltravoxConfig) from vllm.transformers_utils.utils import check_gguf_file if VLLM_USE_MODELSCOPE: From 458cd431f7551f985fed6c03f3b7ae3999b7bcb7 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Thu, 5 Sep 2024 15:27:54 -0700 Subject: [PATCH 3/3] Fix yamf conflict --- vllm/transformers_utils/config.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 12c5fbfea2391..4f4e79d10a677 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -10,10 +10,16 @@ from vllm.envs import VLLM_USE_MODELSCOPE from vllm.logger import init_logger -from vllm.transformers_utils.configs import ( - ChatGLMConfig, DbrxConfig, EAGLEConfig, ExaoneConfig, GraniteConfig, - InternVLChatConfig, JAISConfig, MedusaConfig, MLPSpeculatorConfig, - MPTConfig, NemotronConfig, RWConfig, UltravoxConfig) +# yapf conflicts with isort for this block +# yapf: disable +from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig, + EAGLEConfig, ExaoneConfig, + GraniteConfig, InternVLChatConfig, + JAISConfig, MedusaConfig, + MLPSpeculatorConfig, MPTConfig, + NemotronConfig, RWConfig, + UltravoxConfig) +# yapf: enable from vllm.transformers_utils.utils import check_gguf_file if VLLM_USE_MODELSCOPE: