diff --git a/pyproject.toml b/pyproject.toml index 863df99..7e3b1af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "mistral_inference" -version = "1.2.0" +version = "1.3.0" description = "" authors = ["bam4d "] readme = "README.md" @@ -27,7 +27,7 @@ python = "^3.9.10" xformers = ">=0.0.24" simple-parsing = ">=0.1.5" fire = ">=0.6.0" -mistral_common = "^1.0.0" +mistral_common = "^1.3.0" safetensors = ">=0.4.0" [tool.poetry.group.dev.dependencies] diff --git a/src/mistral_inference/__init__.py b/src/mistral_inference/__init__.py index c68196d..67bc602 100644 --- a/src/mistral_inference/__init__.py +++ b/src/mistral_inference/__init__.py @@ -1 +1 @@ -__version__ = "1.2.0" +__version__ = "1.3.0" diff --git a/src/mistral_inference/main.py b/src/mistral_inference/main.py index 9cafec8..d4302fe 100644 --- a/src/mistral_inference/main.py +++ b/src/mistral_inference/main.py @@ -12,6 +12,8 @@ from mistral_common.protocol.instruct.request import ChatCompletionRequest from mistral_common.tokens.tokenizers.base import Tokenizer from mistral_common.tokens.tokenizers.mistral import MistralTokenizer +from mistral_common.tokens.tokenizers.sentencepiece import is_sentencepiece +from mistral_common.tokens.tokenizers.tekken import is_tekken from mistral_inference.generate import generate, generate_mamba from mistral_inference.mamba import Mamba @@ -24,10 +26,10 @@ def is_torchrun() -> bool: def load_tokenizer(model_path: Path) -> MistralTokenizer: - tokenizer = [f for f in os.listdir(Path(model_path)) if f.startswith("tokenizer.model")] + tokenizer = [f for f in os.listdir(model_path) if is_tekken(model_path / f) or is_sentencepiece(model_path / f)] assert ( len(tokenizer) > 0 - ), f"No tokenizer found in {model_path}, make sure to place a `tokenizer.model.[v1,v2,v3]` file in {model_path}." + ), f"No tokenizer in {model_path}, place a `tokenizer.model.[v1,v2,v3]` or `tekken.json` file in {model_path}." assert ( len(tokenizer) == 1 ), f"Multiple tokenizers {', '.join(tokenizer)} found in `model_path`, make sure to only have one tokenizer"