Skip to content

Commit

Permalink
Add support for chatgpt-4o-latest (#85)
Browse files Browse the repository at this point in the history
Co-authored-by: Roger Zurawicki <zurawiki@users.noreply.github.com>
  • Loading branch information
Congyuwang and zurawiki authored Oct 14, 2024
1 parent b4ce7a6 commit 759340e
Showing 1 changed file with 5 additions and 0 deletions.
5 changes: 5 additions & 0 deletions tiktoken-rs/src/tokenizer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ const MODEL_PREFIX_TO_TOKENIZER: &[(&str, Tokenizer)] = &[
// https://github.com/openai/tiktoken/blob/63527649963def8c759b0f91f2eb69a40934e468/tiktoken/model.py#L22
const MODEL_TO_TOKENIZER: &[(&str, Tokenizer)] = &[
// chat
("chatgpt-4o-latest", Tokenizer::O200kBase),
("gpt-4o", Tokenizer::O200kBase),
("gpt-4", Tokenizer::Cl100kBase),
("gpt-3.5-turbo", Tokenizer::Cl100kBase),
Expand Down Expand Up @@ -152,6 +153,10 @@ mod tests {

#[test]
fn test_get_tokenizer() {
assert_eq!(
get_tokenizer("chatgpt-4o-latest"),
Some(Tokenizer::O200kBase)
);
assert_eq!(
get_tokenizer("gpt-4o-2024-05-13"),
Some(Tokenizer::O200kBase)
Expand Down

0 comments on commit 759340e

Please sign in to comment.