Merge pull request #6 from espadolini/lazy-load

Generate embedded vocabulary maps lazily
tiktoken-go · May 10, 2024 · 7f65fe7 · 7f65fe7
2 parents 70d9b59 + 93a9139
commit 7f65fe7
Show file tree

Hide file tree

Showing 8 changed files with 200,875 additions and 200,821 deletions.
diff --git a/codec/cl100k_base.go b/codec/cl100k_base.go
@@ -3,6 +3,7 @@ package codec
 import "github.com/dlclark/regexp2"
 
 func NewCl100kBase() *Codec {
+	cl100kBaseVocabOnce.Do(cl100kBaseVocabInit)
 	return &Codec{
 		name:        "cl100k_base",
 		vocabulary:  cl100kBaseVocab,