ProteinTokenizerTransform

mkdocs.yml

@@ -43,6 +43,7 @@ nav:
     - "beignet.func":
         - "Molecular dynamics": "reference/func/molecular-dynamics.md"
     - "beignet.io": "reference/beignet.io.md"
+    - "beignet.tokenizers": "reference/beignet.tokenizers.md"
     - "beignet.transforms":
         - "General-purpose": "reference/transforms/general-purpose.md"
 plugins:

pyproject.toml

@@ -22,7 +22,7 @@ requires-python = ">=3.10"
 
 [project.optional-dependencies]
 all = [
-    "beignet[datasets,docs,mdtraj,test]",
+    "beignet[datasets,docs,mdtraj,test,tokenizers]",
 ]
 datasets = [
     "biopython",
@@ -44,6 +44,9 @@ test = [
     "pytest-mock",
     "scipy",
 ]
+tokenizers = [
+    "transformers",
+]
 
 [tool.ruff]
 lint.select = [

src/beignet/data/tokenizers/3di_tokenizer/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+    "cls_token": "<cls>",
+    "eos_token": "<eos>",
+    "mask_token": "<mask>",
+    "pad_token": "<pad>",
+    "unk_token": "<unk>"
+}

src/beignet/data/tokenizers/3di_tokenizer/tokenizer_config.json

@@ -0,0 +1,6 @@
+{
+    "clean_up_tokenization_spaces": true,
+    "do_lower_case": false,
+    "model_max_length": 1024,
+    "tokenizer_class": "SequenceTokenizer"
+}

src/beignet/data/tokenizers/3di_tokenizer/vocab.txt

@@ -0,0 +1,53 @@
+<cls>
+<pad>
+<eos>
+<unk>
+L
+A
+G
+V
+S
+E
+R
+T
+I
+D
+P
+K
+Q
+N
+F
+Y
+M
+H
+W
+C
+X
+B
+U
+Z
+O
+.
+-
+<null_1>
+<mask>
+p
+y
+n
+w
+r
+q
+h
+g
+d
+l
+v
+t
+m
+f
+s
+a
+e
+i
+k
+c

src/beignet/data/tokenizers/ab_tokenizer/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+    "cls_token": "[CLS]",
+    "eos_token": "[SEP]",
+    "mask_token": "[MASK]",
+    "pad_token": "[PAD]",
+    "unk_token": "[UNK]"
+}

src/beignet/data/tokenizers/ab_tokenizer/tokenizer_config.json

@@ -0,0 +1,6 @@
+{
+    "clean_up_tokenization_spaces": true,
+    "do_lower_case": false,
+    "model_max_length": 1000000,
+    "tokenizer_class": "SequenceTokenizer"
+}