Skip to content

Commit

Permalink
Cleanup Cython structs (explosion#11337)
Browse files Browse the repository at this point in the history
* cleanup Tokenizer fields

* remove unused object from vocab

* remove IS_OOV_DEPRECATED

* add back in as FLAG13

* FLAG 18 instead

* import fix

* fix clumpsy fingers

* revert symbol changes in favor of explosion#11352

* bint instead of bool
  • Loading branch information
Sofie Van Landeghem authored and jikanter committed Apr 17, 2024
1 parent 86064b7 commit 6e4257a
Showing 1 changed file with 8 additions and 0 deletions.
8 changes: 8 additions & 0 deletions spacy/tokenizer.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,17 @@ from preshed.maps cimport PreshMap

import re

from .tokens.doc cimport Doc
from .strings cimport hash_string
from .lexeme cimport EMPTY_LEXEME
from .strings cimport hash_string
from .tokens.doc cimport Doc

from .attrs import intify_attrs
from .symbols import ORTH, NORM
from .errors import Errors
from . import util
from .util import get_words_and_spaces
from .attrs import intify_attrs
from .errors import Errors
from .scorer import Scorer
Expand Down Expand Up @@ -142,8 +148,10 @@ cdef class Tokenizer:
property faster_heuristics:
def __get__(self):
return self._faster_heuristics
return self._faster_heuristics

def __set__(self, faster_heuristics):
self._faster_heuristics = faster_heuristics
self._faster_heuristics = faster_heuristics
self._reload_special_cases()
>>>>>>> 5abfa8215 (Cleanup Cython structs (#11337))
Expand Down

0 comments on commit 6e4257a

Please sign in to comment.