From fd097d3d7c6d213445dfb4f3857a7241d5bfca06 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Mon, 22 Aug 2022 15:52:24 +0200 Subject: [PATCH] Cleanup Cython structs (#11337) * cleanup Tokenizer fields * remove unused object from vocab * remove IS_OOV_DEPRECATED * add back in as FLAG13 * FLAG 18 instead * import fix * fix clumpsy fingers * revert symbol changes in favor of #11352 * bint instead of bool --- spacy/tokenizer.pyx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/spacy/tokenizer.pyx b/spacy/tokenizer.pyx index 8485a57c8aa..2d6c879e360 100644 --- a/spacy/tokenizer.pyx +++ b/spacy/tokenizer.pyx @@ -9,11 +9,17 @@ from preshed.maps cimport PreshMap import re +from .tokens.doc cimport Doc +from .strings cimport hash_string from .lexeme cimport EMPTY_LEXEME from .strings cimport hash_string from .tokens.doc cimport Doc +from .attrs import intify_attrs +from .symbols import ORTH, NORM +from .errors import Errors from . import util +from .util import get_words_and_spaces from .attrs import intify_attrs from .errors import Errors from .scorer import Scorer @@ -142,10 +148,8 @@ cdef class Tokenizer: property faster_heuristics: def __get__(self): return self._faster_heuristics - return self._faster_heuristics def __set__(self, faster_heuristics): - self._faster_heuristics = faster_heuristics self._faster_heuristics = faster_heuristics self._reload_special_cases() >>>>>>> 5abfa8215 (Cleanup Cython structs (#11337))