-
Notifications
You must be signed in to change notification settings - Fork 41
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into update-dependencies-v1.2
- Loading branch information
Showing
12 changed files
with
190 additions
and
23 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
"""Wrapper code for using Simplemma functionality in Annif""" | ||
|
||
from typing import Tuple, Union | ||
|
||
from simplemma import LanguageDetector, Lemmatizer | ||
from simplemma.strategies import DefaultStrategy | ||
from simplemma.strategies.dictionaries import DefaultDictionaryFactory | ||
|
||
LANG_CACHE_SIZE = 5 # How many language dictionaries to keep in memory at once (max) | ||
|
||
_dictionary_factory = DefaultDictionaryFactory(cache_max_size=LANG_CACHE_SIZE) | ||
_lemmatization_strategy = DefaultStrategy(dictionary_factory=_dictionary_factory) | ||
lemmatizer = Lemmatizer(lemmatization_strategy=_lemmatization_strategy) | ||
|
||
|
||
def get_language_detector(lang: Union[str, Tuple[str, ...]]) -> LanguageDetector: | ||
return LanguageDetector(lang, lemmatization_strategy=_lemmatization_strategy) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
"""Unit tests for Simplemma utility functions""" | ||
|
||
import pytest | ||
|
||
from annif.simplemma_util import get_language_detector | ||
|
||
|
||
def test_get_language_detector(): | ||
detector = get_language_detector("en") | ||
text = "She said 'au revoir' and left" | ||
proportion = detector.proportion_in_target_languages(text) | ||
assert proportion == pytest.approx(0.75) | ||
|
||
|
||
def test_get_language_detector_many(): | ||
detector = get_language_detector(("en", "fr")) | ||
text = "She said 'au revoir' and left" | ||
proportion = detector.proportion_in_target_languages(text) | ||
assert proportion == pytest.approx(1.0) |