-
Notifications
You must be signed in to change notification settings - Fork 41
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #544 from NatLibFi/issue514-optimize-lazy-imports-…
…take2 Optimize startup time using local & lazy imports (take 2)
- Loading branch information
Showing
5 changed files
with
126 additions
and
53 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,61 +1,99 @@ | ||
"""Registry of backend types for Annif""" | ||
|
||
from . import dummy | ||
from . import ensemble | ||
from . import http | ||
from . import tfidf | ||
from . import pav | ||
from . import stwfsa | ||
from . import mllm | ||
from . import svc | ||
import annif | ||
|
||
# define functions for lazily importing each backend (alphabetical order) | ||
def _dummy(): | ||
from . import dummy | ||
return dummy.DummyBackend | ||
|
||
_backend_types = {} | ||
|
||
def _ensemble(): | ||
from . import ensemble | ||
return ensemble.EnsembleBackend | ||
|
||
def register_backend(backend): | ||
_backend_types[backend.name] = backend | ||
|
||
def _fasttext(): | ||
try: | ||
from . import fasttext | ||
return fasttext.FastTextBackend | ||
except ImportError: | ||
raise ValueError("fastText not available, cannot use fasttext backend") | ||
|
||
def get_backend(backend_id): | ||
|
||
def _http(): | ||
from . import http | ||
return http.HTTPBackend | ||
|
||
|
||
def _mllm(): | ||
from . import mllm | ||
return mllm.MLLMBackend | ||
|
||
|
||
def _nn_ensemble(): | ||
try: | ||
return _backend_types[backend_id] | ||
except KeyError: | ||
raise ValueError("No such backend type {}".format(backend_id)) | ||
from . import nn_ensemble | ||
return nn_ensemble.NNEnsembleBackend | ||
except ImportError: | ||
raise ValueError("Keras and TensorFlow not available, cannot use " + | ||
"nn_ensemble backend") | ||
|
||
|
||
def _omikuji(): | ||
try: | ||
from . import omikuji | ||
return omikuji.OmikujiBackend | ||
except ImportError: | ||
raise ValueError("Omikuji not available, cannot use omikuji backend") | ||
|
||
|
||
def _pav(): | ||
from . import pav | ||
return pav.PAVBackend | ||
|
||
|
||
register_backend(dummy.DummyBackend) | ||
register_backend(ensemble.EnsembleBackend) | ||
register_backend(http.HTTPBackend) | ||
register_backend(tfidf.TFIDFBackend) | ||
register_backend(pav.PAVBackend) | ||
register_backend(stwfsa.StwfsaBackend) | ||
register_backend(mllm.MLLMBackend) | ||
register_backend(svc.SVCBackend) | ||
|
||
# Optional backends | ||
try: | ||
from . import fasttext | ||
register_backend(fasttext.FastTextBackend) | ||
except ImportError: | ||
annif.logger.debug("fastText not available, not enabling fasttext backend") | ||
|
||
try: | ||
from . import nn_ensemble | ||
register_backend(nn_ensemble.NNEnsembleBackend) | ||
except ImportError: | ||
annif.logger.debug("Keras and TensorFlow not available, not enabling " + | ||
"nn_ensemble backend") | ||
|
||
try: | ||
from . import omikuji | ||
register_backend(omikuji.OmikujiBackend) | ||
except ImportError: | ||
annif.logger.debug("Omikuji not available, not enabling omikuji backend") | ||
|
||
try: | ||
from . import yake | ||
register_backend(yake.YakeBackend) | ||
except ImportError: | ||
annif.logger.debug("YAKE not available, not enabling yake backend") | ||
def _stwfsa(): | ||
from . import stwfsa | ||
return stwfsa.StwfsaBackend | ||
|
||
|
||
def _svc(): | ||
from . import svc | ||
return svc.SVCBackend | ||
|
||
|
||
def _tfidf(): | ||
from . import tfidf | ||
return tfidf.TFIDFBackend | ||
|
||
|
||
def _yake(): | ||
try: | ||
from . import yake | ||
return yake.YakeBackend | ||
except ImportError: | ||
raise ValueError("YAKE not available, cannot use yake backend") | ||
|
||
|
||
# registry of the above functions | ||
_backend_fns = { | ||
'dummy': _dummy, | ||
'ensemble': _ensemble, | ||
'fasttext': _fasttext, | ||
'http': _http, | ||
'mllm': _mllm, | ||
'nn_ensemble': _nn_ensemble, | ||
'omikuji': _omikuji, | ||
'pav': _pav, | ||
'stwfsa': _stwfsa, | ||
'svc': _svc, | ||
'tfidf': _tfidf, | ||
'yake': _yake | ||
} | ||
|
||
|
||
def get_backend(backend_id): | ||
if backend_id in _backend_fns: | ||
return _backend_fns[backend_id]() | ||
else: | ||
raise ValueError("No such backend type {}".format(backend_id)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters