Issue273 default values for configuration settings

annif/backend/backend.py

@@ -12,13 +12,30 @@ class AnnifBackend(metaclass=abc.ABCMeta):
     needs_subject_index = False
     needs_subject_vectorizer = False
 
+    DEFAULT_PARAMS = {'limit': 100}
+
     def __init__(self, backend_id, params, datadir):
         """Initialize backend with specific parameters. The
         parameters are a dict. Keys and values depend on the specific
         backend type."""
         self.backend_id = backend_id
-        self.params = params
         self.datadir = datadir
+        self.params = self.fill_params_with_defaults(params)
+
+    def default_params(self):
+        return self.DEFAULT_PARAMS
+
+    def fill_params_with_defaults(self, params):
+        """Set the parameters that are not provided in the projects config file
+        with default values set in the backend class."""
+        params_to_set = {param: str(val)
+                         for param, val in self.default_params().items()
+                         if param not in params}
+        if params_to_set:
+            self.debug('all parameters not set, using following defaults: {}'
+                       .format(params_to_set))
+            params.update(params_to_set)
+        return params
 
     def train(self, corpus, project):
         """train the model on the given document or subject corpus"""

annif/backend/dummy.py

@@ -11,6 +11,10 @@ class DummyBackend(backend.AnnifLearningBackend):
     uri = 'http://example.org/dummy'
     label = 'dummy'
 
+    def default_params(self):
+        params = backend.AnnifBackend.DEFAULT_PARAMS.copy()
+        return params
+
     def initialize(self):
         self.initialized = True
 

annif/backend/fasttext.py

@@ -33,12 +33,25 @@ class FastTextBackend(mixins.ChunkingBackend, backend.AnnifBackend):
         't': float
     }
 
+    DEFAULT_PARAMS = {
+        'dim': 100,
+        'lr': 0.25,
+        'epoch': 5,
+        'loss': 'hs',
+    }
+
     MODEL_FILE = 'fasttext-model'
     TRAIN_FILE = 'fasttext-train.txt'
 
     # defaults for uninitialized instances
     _model = None
 
+    def default_params(self):
+        params = backend.AnnifBackend.DEFAULT_PARAMS.copy()
+        params.update(mixins.ChunkingBackend.DEFAULT_PARAMS)
+        params.update(self.DEFAULT_PARAMS)
+        return params
+
     def initialize(self):
         if self._model is None:
             path = os.path.join(self.datadir, self.MODEL_FILE)

annif/backend/mixins.py

@@ -8,6 +8,11 @@
 class ChunkingBackend(metaclass=abc.ABCMeta):
     """Annif backend mixin that implements chunking of input"""
 
+    DEFAULT_PARAMS = {'chunksize': 1}
+
+    def default_params(self):
+        return self.DEFAULT_PARAMS
+
     @abc.abstractmethod
     def _suggest_chunks(self, chunktexts, project):
         """Suggest subjects for the chunked text; should be implemented by

annif/backend/pav.py

@@ -24,6 +24,8 @@ class PAVBackend(ensemble.EnsembleBackend):
     # defaults for uninitialized instances
     _models = None
 
+    DEFAULT_PARAMS = {'min-docs': 10}
+
     def initialize(self):
         if self._models is not None:
             return  # already initialized

annif/backend/vw_ensemble.py

@@ -10,6 +10,7 @@
 import numpy as np
 from annif.exception import NotInitializedException
 from annif.suggestion import VectorSuggestionResult
+from . import backend
 from . import vw_base
 from . import ensemble
 
@@ -42,7 +43,16 @@ class VWEnsembleBackend(
     # a simple mean of scores. A higher value will mean that the model
     # adapts quicker (and possibly makes more errors) while a lower value
     # will make it more careful so that it will require more training data.
-    DEFAULT_DISCOUNT_RATE = 0.01
+
+    DEFAULT_PARAMS = {'discount_rate': 0.01}
+
+    def default_params(self):
+        params = backend.AnnifBackend.DEFAULT_PARAMS.copy()
+        params.update(self.DEFAULT_PARAMS)
+        params.update({param: default_val
+                       for param, (_, default_val) in self.VW_PARAMS.items()
+                       if default_val is not None})
+        return params
 
     def _load_subject_freq(self):
         path = os.path.join(self.datadir, self.FREQ_FILE)
@@ -75,8 +85,7 @@ def _calculate_scores(self, subj_id, subj_score_vector):
     def _merge_hits_from_sources(self, hits_from_sources, project, params):
         score_vector = np.array([hits.vector
                                  for hits, _ in hits_from_sources])
-        discount_rate = float(self.params.get('discount_rate',
-                                              self.DEFAULT_DISCOUNT_RATE))
+        discount_rate = float(self.params['discount_rate'])
         result = np.zeros(score_vector.shape[1])
         for subj_id in range(score_vector.shape[1]):
             subj_score_vector = score_vector[:, subj_id]

annif/backend/vw_multi.py

@@ -7,6 +7,7 @@
 from annif.suggestion import ListSuggestionResult, VectorSuggestionResult
 from annif.exception import ConfigurationException
 from . import vw_base
+from . import backend
 from . import mixins
 
 
@@ -27,14 +28,24 @@ class VWMultiBackend(mixins.ChunkingBackend, vw_base.VWBaseBackend):
         'probabilities': (bool, None)
     }
 
-    DEFAULT_ALGORITHM = 'oaa'
     SUPPORTED_ALGORITHMS = ('oaa', 'ect', 'log_multi', 'multilabel_oaa')
 
     DEFAULT_INPUTS = '_text_'
 
+    DEFAULT_PARAMS = {'algorithm': 'oaa'}
+
+    def default_params(self):
+        params = backend.AnnifBackend.DEFAULT_PARAMS.copy()
+        params.update(mixins.ChunkingBackend.DEFAULT_PARAMS)
+        params.update(self.DEFAULT_PARAMS)
+        params.update({param: default_val
+                       for param, (_, default_val) in self.VW_PARAMS.items()
+                       if default_val is not None})
+        return params
+
     @property
     def algorithm(self):
-        algorithm = self.params.get('algorithm', self.DEFAULT_ALGORITHM)
+        algorithm = self.params['algorithm']
         if algorithm not in self.SUPPORTED_ALGORITHMS:
             raise ConfigurationException(
                 "{} is not a valid algorithm (allowed: {})".format(

annif/project.py

@@ -45,7 +45,7 @@ class AnnifProject(DatadirMixin):
     def __init__(self, project_id, config, datadir):
         DatadirMixin.__init__(self, datadir, 'projects', project_id)
         self.project_id = project_id
-        self.name = config['name']
+        self.name = config.get('name', project_id)
         self.language = config['language']
         self.analyzer_spec = config.get('analyzer', None)
         self.vocab_id = config.get('vocab', None)

tests/projects.cfg

@@ -50,6 +50,26 @@ language=en
 vocab=dummy
 analyzer=snowball(english)
 
+[noname]
+language=en
+backend=tfidf
+vocab=dummy
+analyzer=snowball(english)
+
+[noparams-tfidf-fi]
+name=TF-IDF Finnish using default params
+language=fi
+backend=tfidf
+analyzer=snowball(finnish)
+vocab=yso-fi
+
+[noparams-fasttext-fi]
+name=fastText Finnish using default params
+language=fi
+backend=fasttext
+analyzer=snowball(finnish)
+vocab=yso-fi
+
 [pav]
 name=PAV Ensemble Finnish
 language=fi

tests/test_backend.py

@@ -1,6 +1,7 @@
 """Unit tests for backends in Annif"""
 
 import pytest
+import logging
 import annif
 import annif.backend
 import annif.corpus
@@ -40,3 +41,16 @@ def test_learn_dummy(app, project, tmpdir):
     assert result[0].uri == 'http://example.org/key1'
     assert result[0].label == 'key1'
     assert result[0].score == 1.0
+
+
+def test_fill_params_with_defaults(app, caplog):
+    logger = annif.logger
+    logger.propagate = True
+    dummy_type = annif.backend.get_backend('dummy')
+    dummy = dummy_type(backend_id='dummy', params={},
+                       datadir=app.config['DATADIR'])
+    expected_default_params = {'limit': '100'}  # From AnnifBackend class
+    expected_msg = 'all parameters not set, using following defaults:'
+    caplog.set_level(logging.DEBUG)
+    assert expected_default_params == dummy.fill_params_with_defaults({})
+    assert expected_msg in caplog.records[0].message

tests/test_backend_fasttext.py

@@ -1,12 +1,39 @@
 """Unit tests for the fastText backend in Annif"""
 
+import logging
 import pytest
 import annif.backend
 import annif.corpus
 
 fasttext = pytest.importorskip("annif.backend.fasttext")
 
 
+def test_fasttext_default_params(datadir, project, caplog):
+    logger = annif.logger
+    logger.propagate = True
+    caplog.set_level(logging.DEBUG)
+
+    fasttext_type = annif.backend.get_backend("fasttext")
+    fasttext = fasttext_type(
+        backend_id='fasttext',
+        params={},
+        datadir=str(datadir))
+
+    expected_default_params = {
+        'limit': 100,
+        'chunksize': 1,
+        'dim': 100,
+        'lr': 0.25,
+        'epoch': 5,
+        'loss': 'hs',
+    }
+    expected_msg = "all parameters not set, using following defaults:"
+    assert expected_msg in caplog.records[0].message
+    actual_params = fasttext.params
+    for param, val in expected_default_params.items():
+        assert param in actual_params and actual_params[param] == str(val)
+
+
 def test_fasttext_train(datadir, document_corpus, project):
     fasttext_type = annif.backend.get_backend("fasttext")
     fasttext = fasttext_type(

tests/test_backend_pav.py

@@ -1,9 +1,31 @@
 """Unit tests for the PAV backend in Annif"""
 
+import logging
 import annif.backend
 import annif.corpus
 
 
+def test_pav_default_params(datadir, document_corpus, project, caplog):
+    logger = annif.logger
+    logger.propagate = True
+    caplog.set_level(logging.DEBUG)
+
+    pav_type = annif.backend.get_backend("pav")
+    pav = pav_type(
+        backend_id='pav',
+        params={},
+        datadir=str(datadir))
+
+    expected_default_params = {
+        'min-docs': 10,
+    }
+    expected_msg = "all parameters not set, using following defaults:"
+    assert expected_msg in caplog.records[0].message
+    actual_params = pav.params
+    for param, val in expected_default_params.items():
+        assert param in actual_params and actual_params[param] == str(val)
+
+
 def test_pav_train(app, datadir, tmpdir, project):
     pav_type = annif.backend.get_backend("pav")
     pav = pav_type(

tests/test_backend_tfidf.py

@@ -4,6 +4,7 @@
 import annif.backend
 import annif.corpus
 from sklearn.feature_extraction.text import TfidfVectorizer
+import logging
 import pytest
 import unittest.mock
 
@@ -18,6 +19,27 @@ def project(document_corpus):
     return proj
 
 
+def test_tfidf_default_params(datadir, project, caplog):
+    logger = annif.logger
+    logger.propagate = True
+    caplog.set_level(logging.DEBUG)
+
+    tfidf_type = annif.backend.get_backend("tfidf")
+    tfidf = tfidf_type(
+        backend_id='tfidf',
+        params={},
+        datadir=str(datadir))
+
+    expected_default_params = {
+        'limit': 100  # From AnnifBackend class
+    }
+    expected_msg = "all parameters not set, using following defaults:"
+    assert expected_msg in caplog.records[0].message
+    actual_params = tfidf.params
+    for param, val in expected_default_params.items():
+        assert param in actual_params and actual_params[param] == str(val)
+
+
 def test_tfidf_train(datadir, document_corpus, project):
     tfidf_type = annif.backend.get_backend("tfidf")
     tfidf = tfidf_type(

tests/test_backend_vw_ensemble.py

@@ -1,5 +1,6 @@
 """Unit tests for the vw_ensemble backend in Annif"""
 
+import logging
 import json
 import time
 import pytest
@@ -11,6 +12,29 @@
 pytest.importorskip("annif.backend.vw_ensemble")
 
 
+def test_vw_ensemble_default_params(datadir, project, caplog):
+    logger = annif.logger
+    logger.propagate = True
+    caplog.set_level(logging.DEBUG)
+
+    vw_type = annif.backend.get_backend("vw_ensemble")
+    vw = vw_type(
+        backend_id='vw_ensemble',
+        params={},
+        datadir=str(datadir))
+
+    expected_default_params = {
+        'limit': 100,
+        'discount_rate': 0.01,
+        'loss_function': 'squared',
+    }
+    expected_msg = "all parameters not set, using following defaults:"
+    assert expected_msg in caplog.records[0].message
+    actual_params = vw.params
+    for param, val in expected_default_params.items():
+        assert param in actual_params and actual_params[param] == str(val)
+
+
 def test_vw_ensemble_suggest_no_model(datadir, project):
     vw_ensemble_type = annif.backend.get_backend('vw_ensemble')
     vw_ensemble = vw_ensemble_type(