Speed up optimize command and avoid high memory usage

annif/cli.py

@@ -267,7 +267,7 @@ def run_optimize(project_id, paths, backend_param):
 
     template = "{:d}\t{:.02f}\t{:.04f}\t{:.04f}\t{:.04f}"
     for params, filter_batch in filter_batches.items():
-        results = filter_batch[1].results()
+        results = filter_batch[1].results(metrics='simple')
         for metric, score in results.items():
             if score >= best_scores[metric]:
                 best_scores[metric] = score

annif/eval.py

@@ -79,53 +79,63 @@ def __init__(self, subject_index):
     def evaluate(self, hits, gold_subjects):
         self._samples.append((hits, gold_subjects))
 
-    def results(self):
+    def _evaluate_samples(self, y_true, y_pred, metrics='all'):
+        y_pred_binary = y_pred > 0.0
+        results = collections.OrderedDict()
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore')
+
+            results['Precision (doc avg)'] = precision_score(
+                y_true, y_pred_binary, average='samples')
+            results['Recall (doc avg)'] = recall_score(
+                y_true, y_pred_binary, average='samples')
+            results['F1 score (doc avg)'] = f1_score(
+                y_true, y_pred_binary, average='samples')
+            if metrics == 'all':
+                results['Precision (conc avg)'] = precision_score(
+                    y_true, y_pred_binary, average='macro')
+                results['Recall (conc avg)'] = recall_score(
+                    y_true, y_pred_binary, average='macro')
+                results['F1 score (conc avg)'] = f1_score(
+                    y_true, y_pred_binary, average='macro')
+                results['Precision (microavg)'] = precision_score(
+                    y_true, y_pred_binary, average='micro')
+                results['Recall (microavg)'] = recall_score(
+                    y_true, y_pred_binary, average='micro')
+                results['F1 score (microavg)'] = f1_score(
+                    y_true, y_pred_binary, average='micro')
+            results['NDCG'] = ndcg_score(y_true, y_pred)
+            results['NDCG@5'] = ndcg_score(y_true, y_pred, limit=5)
+            results['NDCG@10'] = ndcg_score(y_true, y_pred, limit=10)
+            if metrics == 'all':
+                results['Precision@1'] = precision_at_k_score(
+                    y_true, y_pred, limit=1)
+                results['Precision@3'] = precision_at_k_score(
+                    y_true, y_pred, limit=3)
+                results['Precision@5'] = precision_at_k_score(
+                    y_true, y_pred, limit=5)
+                results['LRAP'] = label_ranking_average_precision_score(
+                    y_true, y_pred)
+                results['True positives'] = true_positives(
+                    y_true, y_pred_binary)
+                results['False positives'] = false_positives(
+                    y_true, y_pred_binary)
+                results['False negatives'] = false_negatives(
+                    y_true, y_pred_binary)
+
+        return results
+
+    def results(self, metrics='all'):
         """evaluate a set of selected subjects against a gold standard using
-        different metrics"""
+        different metrics. The set of metrics can be either 'all' or
+        'simple'."""
 
         y_true = np.array([gold_subjects.as_vector(self._subject_index)
                            for hits, gold_subjects in self._samples])
         y_pred = np.array([hits.vector
                            for hits, gold_subjects in self._samples])
-        y_pred_binary = y_pred > 0.0
-
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore')
-
-            results = collections.OrderedDict([
-                ('Precision (doc avg)',
-                 precision_score(y_true, y_pred_binary, average='samples')),
-                ('Recall (doc avg)',
-                 recall_score(y_true, y_pred_binary, average='samples')),
-                ('F1 score (doc avg)',
-                 f1_score(y_true, y_pred_binary, average='samples')),
-                ('Precision (conc avg)',
-                 precision_score(y_true, y_pred_binary, average='macro')),
-                ('Recall (conc avg)',
-                 recall_score(y_true, y_pred_binary, average='macro')),
-                ('F1 score (conc avg)',
-                 f1_score(y_true, y_pred_binary, average='macro')),
-                ('Precision (microavg)',
-                 precision_score(y_true, y_pred_binary, average='micro')),
-                ('Recall (microavg)',
-                 recall_score(y_true, y_pred_binary, average='micro')),
-                ('F1 score (microavg)',
-                 f1_score(y_true, y_pred_binary, average='micro')),
-                ('NDCG', ndcg_score(y_true, y_pred)),
-                ('NDCG@5', ndcg_score(y_true, y_pred, limit=5)),
-                ('NDCG@10', ndcg_score(y_true, y_pred, limit=10)),
-                ('Precision@1',
-                 precision_at_k_score(y_true, y_pred, limit=1)),
-                ('Precision@3',
-                 precision_at_k_score(y_true, y_pred, limit=3)),
-                ('Precision@5',
-                 precision_at_k_score(y_true, y_pred, limit=5)),
-                ('LRAP',
-                 label_ranking_average_precision_score(y_true, y_pred)),
-                ('True positives', true_positives(y_true, y_pred_binary)),
-                ('False positives', false_positives(y_true, y_pred_binary)),
-                ('False negatives', false_negatives(y_true, y_pred_binary)),
-                ('Documents evaluated', y_true.shape[0])
-            ])
 
+        results = self._evaluate_samples(
+            y_true, y_pred, metrics)
+        results['Documents evaluated'] = y_true.shape[0]
         return results

annif/hit.py

@@ -17,7 +17,9 @@ def __init__(self, limit=None, threshold=0.0):
         self._threshold = threshold
 
     def __call__(self, orighits):
-        return orighits.filter(self._limit, self._threshold)
+        return LazyAnalysisResult(
+            lambda: orighits.filter(
+                self._limit, self._threshold))
 
 
 class AnalysisResult(metaclass=abc.ABCMeta):
@@ -53,6 +55,44 @@ def __getitem__(self, idx):
         return self.hits[idx]
 
 
+class LazyAnalysisResult(AnalysisResult):
+    """AnalysisResult implementation that wraps another AnalysisResult which
+    is initialized lazily only when it is actually accessed. Method calls
+    will be proxied to the wrapped AnalysisResult."""
+
+    def __init__(self, construct):
+        """Create the proxy object. The given construct function will be
+        called to create the actual AnalysisResult when it is needed."""
+        self._construct = construct
+        self._object = None
+
+    def _initialize(self):
+        if self._object is None:
+            self._object = self._construct()
+
+    @property
+    def hits(self):
+        self._initialize()
+        return self._object.hits
+
+    @property
+    def vector(self):
+        self._initialize()
+        return self._object.vector
+
+    def filter(self, limit=None, threshold=0.0):
+        self._initialize()
+        return self._object.filter(limit, threshold)
+
+    def __len__(self):
+        self._initialize()
+        return len(self._object)
+
+    def __getitem__(self, idx):
+        self._initialize()
+        return self._object[idx]
+
+
 class VectorAnalysisResult(AnalysisResult):
     """AnalysisResult implementation based primarily on NumPy vectors."""
 

tests/test_hit.py

@@ -1,7 +1,7 @@
 """Unit tests for hit processing in Annif"""
 
-from annif.hit import AnalysisHit, AnalysisResult, ListAnalysisResult, \
-    HitFilter
+from annif.hit import AnalysisHit, AnalysisResult, LazyAnalysisResult, \
+    ListAnalysisResult, HitFilter
 from annif.corpus import SubjectIndex
 import numpy as np
 
@@ -38,6 +38,18 @@ def test_hitfilter_zero_score(subject_index):
     assert len(hits) == 0
 
 
+def test_lazyanalysisresult(subject_index):
+    lar = LazyAnalysisResult(lambda: generate_hits(10, subject_index))
+    assert lar._object is None
+    assert len(lar) == 10
+    assert len(lar.hits) == 10
+    assert lar.vector is not None
+    assert lar[0] is not None
+    filtered = lar.filter(limit=5, threshold=0.0)
+    assert len(filtered) == 5
+    assert lar._object is not None
+
+
 def test_analysishits_vector(document_corpus):
     subjects = SubjectIndex(document_corpus)
     hits = ListAnalysisResult(