Merge subjects in CombinedCorpus instead of concatenating them

annif/corpus/combine.py

@@ -1,7 +1,7 @@
 """Class for combining multiple corpora so they behave like a single corpus"""
 
 import itertools
-from .types import DocumentCorpus, SubjectCorpus
+from .types import DocumentCorpus, SubjectCorpus, Subject
 
 
 class CombinedCorpus(SubjectCorpus, DocumentCorpus):
@@ -13,8 +13,16 @@ def __init__(self, corpora):
 
     @property
     def subjects(self):
-        return itertools.chain.from_iterable(
-            [corpus.subjects for corpus in self._corpora])
+        for source_subjects in zip(
+                *[corpus.subjects for corpus in self._corpora]):
+            uri = None
+            label = None
+            texts = []
+            for subject in source_subjects:
+                uri = subject.uri
+                label = subject.label
+                texts.append(subject.text)
+            yield Subject(uri=uri, label=label, text=" ".join(texts))
 
     @property
     def documents(self):

tests/test_corpus.py

@@ -250,3 +250,19 @@ def test_docfile_is_empty(tmpdir):
     empty_file = tmpdir.ensure('empty.tsv')
     docs = annif.corpus.DocumentFile(str(empty_file))
     assert docs.is_empty()
+
+
+def test_combinedcorpus(tmpdir, subject_index):
+    docfile = tmpdir.join('documents.tsv')
+    docfile.write("""Läntinen\t<http://www.yso.fi/onto/yso/p2557>
+        Oulunlinnan\t<http://www.yso.fi/onto/yso/p7346>
+        Harald Hirmuinen\t<http://www.yso.fi/onto/yso/p6479>""")
+
+    corpus1 = annif.corpus.DocumentFile(str(docfile))
+    corpus2 = annif.corpus.DocumentFile(str(docfile))
+
+    combined = annif.corpus.CombinedCorpus([corpus1, corpus2])
+    combined.set_subject_index(subject_index)
+
+    assert len(list(combined.documents)) == 6
+    assert len(list(combined.subjects)) == len(list(corpus1.subjects))