#323 - Extended referencedCriteria and Valueset Search

- replace query construction to match the one in the terminology service
medizininformatik-initiative · Aug 19, 2024 · 1c7f18e · 1c7f18e
1 parent c420e6c
commit 1c7f18e
Show file tree

Hide file tree

Showing 5 changed files with 88 additions and 90 deletions.
diff --git a/src/main/java/de/numcodex/feasibility_gui_backend/terminology/es/CodeableConceptService.java b/src/main/java/de/numcodex/feasibility_gui_backend/terminology/es/CodeableConceptService.java
@@ -1,5 +1,7 @@
 package de.numcodex.feasibility_gui_backend.terminology.es;
 
+import co.elastic.clients.elasticsearch._types.FieldValue;
+import co.elastic.clients.elasticsearch._types.query_dsl.*;
 import de.numcodex.feasibility_gui_backend.common.api.TermCode;
 import de.numcodex.feasibility_gui_backend.terminology.api.CcSearchResult;
 import de.numcodex.feasibility_gui_backend.terminology.es.model.CodeableConceptDocument;
@@ -8,9 +10,11 @@
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
-import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageRequest;
+import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder;
 import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
+import org.springframework.data.elasticsearch.core.SearchHits;
+import org.springframework.data.util.Pair;
 import org.springframework.lang.Nullable;
 import org.springframework.stereotype.Service;
 
@@ -35,29 +39,59 @@ public CcSearchResult performCodeableConceptSearchWithRepoAndPaging(String keywo
                                                                @Nullable List<String> valueSets,
                                                                @Nullable int pageSize,
                                                                @Nullable int page) {
-    Page<CodeableConceptDocument> searchHitPage;
 
+    List<Pair<String, List<String>>> filterList = new ArrayList<>();
     if (valueSets != null && !valueSets.isEmpty()) {
-      searchHitPage = repo
-          .findByNameOrTermcodeMultiMatch1Filter(keyword,
-              "value_sets",
-              valueSets,
-              PageRequest.of(page, pageSize));
-    } else {
-      searchHitPage = repo
-          .findByNameOrTermcodeMultiMatch0Filters(keyword,
-              PageRequest.of(page, pageSize));
+      filterList.add(Pair.of("value_sets", valueSets));
     }
+
+    var searchHitPage = findByCodeOrDisplay(keyword, filterList, PageRequest.of(page, pageSize));
     List<TermCode> codeableConceptEntries = new ArrayList<>();
 
-    searchHitPage.getContent().forEach(hit -> codeableConceptEntries.add(hit.termCode()));
+    searchHitPage.getSearchHits().forEach(hit -> codeableConceptEntries.add(hit.getContent().termCode()));
     return CcSearchResult.builder()
-        .totalHits(searchHitPage.getTotalElements())
+        .totalHits(searchHitPage.getTotalHits())
         .results(codeableConceptEntries)
         .build();
   }
 
   public TermCode getSearchResultEntryByCode(String code) {
     return repo.findById(code).orElseThrow(OntologyItemNotFoundException::new).termCode();
   }
+
+  private SearchHits<CodeableConceptDocument> findByCodeOrDisplay(String keyword,
+                                                                  List<Pair<String,List<String>>> filterList,
+                                                                  PageRequest pageRequest) {
+    List<Query> filterTerms = new ArrayList<>();
+
+    if (!filterList.isEmpty()) {
+      var fieldValues = new ArrayList<FieldValue>();
+      filterList.forEach(f -> {
+        f.getSecond().forEach(s -> {
+          fieldValues.add(new FieldValue.Builder().stringValue(s).build());
+        });
+        filterTerms.add(new TermsQuery.Builder()
+            .field(f.getFirst())
+            .terms(new TermsQueryField.Builder().value(fieldValues).build())
+            .build()._toQuery());
+      });
+    }
+
+    var mmQuery = new MultiMatchQuery.Builder()
+        .query(keyword)
+        .fields(List.of("termcode.display", "termcode.code^2"))
+        .build();
+
+    var boolQuery = new BoolQuery.Builder()
+        .must(List.of(mmQuery._toQuery()))
+        .filter(filterTerms.isEmpty() ? List.of() : filterTerms)
+        .build();
+
+    var query = new NativeQueryBuilder()
+        .withQuery(boolQuery._toQuery())
+        .withPageable(pageRequest)
+        .build();
+
+    return operations.search(query, CodeableConceptDocument.class);
+  }
 }
diff --git a/...mcodex/feasibility_gui_backend/terminology/es/repository/CodeableConceptEsRepository.java b/...mcodex/feasibility_gui_backend/terminology/es/repository/CodeableConceptEsRepository.java
@@ -2,63 +2,8 @@
 
 import de.numcodex.feasibility_gui_backend.terminology.es.model.CodeableConceptDocument;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
-import org.springframework.data.domain.Page;
-import org.springframework.data.domain.Pageable;
-import org.springframework.data.elasticsearch.annotations.Query;
 import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
 
-import java.util.List;
-
 @ConditionalOnExpression("${app.elastic.enabled}")
 public interface CodeableConceptEsRepository extends ElasticsearchRepository<CodeableConceptDocument, String> {
-  @Query("""
-      {
-        "bool": {
-          "must": [
-            {
-              "multi_match": {
-                "query": "?0",
-                "fields": [
-                  "termcode.display",
-                  "termcode.code^2"
-                ]
-              }
-            }
-          ]
-        }
-      }
-      """
-  )
-  Page<CodeableConceptDocument> findByNameOrTermcodeMultiMatch0Filters(String searchterm,
-                                                                       Pageable pageable);
-
-  @Query("""
-      {
-        "bool": {
-          "must": [
-            {
-              "multi_match": {
-                "query": "?0",
-                "fields": [
-                  "termcode.display",
-                  "termcode.code^2"
-                ]
-              }
-            }
-          ],
-          "filter": {
-            "bool" : {
-              "must" : [
-                {"terms" : { "?1": ?2 } }
-              ]
-            }
-          }
-        }
-      }
-      """
-  )
-  Page<CodeableConceptDocument> findByNameOrTermcodeMultiMatch1Filter(String searchterm,
-                                                                      String filterKey,
-                                                                      List<String> filterValues,
-                                                                      Pageable pageable);
 }
diff --git a/...t/java/de/numcodex/feasibility_gui_backend/terminology/es/CodeableConceptServiceTest.java b/...t/java/de/numcodex/feasibility_gui_backend/terminology/es/CodeableConceptServiceTest.java
@@ -11,17 +11,17 @@
 import org.mockito.Mock;
 import org.mockito.Mockito;
 import org.mockito.junit.jupiter.MockitoExtension;
-import org.springframework.data.domain.Page;
-import org.springframework.data.domain.PageImpl;
-import org.springframework.data.domain.Pageable;
-import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
+import org.springframework.data.elasticsearch.client.elc.NativeQuery;
+import org.springframework.data.elasticsearch.core.*;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
+import java.util.UUID;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.*;
 import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyList;
 import static org.mockito.Mockito.doReturn;
 
 @ExtendWith(MockitoExtension.class)
@@ -49,41 +49,42 @@ void setUp() {
 
   @Test
   void testPerformCodeableConceptSearchWithRepoAndPaging_succeedsWithoutValueSetFilter() {
-    Page<CodeableConceptDocument> dummyCodeableConceptDocumentPage = createDummyCodeableConceptDocumentPage();
-    doReturn(dummyCodeableConceptDocumentPage).when(repository).findByNameOrTermcodeMultiMatch0Filters(any(String.class), any(Pageable.class));
+    SearchHits<CodeableConceptDocument> dummySearchHitsPage = createDummySearchHitsPage(5);
+    doReturn(dummySearchHitsPage).when(operations).search(any(NativeQuery.class), any(Class.class));
 
     var result = assertDoesNotThrow(() -> codeableConceptService.performCodeableConceptSearchWithRepoAndPaging("foo", List.of(), 20, 0));
 
     assertNotNull(result);
-    assertEquals(dummyCodeableConceptDocumentPage.getTotalElements(), result.getTotalHits());
-    assertEquals(dummyCodeableConceptDocumentPage.get().toList().get(0).termCode(), result.getResults().get(0));
+    assertEquals(dummySearchHitsPage.getTotalHits(), result.getTotalHits());
+    assertEquals(dummySearchHitsPage.getSearchHits().get(0).getContent().termCode(), result.getResults().get(0));
   }
 
   @Test
   void testPerformCodeableConceptSearchWithRepoAndPaging_succeedsWithValueSetFilter() {
-    Page<CodeableConceptDocument> dummyCodeableConceptDocumentPage = createDummyCodeableConceptDocumentPage();
-    doReturn(dummyCodeableConceptDocumentPage).when(repository).findByNameOrTermcodeMultiMatch1Filter(any(String.class), any(String.class), anyList(), any(Pageable.class));
+    SearchHits<CodeableConceptDocument> dummySearchHitsPage = createDummySearchHitsPage(5);
+    doReturn(dummySearchHitsPage).when(operations).search(any(NativeQuery.class), any(Class.class));
 
     var result = assertDoesNotThrow(() -> codeableConceptService.performCodeableConceptSearchWithRepoAndPaging("foo", List.of("bar"), 20, 0));
 
     assertNotNull(result);
-    assertEquals(dummyCodeableConceptDocumentPage.getTotalElements(), result.getTotalHits());
-    assertEquals(dummyCodeableConceptDocumentPage.get().toList().get(0).termCode(), result.getResults().get(0));
+    assertEquals(dummySearchHitsPage.getTotalHits(), result.getTotalHits());
+    assertEquals(dummySearchHitsPage.getSearchHits().get(0).getContent().termCode(), result.getResults().get(0));
   }
 
   @Test
   void testPerformCodeableConceptSearchWithRepoAndPaging_succeedsWithEmptyResult() {
-    doReturn(new PageImpl<CodeableConceptDocument>(List.of())).when(repository).findByNameOrTermcodeMultiMatch0Filters(any(String.class), any(Pageable.class));
+    SearchHits<CodeableConceptDocument> dummySearchHitsPage = createDummySearchHitsPage(0);
+    doReturn(dummySearchHitsPage).when(operations).search(any(NativeQuery.class), any(Class.class));
 
     var result = assertDoesNotThrow(() -> codeableConceptService.performCodeableConceptSearchWithRepoAndPaging("foo", List.of(), 20, 0));
 
     assertNotNull(result);
-    assertEquals(0, result.getTotalHits());
+    assertThat(result.getTotalHits()).isZero();
   }
 
   @Test
   void testGetSearchResultEntryByCode_succeeds() {
-    CodeableConceptDocument dummyCodeableConceptDocument = createDummyCodeableConceptDocument();
+    CodeableConceptDocument dummyCodeableConceptDocument = createDummyCodeableConceptDocument("1");
     doReturn(Optional.of(dummyCodeableConceptDocument)).when(repository).findById(any());
 
     var result = assertDoesNotThrow(() -> codeableConceptService.getSearchResultEntryByCode("1"));
@@ -99,13 +100,32 @@ void testGetSearchResultEntryByCode_throwsOnNotFound() {
     assertThrows(OntologyItemNotFoundException.class, () -> codeableConceptService.getSearchResultEntryByCode("foo"));
   }
 
-  private Page<CodeableConceptDocument> createDummyCodeableConceptDocumentPage() {
-    return new PageImpl<>(List.of(createDummyCodeableConceptDocument()));
+  private SearchHits<CodeableConceptDocument> createDummySearchHitsPage(int totalHits) {
+    var searchHitsList = new ArrayList<SearchHit<CodeableConceptDocument>>();
+
+    for (int i = 0; i < totalHits; ++i) {
+      searchHitsList.add(
+          new SearchHit<>(
+              null,
+              null,
+              null,
+              10.0F,
+              null,
+              null,
+              null,
+              null,
+              null,
+              null,
+              createDummyCodeableConceptDocument(UUID.randomUUID().toString())
+          )
+      );
+    }
+    return new SearchHitsImpl<>(totalHits, TotalHitsRelation.OFF, 10.0F, null, null, searchHitsList, null, null, null);
   }
 
-  private CodeableConceptDocument createDummyCodeableConceptDocument() {
+  private CodeableConceptDocument createDummyCodeableConceptDocument(String id) {
     return CodeableConceptDocument.builder()
-        .id("1")
+        .id(id)
         .termCode(createDummyTermcode())
         .valueSets(List.of())
         .build();

diff --git a/src/test/java/de/numcodex/feasibility_gui_backend/terminology/es/TerminologyEsServiceIT.java b/src/test/java/de/numcodex/feasibility_gui_backend/terminology/es/TerminologyEsServiceIT.java
@@ -51,7 +51,7 @@ public class TerminologyEsServiceIT {
   private TerminologyEsService terminologyEsService;
 
   @Container
-  public static ElasticsearchContainer elastic = new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:8.14.1")
+  public static ElasticsearchContainer elastic = new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:8.15.0")
       .withEnv("discovery.type", "single-node")
       .withEnv("xpack.security.enabled", "false")
       .withExposedPorts(9200)

diff --git a/...est/java/de/numcodex/feasibility_gui_backend/terminology/es/TerminologyEsServiceTest.java b/...est/java/de/numcodex/feasibility_gui_backend/terminology/es/TerminologyEsServiceTest.java
@@ -267,7 +267,6 @@ private List<TermFilter> createTermFilterList(String[] values) {
   }
 
   private SearchHits<OntologyListItemDocument> createDummySearchHitsPage(int totalHits) {
-    var dummyItemList = new ArrayList<OntologyListItemDocument>();
     var searchHitsList = new ArrayList<SearchHit<OntologyListItemDocument>>();
 
     for (int i = 0; i < totalHits; ++i) {