#323 - Extended referencedCriteria and Valueset Search (#324)

* #323 - Extended referencedCriteria and Valueset Search

- Add new rest controller and services to search in a different elastic search index for Codeable Concepts
- tests and openapi docs missing
- fix inconsistent variable naming in new api parts (no longer using camelCase)
- add codeable_concept path to WebSecurityConfig

pom.xml

@@ -273,16 +273,8 @@
     </dependency>
 
     <dependency>
-      <groupId>org.testcontainers</groupId>
-      <artifactId>testcontainers</artifactId>
-      <version>1.19.8</version>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.testcontainers</groupId>
-      <artifactId>elasticsearch</artifactId>
-      <version>1.19.8</version>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-testcontainers</artifactId>
       <scope>test</scope>
     </dependency>
 
@@ -304,14 +296,18 @@
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>junit-jupiter</artifactId>
-      <version>1.19.8</version>
       <scope>test</scope>
     </dependency>
 
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>postgresql</artifactId>
-      <version>1.19.8</version>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.testcontainers</groupId>
+      <artifactId>elasticsearch</artifactId>
       <scope>test</scope>
     </dependency>
 

src/main/java/de/numcodex/feasibility_gui_backend/config/WebSecurityConfig.java

@@ -49,6 +49,7 @@ public class WebSecurityConfig {
   public static final String PATH_TERMINOLOGY = "/terminology";
   public static final String PATH_TEMPLATE = "/template";
   public static final String PATH_DSE = "/dse";
+  public static final String PATH_CODEABLE_CONCEPT = "/codeable-concept";
   public static final String PATH_SWAGGER_UI = "/swagger-ui/**";
   public static final String PATH_SWAGGER_CONFIG = "/v3/api-docs/**";
   @Value("${app.keycloakAllowedRole}")
@@ -109,6 +110,7 @@ public SecurityFilterChain apiFilterChain(
                     .requestMatchers(new MvcRequestMatcher(introspector, PATH_API + PATH_QUERY + PATH_TEMPLATE + "/*")).hasAuthority(keycloakAllowedRole)
                     .requestMatchers(new MvcRequestMatcher(introspector, PATH_API + "/**")).hasAnyAuthority(keycloakAdminRole, keycloakAllowedRole)
                     .requestMatchers(new MvcRequestMatcher(introspector, PATH_API + PATH_DSE + "/**")).hasAnyAuthority(keycloakAdminRole, keycloakAllowedRole)
+                    .requestMatchers(new MvcRequestMatcher(introspector, PATH_API + PATH_CODEABLE_CONCEPT + "/**")).hasAnyAuthority(keycloakAdminRole, keycloakAllowedRole)
                     .requestMatchers(new MvcRequestMatcher(introspector, PATH_ACTUATOR_HEALTH)).anonymous()
                     .requestMatchers(new MvcRequestMatcher(introspector, PATH_SWAGGER_UI)).anonymous()
                     .requestMatchers(new MvcRequestMatcher(introspector, PATH_SWAGGER_CONFIG)).anonymous()

src/main/java/de/numcodex/feasibility_gui_backend/terminology/api/CcSearchResult.java

@@ -0,0 +1,14 @@
+package de.numcodex.feasibility_gui_backend.terminology.api;
+
+import de.numcodex.feasibility_gui_backend.common.api.TermCode;
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+@Builder
+public class CcSearchResult {
+  private long totalHits;
+  private List<TermCode> results;
+}

src/main/java/de/numcodex/feasibility_gui_backend/terminology/es/CodeableConceptService.java

@@ -0,0 +1,97 @@
+package de.numcodex.feasibility_gui_backend.terminology.es;
+
+import co.elastic.clients.elasticsearch._types.FieldValue;
+import co.elastic.clients.elasticsearch._types.query_dsl.*;
+import de.numcodex.feasibility_gui_backend.common.api.TermCode;
+import de.numcodex.feasibility_gui_backend.terminology.api.CcSearchResult;
+import de.numcodex.feasibility_gui_backend.terminology.es.model.CodeableConceptDocument;
+import de.numcodex.feasibility_gui_backend.terminology.es.repository.CodeableConceptEsRepository;
+import de.numcodex.feasibility_gui_backend.terminology.es.repository.OntologyItemNotFoundException;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder;
+import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
+import org.springframework.data.elasticsearch.core.SearchHits;
+import org.springframework.data.util.Pair;
+import org.springframework.lang.Nullable;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Service
+@Slf4j
+@ConditionalOnExpression("${app.elastic.enabled}")
+public class CodeableConceptService {
+  private ElasticsearchOperations operations;
+
+  private CodeableConceptEsRepository repo;
+
+  @Autowired
+  public CodeableConceptService(ElasticsearchOperations operations, CodeableConceptEsRepository repo) {
+    this.operations = operations;
+    this.repo = repo;
+  }
+
+  public CcSearchResult performCodeableConceptSearchWithRepoAndPaging(String keyword,
+                                                               @Nullable List<String> valueSets,
+                                                               @Nullable int pageSize,
+                                                               @Nullable int page) {
+
+    List<Pair<String, List<String>>> filterList = new ArrayList<>();
+    if (valueSets != null && !valueSets.isEmpty()) {
+      filterList.add(Pair.of("value_sets", valueSets));
+    }
+
+    var searchHitPage = findByCodeOrDisplay(keyword, filterList, PageRequest.of(page, pageSize));
+    List<TermCode> codeableConceptEntries = new ArrayList<>();
+
+    searchHitPage.getSearchHits().forEach(hit -> codeableConceptEntries.add(hit.getContent().termCode()));
+    return CcSearchResult.builder()
+        .totalHits(searchHitPage.getTotalHits())
+        .results(codeableConceptEntries)
+        .build();
+  }
+
+  public TermCode getSearchResultEntryByCode(String code) {
+    return repo.findById(code).orElseThrow(OntologyItemNotFoundException::new).termCode();
+  }
+
+  private SearchHits<CodeableConceptDocument> findByCodeOrDisplay(String keyword,
+                                                                  List<Pair<String,List<String>>> filterList,
+                                                                  PageRequest pageRequest) {
+    List<Query> filterTerms = new ArrayList<>();
+
+    if (!filterList.isEmpty()) {
+      var fieldValues = new ArrayList<FieldValue>();
+      filterList.forEach(f -> {
+        f.getSecond().forEach(s -> {
+          fieldValues.add(new FieldValue.Builder().stringValue(s).build());
+        });
+        filterTerms.add(new TermsQuery.Builder()
+            .field(f.getFirst())
+            .terms(new TermsQueryField.Builder().value(fieldValues).build())
+            .build()._toQuery());
+      });
+    }
+
+    var mmQuery = new MultiMatchQuery.Builder()
+        .query(keyword)
+        .fields(List.of("termcode.display", "termcode.code^2"))
+        .build();
+
+    var boolQuery = new BoolQuery.Builder()
+        .must(List.of(mmQuery._toQuery()))
+        .filter(filterTerms.isEmpty() ? List.of() : filterTerms)
+        .build();
+
+    var query = new NativeQueryBuilder()
+        .withQuery(boolQuery._toQuery())
+        .withPageable(pageRequest)
+        .build();
+
+    return operations.search(query, CodeableConceptDocument.class);
+  }
+}

src/main/java/de/numcodex/feasibility_gui_backend/terminology/es/TerminologyEsService.java

@@ -71,15 +71,19 @@ public List<TermFilter> getAvailableFilters() {
   }
 
   public EsSearchResult performOntologySearchWithPaging(String keyword,
-                                                               @Nullable List<String> context,
-                                                               @Nullable List<String> kdsModule,
-                                                               @Nullable List<String> terminology,
-                                                               @Nullable boolean availability,
-                                                               @Nullable int pageSize,
-                                                               @Nullable int page) {
+                                                        @Nullable List<String> criteriaSets,
+                                                        @Nullable List<String> context,
+                                                        @Nullable List<String> kdsModule,
+                                                        @Nullable List<String> terminology,
+                                                        @Nullable boolean availability,
+                                                        @Nullable int pageSize,
+                                                        @Nullable int page) {
 
 
     List<Pair<String, List<String>>> filterList = new ArrayList<>();
+    if (criteriaSets != null) {
+      filterList.add(Pair.of("criteria_sets", criteriaSets));
+    }
     if (context != null) {
       filterList.add(Pair.of("context.code", context));
     }

src/main/java/de/numcodex/feasibility_gui_backend/terminology/es/model/CodeableConceptDocument.java

@@ -0,0 +1,21 @@
+package de.numcodex.feasibility_gui_backend.terminology.es.model;
+
+import de.numcodex.feasibility_gui_backend.common.api.TermCode;
+import jakarta.persistence.Id;
+import lombok.Builder;
+import org.springframework.data.elasticsearch.annotations.Document;
+import org.springframework.data.elasticsearch.annotations.Field;
+import org.springframework.data.elasticsearch.annotations.FieldType;
+
+import java.util.List;
+
+@Builder
+@Document(indexName = "codeable_concept")
+public record CodeableConceptDocument(
+    @Id String id,
+    @Field(type = FieldType.Nested, includeInParent = true, name = "termcode")
+    TermCode termCode,
+    @Field(type = FieldType.Nested, includeInParent = true, name = "value_sets")
+    List<String> valueSets
+) {
+}

src/main/java/de/numcodex/feasibility_gui_backend/terminology/es/repository/CodeableConceptEsRepository.java

@@ -0,0 +1,9 @@
+package de.numcodex.feasibility_gui_backend.terminology.es.repository;
+
+import de.numcodex.feasibility_gui_backend.terminology.es.model.CodeableConceptDocument;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
+import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
+
+@ConditionalOnExpression("${app.elastic.enabled}")
+public interface CodeableConceptEsRepository extends ElasticsearchRepository<CodeableConceptDocument, String> {
+}

src/main/java/de/numcodex/feasibility_gui_backend/terminology/v3/CodeableConceptEsRestController.java

@@ -0,0 +1,40 @@
+package de.numcodex.feasibility_gui_backend.terminology.v3;
+
+import de.numcodex.feasibility_gui_backend.common.api.TermCode;
+import de.numcodex.feasibility_gui_backend.terminology.api.CcSearchResult;
+import de.numcodex.feasibility_gui_backend.terminology.es.CodeableConceptService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.*;
+
+import java.util.List;
+
+@RestController
+@RequestMapping("api/v3/codeable-concept")
+@ConditionalOnExpression("${app.elastic.enabled}")
+@CrossOrigin
+public class CodeableConceptEsRestController {
+
+  private CodeableConceptService codeableConceptService;
+
+  @Autowired
+  public CodeableConceptEsRestController(CodeableConceptService codeableConceptService) {
+    this.codeableConceptService = codeableConceptService;
+  }
+
+  @GetMapping(value = "/entry/search", produces = MediaType.APPLICATION_JSON_VALUE)
+  public CcSearchResult searchOntologyItemsCriteriaQuery(@RequestParam("searchterm") String keyword,
+                                                         @RequestParam(value = "value-sets", required = false) List<String> valueSets,
+                                                         @RequestParam(value = "page-size", required = false, defaultValue = "20") int pageSize,
+                                                         @RequestParam(value = "page", required = false, defaultValue = "0") int page) {
+
+    return codeableConceptService
+        .performCodeableConceptSearchWithRepoAndPaging(keyword, valueSets, pageSize, page);
+  }
+
+  @GetMapping(value = "/entry/{code}",  produces = MediaType.APPLICATION_JSON_VALUE)
+  public TermCode getCodeableConceptByCode(@PathVariable("code") String code) {
+    return codeableConceptService.getSearchResultEntryByCode(code);
+  }
+}

src/main/java/de/numcodex/feasibility_gui_backend/terminology/v3/TerminologyEsRestController.java

@@ -30,6 +30,7 @@ public List<TermFilter> getAvailableFilters() {
 
   @GetMapping("api/v3/terminology/entry/search")
   public EsSearchResult searchOntologyItemsCriteriaQuery2(@RequestParam("searchterm") String keyword,
+                                                         @RequestParam(value = "criteria-sets", required = false) List<String> criteriaSets,
                                                          @RequestParam(value = "contexts", required = false) List<String> contexts,
                                                          @RequestParam(value = "kds-modules", required = false) List<String> kdsModules,
                                                          @RequestParam(value = "terminologies", required = false) List<String> terminologies,
@@ -39,7 +40,7 @@ public EsSearchResult searchOntologyItemsCriteriaQuery2(@RequestParam("searchter
 
 
     return terminologyEsService
-        .performOntologySearchWithPaging(keyword, contexts, kdsModules, terminologies, availability, pageSize, page);
+        .performOntologySearchWithPaging(keyword, criteriaSets, contexts, kdsModules, terminologies, availability, pageSize, page);
   }
 
   @GetMapping("api/v3/terminology/entry/{hash}/relations")