diff --git a/cis/com.b2international.snowowl.snomed.cis/META-INF/MANIFEST.MF b/cis/com.b2international.snowowl.snomed.cis/META-INF/MANIFEST.MF index 4fa95440db9..a012748f36b 100644 --- a/cis/com.b2international.snowowl.snomed.cis/META-INF/MANIFEST.MF +++ b/cis/com.b2international.snowowl.snomed.cis/META-INF/MANIFEST.MF @@ -8,9 +8,9 @@ Automatic-Module-Name: com.b2international.snowowl.snomed.cis Bundle-RequiredExecutionEnvironment: JavaSE-17 Bundle-ActivationPolicy: lazy Require-Bundle: org.eclipse.core.runtime;bundle-version="3.9.0", - org.apache.httpcomponents.httpasyncclient;bundle-version="[4.1.4,4.1.5)", - org.apache.httpcomponents.httpclient;bundle-version="[4.5.3,4.5.4)", - org.apache.httpcomponents.httpcore;bundle-version="[4.4.10,4.4.11)", + org.apache.httpcomponents.httpasyncclient;bundle-version="[4.1.4,4.2.0)", + org.apache.httpcomponents.httpclient;bundle-version="[4.5.10,4.6.0)", + org.apache.httpcomponents.httpcore;bundle-version="[4.4.12,4.5.0)", com.b2international.snowowl.core Import-Package: org.slf4j;version="1.7.25" Export-Package: com.b2international.snowowl.snomed.cis, diff --git a/commons/com.b2international.index.es8/.classpath b/commons/com.b2international.index.es8/.classpath new file mode 100644 index 00000000000..c53d91ef031 --- /dev/null +++ b/commons/com.b2international.index.es8/.classpath @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/commons/com.b2international.index.es8/.project b/commons/com.b2international.index.es8/.project new file mode 100644 index 00000000000..1d9fda409f6 --- /dev/null +++ b/commons/com.b2international.index.es8/.project @@ -0,0 +1,34 @@ + + + com.b2international.index.es8 + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.pde.ManifestBuilder + + + + + org.eclipse.pde.SchemaBuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.m2e.core.maven2Nature + org.eclipse.pde.PluginNature + org.eclipse.jdt.core.javanature + + diff --git a/commons/com.b2international.index.es8/META-INF/MANIFEST.MF b/commons/com.b2international.index.es8/META-INF/MANIFEST.MF new file mode 100644 index 00000000000..b2fa9b55728 --- /dev/null +++ b/commons/com.b2international.index.es8/META-INF/MANIFEST.MF @@ -0,0 +1,207 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Elasticsearch 8 Index Client Module +Bundle-SymbolicName: com.b2international.index.es8;singleton:=true +Bundle-Version: 8.4.1.qualifier +Bundle-Vendor: B2i Healthcare +Automatic-Module-Name: com.b2international.index.es8 +Bundle-RequiredExecutionEnvironment: JavaSE-17 +Bundle-ActivationPolicy: lazy +Require-Bundle: org.eclipse.core.runtime;bundle-version="3.9.0", + org.apache.commons.codec;bundle-version="1.11.0", + com.b2international.commons;visibility:=reexport, + org.apache.httpcomponents.httpasyncclient;bundle-version="[4.1.4,4.2.0)", + org.apache.httpcomponents.httpclient;bundle-version="[4.5.10,4.6.0)", + org.apache.httpcomponents.httpcore;bundle-version="[4.4.12,4.5.0)" +Bundle-ClassPath: ., + lib/elasticsearch-java-8.3.2.jar, + lib/elasticsearch-rest-client-8.3.2.jar, + lib/jakarta.json-api-2.0.1.jar, + lib/jsr305-3.0.2.jar, + lib/parsson-1.0.0.jar +Export-Package: co.elastic.clients, + co.elastic.clients.elasticsearch, + co.elastic.clients.elasticsearch._types, + co.elastic.clients.elasticsearch._types.aggregations, + co.elastic.clients.elasticsearch._types.analysis, + co.elastic.clients.elasticsearch._types.mapping, + co.elastic.clients.elasticsearch._types.query_dsl, + co.elastic.clients.elasticsearch.async_search, + co.elastic.clients.elasticsearch.async_search.status, + co.elastic.clients.elasticsearch.autoscaling, + co.elastic.clients.elasticsearch.autoscaling.get_autoscaling_capacity, + co.elastic.clients.elasticsearch.cat, + co.elastic.clients.elasticsearch.cat.aliases, + co.elastic.clients.elasticsearch.cat.allocation, + co.elastic.clients.elasticsearch.cat.component_templates, + co.elastic.clients.elasticsearch.cat.count, + co.elastic.clients.elasticsearch.cat.fielddata, + co.elastic.clients.elasticsearch.cat.health, + co.elastic.clients.elasticsearch.cat.help, + co.elastic.clients.elasticsearch.cat.indices, + co.elastic.clients.elasticsearch.cat.master, + co.elastic.clients.elasticsearch.cat.ml_data_frame_analytics, + co.elastic.clients.elasticsearch.cat.ml_datafeeds, + co.elastic.clients.elasticsearch.cat.ml_jobs, + co.elastic.clients.elasticsearch.cat.ml_trained_models, + co.elastic.clients.elasticsearch.cat.nodeattrs, + co.elastic.clients.elasticsearch.cat.nodes, + co.elastic.clients.elasticsearch.cat.pending_tasks, + co.elastic.clients.elasticsearch.cat.plugins, + co.elastic.clients.elasticsearch.cat.recovery, + co.elastic.clients.elasticsearch.cat.repositories, + co.elastic.clients.elasticsearch.cat.segments, + co.elastic.clients.elasticsearch.cat.shards, + co.elastic.clients.elasticsearch.cat.snapshots, + co.elastic.clients.elasticsearch.cat.tasks, + co.elastic.clients.elasticsearch.cat.templates, + co.elastic.clients.elasticsearch.cat.thread_pool, + co.elastic.clients.elasticsearch.cat.transforms, + co.elastic.clients.elasticsearch.ccr, + co.elastic.clients.elasticsearch.ccr.follow_info, + co.elastic.clients.elasticsearch.ccr.get_auto_follow_pattern, + co.elastic.clients.elasticsearch.ccr.stats, + co.elastic.clients.elasticsearch.cluster, + co.elastic.clients.elasticsearch.cluster.allocation_explain, + co.elastic.clients.elasticsearch.cluster.health, + co.elastic.clients.elasticsearch.cluster.pending_tasks, + co.elastic.clients.elasticsearch.cluster.remote_info, + co.elastic.clients.elasticsearch.cluster.reroute, + co.elastic.clients.elasticsearch.cluster.stats, + co.elastic.clients.elasticsearch.core, + co.elastic.clients.elasticsearch.core.bulk, + co.elastic.clients.elasticsearch.core.explain, + co.elastic.clients.elasticsearch.core.field_caps, + co.elastic.clients.elasticsearch.core.get, + co.elastic.clients.elasticsearch.core.get_script_context, + co.elastic.clients.elasticsearch.core.get_script_languages, + co.elastic.clients.elasticsearch.core.knn_search, + co.elastic.clients.elasticsearch.core.mget, + co.elastic.clients.elasticsearch.core.msearch, + co.elastic.clients.elasticsearch.core.msearch_template, + co.elastic.clients.elasticsearch.core.mtermvectors, + co.elastic.clients.elasticsearch.core.rank_eval, + co.elastic.clients.elasticsearch.core.reindex, + co.elastic.clients.elasticsearch.core.reindex_rethrottle, + co.elastic.clients.elasticsearch.core.scripts_painless_execute, + co.elastic.clients.elasticsearch.core.search, + co.elastic.clients.elasticsearch.core.search_shards, + co.elastic.clients.elasticsearch.core.termvectors, + co.elastic.clients.elasticsearch.core.update, + co.elastic.clients.elasticsearch.core.update_by_query_rethrottle, + co.elastic.clients.elasticsearch.dangling_indices, + co.elastic.clients.elasticsearch.dangling_indices.list_dangling_indices, + co.elastic.clients.elasticsearch.enrich, + co.elastic.clients.elasticsearch.enrich.execute_policy, + co.elastic.clients.elasticsearch.enrich.stats, + co.elastic.clients.elasticsearch.eql, + co.elastic.clients.elasticsearch.eql.search, + co.elastic.clients.elasticsearch.features, + co.elastic.clients.elasticsearch.fleet, + co.elastic.clients.elasticsearch.graph, + co.elastic.clients.elasticsearch.ilm, + co.elastic.clients.elasticsearch.ilm.explain_lifecycle, + co.elastic.clients.elasticsearch.ilm.get_lifecycle, + co.elastic.clients.elasticsearch.ilm.move_to_step, + co.elastic.clients.elasticsearch.indices, + co.elastic.clients.elasticsearch.indices.add_block, + co.elastic.clients.elasticsearch.indices.analyze, + co.elastic.clients.elasticsearch.indices.close, + co.elastic.clients.elasticsearch.indices.data_streams_stats, + co.elastic.clients.elasticsearch.indices.field_usage_stats, + co.elastic.clients.elasticsearch.indices.get, + co.elastic.clients.elasticsearch.indices.get_alias, + co.elastic.clients.elasticsearch.indices.get_field_mapping, + co.elastic.clients.elasticsearch.indices.get_index_template, + co.elastic.clients.elasticsearch.indices.get_mapping, + co.elastic.clients.elasticsearch.indices.modify_data_stream, + co.elastic.clients.elasticsearch.indices.put_index_template, + co.elastic.clients.elasticsearch.indices.recovery, + co.elastic.clients.elasticsearch.indices.reload_search_analyzers, + co.elastic.clients.elasticsearch.indices.resolve_index, + co.elastic.clients.elasticsearch.indices.rollover, + co.elastic.clients.elasticsearch.indices.segments, + co.elastic.clients.elasticsearch.indices.shard_stores, + co.elastic.clients.elasticsearch.indices.simulate_template, + co.elastic.clients.elasticsearch.indices.stats, + co.elastic.clients.elasticsearch.indices.update_aliases, + co.elastic.clients.elasticsearch.indices.validate_query, + co.elastic.clients.elasticsearch.ingest, + co.elastic.clients.elasticsearch.ingest.geo_ip_stats, + co.elastic.clients.elasticsearch.ingest.simulate, + co.elastic.clients.elasticsearch.license, + co.elastic.clients.elasticsearch.license.get, + co.elastic.clients.elasticsearch.license.post, + co.elastic.clients.elasticsearch.logstash, + co.elastic.clients.elasticsearch.migration, + co.elastic.clients.elasticsearch.migration.deprecations, + co.elastic.clients.elasticsearch.migration.get_feature_upgrade_status, + co.elastic.clients.elasticsearch.migration.post_feature_upgrade, + co.elastic.clients.elasticsearch.ml, + co.elastic.clients.elasticsearch.ml.evaluate_data_frame, + co.elastic.clients.elasticsearch.ml.get_calendars, + co.elastic.clients.elasticsearch.ml.get_memory_stats, + co.elastic.clients.elasticsearch.ml.info, + co.elastic.clients.elasticsearch.ml.preview_data_frame_analytics, + co.elastic.clients.elasticsearch.ml.put_trained_model, + co.elastic.clients.elasticsearch.monitoring, + co.elastic.clients.elasticsearch.nodes, + co.elastic.clients.elasticsearch.nodes.clear_repositories_metering_archive, + co.elastic.clients.elasticsearch.nodes.get_repositories_metering_info, + co.elastic.clients.elasticsearch.nodes.hot_threads, + co.elastic.clients.elasticsearch.nodes.info, + co.elastic.clients.elasticsearch.nodes.reload_secure_settings, + co.elastic.clients.elasticsearch.nodes.stats, + co.elastic.clients.elasticsearch.nodes.usage, + co.elastic.clients.elasticsearch.rollup, + co.elastic.clients.elasticsearch.rollup.get_jobs, + co.elastic.clients.elasticsearch.rollup.get_rollup_caps, + co.elastic.clients.elasticsearch.rollup.get_rollup_index_caps, + co.elastic.clients.elasticsearch.searchable_snapshots, + co.elastic.clients.elasticsearch.searchable_snapshots.cache_stats, + co.elastic.clients.elasticsearch.searchable_snapshots.mount, + co.elastic.clients.elasticsearch.security, + co.elastic.clients.elasticsearch.security.authenticate, + co.elastic.clients.elasticsearch.security.create_api_key, + co.elastic.clients.elasticsearch.security.create_service_token, + co.elastic.clients.elasticsearch.security.delete_privileges, + co.elastic.clients.elasticsearch.security.enroll_kibana, + co.elastic.clients.elasticsearch.security.get_role, + co.elastic.clients.elasticsearch.security.get_service_accounts, + co.elastic.clients.elasticsearch.security.get_service_credentials, + co.elastic.clients.elasticsearch.security.get_token, + co.elastic.clients.elasticsearch.security.grant_api_key, + co.elastic.clients.elasticsearch.security.has_privileges, + co.elastic.clients.elasticsearch.security.put_privileges, + co.elastic.clients.elasticsearch.security.suggest_user_profiles, + co.elastic.clients.elasticsearch.shutdown, + co.elastic.clients.elasticsearch.shutdown.get_node, + co.elastic.clients.elasticsearch.slm, + co.elastic.clients.elasticsearch.snapshot, + co.elastic.clients.elasticsearch.snapshot.cleanup_repository, + co.elastic.clients.elasticsearch.snapshot.get, + co.elastic.clients.elasticsearch.snapshot.restore, + co.elastic.clients.elasticsearch.snapshot.verify_repository, + co.elastic.clients.elasticsearch.sql, + co.elastic.clients.elasticsearch.ssl, + co.elastic.clients.elasticsearch.ssl.certificates, + co.elastic.clients.elasticsearch.tasks, + co.elastic.clients.elasticsearch.transform, + co.elastic.clients.elasticsearch.transform.get_transform, + co.elastic.clients.elasticsearch.transform.get_transform_stats, + co.elastic.clients.elasticsearch.watcher, + co.elastic.clients.elasticsearch.watcher.execute_watch, + co.elastic.clients.elasticsearch.watcher.stats, + co.elastic.clients.elasticsearch.xpack, + co.elastic.clients.elasticsearch.xpack.info, + co.elastic.clients.elasticsearch.xpack.usage, + co.elastic.clients.json, + co.elastic.clients.json.jackson, + co.elastic.clients.json.jsonb, + co.elastic.clients.transport, + co.elastic.clients.transport.endpoints, + co.elastic.clients.transport.rest_client, + co.elastic.clients.util, + com.b2international.index.es8 +Import-Package: org.apache.commons.logging;version="1.2.0", + org.slf4j;version="1.7.2" diff --git a/commons/com.b2international.index.es8/build.properties b/commons/com.b2international.index.es8/build.properties new file mode 100644 index 00000000000..6594de1acda --- /dev/null +++ b/commons/com.b2international.index.es8/build.properties @@ -0,0 +1,5 @@ +source.. = src/ +output.. = target/classes +bin.includes = META-INF/,\ + .,\ + lib/ diff --git a/commons/com.b2international.index.test.tools/lib/.gitignore b/commons/com.b2international.index.es8/lib/.gitignore similarity index 81% rename from commons/com.b2international.index.test.tools/lib/.gitignore rename to commons/com.b2international.index.es8/lib/.gitignore index 1a5126b1855..95efc4fe3c7 100644 --- a/commons/com.b2international.index.test.tools/lib/.gitignore +++ b/commons/com.b2international.index.es8/lib/.gitignore @@ -2,4 +2,5 @@ # Ignore everything in this directory * # Except this file -!.gitignore \ No newline at end of file +!.gitignore +!elasticsearch-log4j-*.jar \ No newline at end of file diff --git a/commons/com.b2international.index.es8/pom.xml b/commons/com.b2international.index.es8/pom.xml new file mode 100644 index 00000000000..29c5cbce35b --- /dev/null +++ b/commons/com.b2international.index.es8/pom.xml @@ -0,0 +1,94 @@ + + 4.0.0 + + com.b2international.index.es8 + eclipse-plugin + + + com.b2international.snowowl + commons-parent + 8.4.1-SNAPSHOT + + + + 8.3.2 + + + + + co.elastic.clients + elasticsearch-java + ${elasticsearch8.version} + provided + + + jakarta.json + jakarta.json-api + 2.0.1 + provided + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.1.1 + + + copy-runtime-dependencies + generate-sources + + copy-dependencies + + + + + elasticsearch-java, + elasticsearch-rest-client, + jakarta.json-api, + jsr305, + parsson + + lib + + + + + + + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + org.apache.maven.plugins + maven-dependency-plugin + [3.0.2,) + + copy-dependencies + + + + + false + + + + + + + + + + + + \ No newline at end of file diff --git a/commons/com.b2international.index.es8/src/com/b2international/index/es8/Es8Client.java b/commons/com.b2international.index.es8/src/com/b2international/index/es8/Es8Client.java new file mode 100644 index 00000000000..e9c6770a5bc --- /dev/null +++ b/commons/com.b2international.index.es8/src/com/b2international/index/es8/Es8Client.java @@ -0,0 +1,107 @@ +/* + * Copyright 2022 B2i Healthcare Pte Ltd, http://b2i.sg + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.b2international.index.es8; + +import java.io.Closeable; +import java.io.IOException; + +import javax.net.ssl.SSLContext; + +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.elasticsearch.client.HttpAsyncResponseConsumerFactory; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestClientBuilder.HttpClientConfigCallback; +import org.elasticsearch.client.RestClientBuilder.RequestConfigCallback; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Strings; + +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.json.jackson.JacksonJsonpMapper; +import co.elastic.clients.transport.ElasticsearchTransport; +import co.elastic.clients.transport.TransportOptions; +import co.elastic.clients.transport.rest_client.RestClientOptions; +import co.elastic.clients.transport.rest_client.RestClientTransport; + +/** + * Special Elasticsearch 8 compatible Java Client that creates its own Java HTTP client and a {@link ElasticsearchClient} and makes it available to + * the index services. + * + * @since 8.5 + */ +public class Es8Client implements Closeable { + + /* + * Customize the HTTP response consumer factory to allow processing greater than the default 100 MB of data (currently 1 GB) as the input. + */ + private static final int BUFFER_LIMIT = 1024 * 1024 * 1024; + + private final HttpHost host; + + private final ElasticsearchTransport transport; + private final ElasticsearchClient client; + + public Es8Client(String clusterName, String clusterUrl, String username, String password, int connectTimeout, int socketTimeout, SSLContext sslContext, ObjectMapper mapper) { + this.host = HttpHost.create(clusterUrl); + + final RequestConfigCallback requestConfigCallback = requestConfigBuilder -> requestConfigBuilder + .setConnectTimeout(connectTimeout) + .setSocketTimeout(socketTimeout); + + final RestClientBuilder restClientBuilder = RestClient.builder(host) + .setRequestConfigCallback(requestConfigCallback); + + final boolean isProtected = !Strings.isNullOrEmpty(username) && !Strings.isNullOrEmpty(password); + if (isProtected) { + + final HttpClientConfigCallback httpClientConfigCallback = httpClientConfigBuilder -> { + final BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, + new UsernamePasswordCredentials(username, password)); + return httpClientConfigBuilder + .setDefaultCredentialsProvider(credentialsProvider) + .setSSLContext(sslContext); + }; + + restClientBuilder.setHttpClientConfigCallback(httpClientConfigCallback); + + } + + // Create the transport with a Jackson mapper + this.transport = new RestClientTransport(restClientBuilder.build(), new JacksonJsonpMapper(mapper)); + // override DEFAULT transport options from transport with a client with increased HTTP response buffer limit + TransportOptions transportOptions = new RestClientOptions.Builder(((RestClientOptions) this.transport.options()).restClientRequestOptions().toBuilder() + .setHttpAsyncResponseConsumerFactory(new HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory(BUFFER_LIMIT))) + .build(); + this.client = new ElasticsearchClient(transport, transportOptions); + } + + public ElasticsearchClient client() { + return client; + } + + @Override + public void close() throws IOException { + if (this.transport != null) { + this.transport.close(); + } + } + +} diff --git a/commons/com.b2international.index.test.tools/src/com/b2international/index/BaseIndexTest.java b/commons/com.b2international.index.test.tools/src/com/b2international/index/BaseIndexTest.java index 6abe962e26e..eb08d90fa40 100644 --- a/commons/com.b2international.index.test.tools/src/com/b2international/index/BaseIndexTest.java +++ b/commons/com.b2international.index.test.tools/src/com/b2international/index/BaseIndexTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2011-2021 B2i Healthcare Pte Ltd, http://b2i.sg + * Copyright 2011-2022 B2i Healthcare Pte Ltd, http://b2i.sg * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,13 +43,21 @@ public abstract class BaseIndexTest { protected static final String KEY2 = "key2"; @Rule - public final IndexResource index = IndexResource.create(getTypes(), this::configureMapper, this::getIndexSettings); + public final IndexResource index = IndexResource.create(getTypes(), this::configureMapper, this::getIndexSettings, this::version); /** * @return the document types used by this test case */ protected abstract Collection> getTypes(); + /** + * Subclasses may override this method to return an Elasticsearch major that they support. By default it returns `*`, which represents all versions are supported and tests should run on all versions. + * + * @return + */ + protected String version() { + return "*"; + } protected void configureMapper(ObjectMapper mapper) { diff --git a/commons/com.b2international.index.test.tools/src/com/b2international/index/IndexResource.java b/commons/com.b2international.index.test.tools/src/com/b2international/index/IndexResource.java index f02f6c897d6..65b9d5a2375 100644 --- a/commons/com.b2international.index.test.tools/src/com/b2international/index/IndexResource.java +++ b/commons/com.b2international.index.test.tools/src/com/b2international/index/IndexResource.java @@ -15,6 +15,8 @@ */ package com.b2international.index; +import static org.junit.Assume.assumeTrue; + import java.util.Collection; import java.util.Map; import java.util.UUID; @@ -49,7 +51,7 @@ public final class IndexResource extends ExternalResource { */ public static final String ES_USE_TEST_CONTAINER_VARIABLE = "so.index.es.useDocker"; - public static final String DEFAULT_ES_DOCKER_IMAGE = "docker.elastic.co/elasticsearch/elasticsearch:8.1.3"; + public static final String DEFAULT_ES_DOCKER_IMAGE = "docker.elastic.co/elasticsearch/elasticsearch:8.3.2"; private static final AtomicBoolean INIT = new AtomicBoolean(false); @@ -62,11 +64,13 @@ public final class IndexResource extends ExternalResource { private final Collection> types; private final Consumer objectMapperConfigurator; private final Supplier> indexSettings; + private final Supplier supportedVersion; - private IndexResource(Collection> types, Consumer objectMapperConfigurator, Supplier> indexSettings) { + private IndexResource(Collection> types, Consumer objectMapperConfigurator, Supplier> indexSettings, Supplier supportedVersion) { this.types = types; this.objectMapperConfigurator = objectMapperConfigurator; this.indexSettings = indexSettings; + this.supportedVersion = supportedVersion; } @Override @@ -100,6 +104,9 @@ protected void before() throws Throwable { revisionIndex = new DefaultRevisionIndex(index, new TimestampProvider.Default(), mapper); } + // when init is ready check version and ignore test if connected cluster is not supported + assumeTrue(supportedVersion.get().equals("*") || index.admin().client().version().startsWith(supportedVersion.get())); + if (container != null) { // make sure we update the synonyms.txt inside the test container final MountableFile localSynonymFilePath = MountableFile.forHostPath(EsIndexClientFactory.DEFAULT_PATH.resolve(IndexClientFactory.DEFAULT_CLUSTER_NAME).resolve(EsNode.CONFIG_DIR).resolve(EsNode.SYNONYMS_FILE)); @@ -150,8 +157,8 @@ public ObjectMapper getMapper() { return mapper; } - public static IndexResource create(Collection> types, Consumer objectMapperConfigurator, Supplier> indexSettings) { - return new IndexResource(types, objectMapperConfigurator, indexSettings); + public static IndexResource create(Collection> types, Consumer objectMapperConfigurator, Supplier> indexSettings, Supplier supportedVersion) { + return new IndexResource(types, objectMapperConfigurator, indexSettings, supportedVersion); } } diff --git a/commons/com.b2international.index.test.tools/src/com/b2international/index/revision/BaseRevisionIndexTest.java b/commons/com.b2international.index.test.tools/src/com/b2international/index/revision/BaseRevisionIndexTest.java index 4afc3e814d6..8af9a8ae656 100644 --- a/commons/com.b2international.index.test.tools/src/com/b2international/index/revision/BaseRevisionIndexTest.java +++ b/commons/com.b2international.index.test.tools/src/com/b2international/index/revision/BaseRevisionIndexTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2011-2021 B2i Healthcare Pte Ltd, http://b2i.sg + * Copyright 2011-2022 B2i Healthcare Pte Ltd, http://b2i.sg * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -49,7 +49,7 @@ public abstract class BaseRevisionIndexTest { private final Collection hooks = newArrayListWithCapacity(2); @Rule - public final IndexResource index = IndexResource.create(getTypes(), this::configureMapper, this::getIndexSettings); + public final IndexResource index = IndexResource.create(getTypes(), this::configureMapper, this::getIndexSettings, this::version); @After public void after() { @@ -64,6 +64,15 @@ protected Collection> getTypes() { return Collections.emptySet(); } + /** + * Subclasses may override this method to return an Elasticsearch major that they support. By default it returns `*`, which represents all versions are supported and tests should run on all versions. + * + * @return + */ + protected String version() { + return "*"; + } + protected void configureMapper(ObjectMapper mapper) { } diff --git a/commons/com.b2international.index.tests/.launch/index-http-unit-tests (es8).launch b/commons/com.b2international.index.tests/.launch/index-http-unit-tests (es8).launch index 9dbed6aa6d7..d12f04c8828 100644 --- a/commons/com.b2international.index.tests/.launch/index-http-unit-tests (es8).launch +++ b/commons/com.b2international.index.tests/.launch/index-http-unit-tests (es8).launch @@ -82,8 +82,8 @@ - - + + @@ -118,6 +118,7 @@ + diff --git a/commons/com.b2international.index.tests/.launch/index-http-unit-tests.launch b/commons/com.b2international.index.tests/.launch/index-http-unit-tests.launch index 835fd19ff24..3caf79924c5 100644 --- a/commons/com.b2international.index.tests/.launch/index-http-unit-tests.launch +++ b/commons/com.b2international.index.tests/.launch/index-http-unit-tests.launch @@ -82,8 +82,8 @@ - - + + @@ -118,11 +118,12 @@ + - + diff --git a/commons/com.b2international.index.tests/.launch/index-tcp-unit-tests.launch b/commons/com.b2international.index.tests/.launch/index-tcp-unit-tests.launch index dd743a3b50b..a6ef34da086 100644 --- a/commons/com.b2international.index.tests/.launch/index-tcp-unit-tests.launch +++ b/commons/com.b2international.index.tests/.launch/index-tcp-unit-tests.launch @@ -84,8 +84,8 @@ - - + + @@ -117,6 +117,7 @@ + diff --git a/commons/com.b2international.index.tests/META-INF/MANIFEST.MF b/commons/com.b2international.index.tests/META-INF/MANIFEST.MF index 084ba5a941e..0df4d125b42 100644 --- a/commons/com.b2international.index.tests/META-INF/MANIFEST.MF +++ b/commons/com.b2international.index.tests/META-INF/MANIFEST.MF @@ -9,6 +9,7 @@ Bundle-ActivationPolicy: lazy Fragment-Host: com.b2international.index Require-Bundle: org.junit;bundle-version="4.13.0", com.b2international.index.test.tools, + com.b2international.collections.jackson, org.apache.commons.lang, com.fasterxml.jackson.core.jackson-annotations, ch.qos.logback.classic;bundle-version="1.2.3", diff --git a/commons/com.b2international.index.tests/src/com/b2international/index/es8/Elasticsearch8ClientTest.java b/commons/com.b2international.index.tests/src/com/b2international/index/es8/Elasticsearch8ClientTest.java new file mode 100644 index 00000000000..6cdbaa79164 --- /dev/null +++ b/commons/com.b2international.index.tests/src/com/b2international/index/es8/Elasticsearch8ClientTest.java @@ -0,0 +1,128 @@ +/* + * Copyright 2022 B2i Healthcare Pte Ltd, http://b2i.sg + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.b2international.index.es8; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Collection; +import java.util.Map; + +import org.elasticsearch.core.List; +import org.junit.Test; + +import com.b2international.collections.PrimitiveCollectionModule; +import com.b2international.collections.PrimitiveLists; +import com.b2international.collections.floats.FloatList; +import com.b2international.index.Doc; +import com.b2international.index.Hits; +import com.b2international.index.query.Expressions; +import com.b2international.index.query.Knn; +import com.b2international.index.revision.BaseRevisionIndexTest; +import com.b2international.index.revision.Revision; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; + +/** + * @since 8.5 + */ +public class Elasticsearch8ClientTest extends BaseRevisionIndexTest { + + @Doc(type = "rev-dense-vector") + private static final class RevisionWithDenseVector extends Revision { + + private final FloatList value; + + @JsonCreator + public RevisionWithDenseVector(@JsonProperty("id") String id, @JsonProperty("value") FloatList value) { + super(id); + this.value = value; + } + + public FloatList getValue() { + return value; + } + + } + + @Override + protected String version() { + return "8"; + } + + @Override + protected void configureMapper(ObjectMapper mapper) { + super.configureMapper(mapper); + mapper.registerModule(new PrimitiveCollectionModule()); + } + + @Override + protected Map getIndexSettings() { + return ImmutableMap.builder() + .putAll(super.getIndexSettings()) + .put("rev-dense-vector", Map.of( + "mappings", Map.of( + "properties", Map.of( + "value", Map.of( + "type", "dense_vector", + "index", true, + "dims", 3, + "similarity", "cosine" + ) + ) + ) + )) + .build(); + } + + @Override + protected Collection> getTypes() { + return List.of(RevisionWithDenseVector.class); + } + + @Test + public void knn_revisions() throws Exception { + indexRevision(MAIN, + new RevisionWithDenseVector(STORAGE_KEY1, PrimitiveLists.newFloatArrayList(0.31f, 0.61f, 0.71f)), + new RevisionWithDenseVector(STORAGE_KEY2, PrimitiveLists.newFloatArrayList(0.32f, 0.62f, 0.72f)) + ); + String branchA = createBranch(MAIN, "a"); + indexRevision(branchA, + new RevisionWithDenseVector(STORAGE_KEY3, PrimitiveLists.newFloatArrayList(0.29f, 0.59f, 0.69f)), + new RevisionWithDenseVector(STORAGE_KEY4, PrimitiveLists.newFloatArrayList(0.28f, 0.58f, 0.68f)) + ); + + Hits matches = index().read(MAIN, searcher -> { + return searcher.knn( + Knn + .select(RevisionWithDenseVector.class) + .field("value") + .k(5) + .numCandidates(5) + .queryVector(0.3f, 0.6f, 0.7f) + .filter(Expressions.matchAll()) + .build() + ); + }); + + assertThat(matches) + .extracting(RevisionWithDenseVector::getId) + .containsOnly(STORAGE_KEY1, STORAGE_KEY2); + + } + +} diff --git a/commons/com.b2international.index/META-INF/MANIFEST.MF b/commons/com.b2international.index/META-INF/MANIFEST.MF index 668b3da4b45..72e72692c58 100644 --- a/commons/com.b2international.index/META-INF/MANIFEST.MF +++ b/commons/com.b2international.index/META-INF/MANIFEST.MF @@ -15,15 +15,16 @@ Require-Bundle: org.eclipse.core.runtime;bundle-version="3.9.0", io.netty.transport;bundle-version="4.1.45", org.apache.commons.codec;bundle-version="1.11.0", org.apache.httpcomponents.httpasyncclient;bundle-version="[4.1.4,4.2.0)", - org.apache.httpcomponents.httpclient;bundle-version="[4.5.3,4.5.4)", - org.apache.httpcomponents.httpcore;bundle-version="[4.4.10,4.5.0)", + org.apache.httpcomponents.httpclient;bundle-version="[4.5.10,4.6.0)", + org.apache.httpcomponents.httpcore;bundle-version="[4.4.12,4.5.0)", org.hdrhistogram.HdrHistogram;bundle-version="2.1.10", com.b2international.commons;visibility:=reexport, com.fasterxml.jackson.dataformat.jackson-dataformat-smile;bundle-version="[2.9.9,3.0.0)", com.fasterxml.jackson.dataformat.jackson-dataformat-cbor;bundle-version="[2.9.9,3.0.0)", com.fasterxml.jackson.core.jackson-databind;bundle-version="[2.9.9,3.0.0)", com.google.guava;bundle-version="[27.1.0,28.0.0)", - net.jodah.failsafe;bundle-version="2.3.1" + net.jodah.failsafe;bundle-version="2.3.1", + com.b2international.index.es8 Export-Package: com.b2international.index, com.b2international.index.admin, com.b2international.index.aggregations, diff --git a/commons/com.b2international.index/src/com/b2international/index/Searcher.java b/commons/com.b2international.index/src/com/b2international/index/Searcher.java index 2ea1797c91e..a14e21264ba 100644 --- a/commons/com.b2international.index/src/com/b2international/index/Searcher.java +++ b/commons/com.b2international.index/src/com/b2international/index/Searcher.java @@ -1,5 +1,5 @@ /* - * Copyright 2011-2021 B2i Healthcare Pte Ltd, http://b2i.sg + * Copyright 2011-2022 B2i Healthcare Pte Ltd, http://b2i.sg * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import com.b2international.index.aggregations.Aggregation; import com.b2international.index.aggregations.AggregationBuilder; +import com.b2international.index.query.Knn; import com.b2international.index.query.Query; import com.google.common.collect.Streams; @@ -72,6 +73,18 @@ public interface Searcher { */ Iterable get(Class type, Iterable keys) throws IOException; + /** + * Perform k nearest neighbor search on the select index using the provided search parameters. + * + * @param - the response type + * @param knn - the knn query to run + * @return a {@link Hits} containing all hits returned by the knn search + * @throws IOException + */ + default Hits knn(Knn knn) throws IOException { + throw new UnsupportedOperationException("This feature is only supported in specific searcher implementations"); + } + /** * Returns a {@link Stream} that computes all matches of the given query, * returning them in chunks defined in the query limit. diff --git a/commons/com.b2international.index/src/com/b2international/index/admin/IndexAdmin.java b/commons/com.b2international.index/src/com/b2international/index/admin/IndexAdmin.java index 52adcac23a3..3bdfa62c24b 100644 --- a/commons/com.b2international.index/src/com/b2international/index/admin/IndexAdmin.java +++ b/commons/com.b2international.index/src/com/b2international/index/admin/IndexAdmin.java @@ -1,5 +1,5 @@ /* - * Copyright 2011-2021 B2i Healthcare Pte Ltd, http://b2i.sg + * Copyright 2011-2022 B2i Healthcare Pte Ltd, http://b2i.sg * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,8 +20,10 @@ import java.util.Map; import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.b2international.index.es.client.EsClient; +import com.b2international.index.es8.Es8Client; import com.b2international.index.mapping.DocumentMapping; import com.b2international.index.mapping.Mappings; @@ -123,6 +125,14 @@ public interface IndexAdmin { * @return the Elasticsearch client used by this {@link IndexAdmin}. */ EsClient client(); + + /** + * NOTE: depending on configuration, this client might not be available. + * + * @return the Elasticsearch high-level client that supports all Elasticsearch 8 features + * @throws UnsupportedOperationException - if es8Client is not available + */ + Es8Client es8Client() throws UnsupportedOperationException; /** * @return the indices maintained by this {@link IndexAdmin} @@ -134,5 +144,9 @@ default String[] indices() { .distinct() .toArray(String[]::new); } + + static Logger createIndexLogger(String name) { + return LoggerFactory.getLogger(String.join(".", "index", name)); + } } diff --git a/commons/com.b2international.index/src/com/b2international/index/es/EsDocumentSearcher.java b/commons/com.b2international.index/src/com/b2international/index/es/EsDocumentSearcher.java index 704a73dd3c8..3239da3c8cc 100644 --- a/commons/com.b2international.index/src/com/b2international/index/es/EsDocumentSearcher.java +++ b/commons/com.b2international.index/src/com/b2international/index/es/EsDocumentSearcher.java @@ -24,6 +24,7 @@ import java.io.DataInputStream; import java.io.IOException; import java.util.*; +import java.util.stream.Collectors; import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.TotalHits.Relation; @@ -50,6 +51,7 @@ import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; +import com.b2international.collections.floats.FloatIterator; import com.b2international.commons.CompareUtils; import com.b2international.commons.exceptions.BadRequestException; import com.b2international.commons.exceptions.FormattedRuntimeException; @@ -59,9 +61,11 @@ import com.b2international.index.aggregations.Bucket; import com.b2international.index.es.admin.EsIndexAdmin; import com.b2international.index.es.client.EsClient; +import com.b2international.index.es.query.Es8QueryBuilder; import com.b2international.index.es.query.EsQueryBuilder; import com.b2international.index.mapping.DocumentMapping; import com.b2international.index.query.Expressions; +import com.b2international.index.query.Knn; import com.b2international.index.query.Query; import com.b2international.index.query.SortBy; import com.b2international.index.query.SortBy.MultiSortBy; @@ -73,6 +77,11 @@ import com.google.common.collect.*; import com.google.common.primitives.Ints; +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch.core.KnnSearchRequest; +import co.elastic.clients.elasticsearch.core.KnnSearchResponse; +import co.elastic.clients.elasticsearch.core.knn_search.KnnSearchQuery; + /** * @since 5.10 */ @@ -532,5 +541,41 @@ public int resultWindow() { public int maxTermsCount() { return maxTermsCount; } + + @Override + public Hits knn(Knn knn) throws IOException { + if (admin.es8Client() == null) { + throw new BadRequestException("Approximate knn search is only available in Elastiscearch 8 clusters. The currently connected ES cluster version is %s.", admin.client().version()); + } + + final ElasticsearchClient client = admin.es8Client().client(); + final DocumentMapping mapping = admin.mappings().getMapping(knn.getFrom()); + + Es8QueryBuilder q = new Es8QueryBuilder(mapping, admin.settings(), admin.log()); + co.elastic.clients.elasticsearch._types.query_dsl.Query esQuery = q.build(knn.getFilter()); + + // TODO consider adding support for double primitive lists + FloatIterator it = knn.getQueryVector().iterator(); + final List queryVector = new ArrayList<>(knn.getQueryVector().size()); + while (it.hasNext()) { + queryVector.add((double) it.next()); + } + + KnnSearchResponse response = client.knnSearch(KnnSearchRequest.of(search -> + search + .knn(KnnSearchQuery.of(query -> query + .field(knn.getField()) + .k(knn.getK()) + .numCandidates(knn.getNumCandidates()) + .queryVector(queryVector) + )) + .index(admin.getTypeIndex(mapping)) + .filter(esQuery) + ), knn.getFrom()); + + return new Hits(response.hits().hits().stream().map(h -> { + return h.source(); + }).collect(Collectors.toList()), null, response.hits().hits().size(), (int) response.hits().total().value()); + } } diff --git a/commons/com.b2international.index/src/com/b2international/index/es/EsIndexClientFactory.java b/commons/com.b2international.index/src/com/b2international/index/es/EsIndexClientFactory.java index 1b6ea6028d2..13c9a2d5d10 100644 --- a/commons/com.b2international.index/src/com/b2international/index/es/EsIndexClientFactory.java +++ b/commons/com.b2international.index/src/com/b2international/index/es/EsIndexClientFactory.java @@ -15,6 +15,7 @@ */ package com.b2international.index.es; +import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Map; @@ -25,9 +26,12 @@ import com.b2international.index.IndexClient; import com.b2international.index.IndexClientFactory; +import com.b2international.index.admin.IndexAdmin; import com.b2international.index.es.admin.EsIndexAdmin; import com.b2international.index.es.client.EsClient; +import com.b2international.index.es.client.http.EsHttpClient; import com.b2international.index.es.client.tcp.EsTcpClient; +import com.b2international.index.es8.Es8Client; import com.b2international.index.mapping.Mappings; import com.fasterxml.jackson.databind.ObjectMapper; @@ -54,11 +58,11 @@ public IndexClient createClient(String name, ObjectMapper mapper, Mappings mappi final int socketTimeout = socketTimeoutSetting instanceof Integer ? (int) socketTimeoutSetting : Integer.parseInt((String) socketTimeoutSetting); final String username = (String) settings.getOrDefault(CLUSTER_USERNAME, ""); final String password = (String) settings.getOrDefault(CLUSTER_PASSWORD, ""); + final SSLContext sslContext = (SSLContext) settings.get(CLUSTER_SSL_CONTEXT); final EsClient client; if (settings.containsKey(CLUSTER_URL)) { final String clusterUrl = (String) settings.get(CLUSTER_URL); - SSLContext sslContext = (SSLContext) settings.get(CLUSTER_SSL_CONTEXT); client = EsClient.create(new EsClientConfiguration(clusterName, clusterUrl, username, password, connectTimeout, socketTimeout, sslContext)); } else { // Start an embedded ES node only if a cluster URL is not set @@ -72,6 +76,22 @@ public IndexClient createClient(String name, ObjectMapper mapper, Mappings mappi } } - return new EsIndexClient(new EsIndexAdmin(client, mapper, name, mappings, settings), mapper); + // check version, and if needed create additional ES8 client as well, for certain ES8 features + boolean isElasticsearch8 = false; + try { + isElasticsearch8 = client.version().startsWith("8."); + } catch (IOException e) { + // report error as warning that Elasticsearch 8 client could not be connected succesfully and certain features will be disabled/unavailable + IndexAdmin.createIndexLogger(name).warn("Failed to determine version of underlying Elasticsearch cluster. Certain Elasticsearch 8 only features won't be available. Diagnosis: {}", e.getMessage()); + } + + Es8Client es8Client = null; + // external Elasticsearch 8 cluster through http is supported, embedded and tcp support is not available + if (isElasticsearch8 && settings.containsKey(CLUSTER_URL) && client instanceof EsHttpClient) { + final String clusterUrl = (String) settings.get(CLUSTER_URL); + es8Client = new Es8Client(clusterName, clusterUrl, username, password, connectTimeout, socketTimeout, sslContext, mapper); + } + + return new EsIndexClient(new EsIndexAdmin(client, mapper, name, mappings, settings).withEs8Client(es8Client), mapper); } } diff --git a/commons/com.b2international.index/src/com/b2international/index/es/admin/EsIndexAdmin.java b/commons/com.b2international.index/src/com/b2international/index/es/admin/EsIndexAdmin.java index c611432022f..f1c89e2408c 100644 --- a/commons/com.b2international.index/src/com/b2international/index/es/admin/EsIndexAdmin.java +++ b/commons/com.b2international.index/src/com/b2international/index/es/admin/EsIndexAdmin.java @@ -52,7 +52,6 @@ import org.elasticsearch.script.ScriptType; import org.elasticsearch.xcontent.XContentType; import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.b2international.commons.CompareUtils; import com.b2international.commons.ReflectionUtils; @@ -61,6 +60,7 @@ import com.b2international.index.admin.IndexAdmin; import com.b2international.index.es.client.EsClient; import com.b2international.index.es.query.EsQueryBuilder; +import com.b2international.index.es8.Es8Client; import com.b2international.index.mapping.DocumentMapping; import com.b2international.index.mapping.FieldAlias; import com.b2international.index.mapping.Mappings; @@ -113,6 +113,9 @@ public final class EsIndexAdmin implements IndexAdmin { private final Logger log; private final String prefix; + // optionally available Elasticsearch 8 client API + private Es8Client es8Client; + public EsIndexAdmin(EsClient client, ObjectMapper mapper, String name, Mappings mappings, Map settings) { this.client = client; this.mapper = mapper; @@ -120,7 +123,7 @@ public EsIndexAdmin(EsClient client, ObjectMapper mapper, String name, Mappings this.mappings = mappings; this.settings = newHashMap(settings); - this.log = LoggerFactory.getLogger(String.format("index.%s", this.name)); + this.log = IndexAdmin.createIndexLogger(name); // configuration settings for ES index this.settings.putIfAbsent(IndexClientFactory.NUMBER_OF_SHARDS, IndexClientFactory.DEFAULT_NUMBER_OF_SHARDS); @@ -139,6 +142,11 @@ public EsIndexAdmin(EsClient client, ObjectMapper mapper, String name, Mappings final String prefix = (String) settings.getOrDefault(IndexClientFactory.INDEX_PREFIX, IndexClientFactory.DEFAULT_INDEX_PREFIX); this.prefix = prefix.isEmpty() ? "" : prefix + "."; } + + public EsIndexAdmin withEs8Client(Es8Client es8Client) { + this.es8Client = es8Client; + return this; + } @Override public Logger log() { @@ -553,6 +561,8 @@ public void updateSettings(Map newSettings) { Map esSettings = new HashMap<>(newSettings); // remove any local settings from esSettings esSettings.keySet().removeAll(LOCAL_SETTINGS); + // also remove type index specific mapping settings, those are dynamically not adjustable + esSettings.keySet().removeAll(mappings.getTypeIndexNames()); for (DocumentMapping mapping : mappings.getMappings()) { final String index = getTypeIndex(mapping); @@ -616,6 +626,14 @@ public EsClient client() { return client; } + @Override + public Es8Client es8Client() throws UnsupportedOperationException { + if (es8Client == null) { + throw new UnsupportedOperationException("Elasticsearch high-level client with new ES8 features is not available."); + } + return es8Client; + } + public void refresh(Set typesToRefresh) { if (!CompareUtils.isEmpty(typesToRefresh)) { final String[] indicesToRefresh; @@ -783,5 +801,5 @@ private boolean bulkIndexByScroll(final EsClient client, return needsRefresh; } - + } diff --git a/commons/com.b2international.index/src/com/b2international/index/es/client/EsClient.java b/commons/com.b2international.index/src/com/b2international/index/es/client/EsClient.java index 8f49afd4b52..e6c80a79321 100644 --- a/commons/com.b2international.index/src/com/b2international/index/es/client/EsClient.java +++ b/commons/com.b2international.index/src/com/b2international/index/es/client/EsClient.java @@ -54,6 +54,13 @@ public interface EsClient extends AutoCloseable { Logger LOG = LoggerFactory.getLogger("elastic-snowowl"); + /** + * Gets the Elasticsearch version from the currently configured host using the Info Endpoint. + * @return a version number in the form of "major.minor.patch", never null + * @throws IOException + */ + String version() throws IOException; + EsClusterStatus status(String...indices); IndicesClient indices(); diff --git a/commons/com.b2international.index/src/com/b2international/index/es/client/http/EsHttpClient.java b/commons/com.b2international.index/src/com/b2international/index/es/client/http/EsHttpClient.java index 45a1843a046..4bd8f4c6b63 100644 --- a/commons/com.b2international.index/src/com/b2international/index/es/client/http/EsHttpClient.java +++ b/commons/com.b2international.index/src/com/b2international/index/es/client/http/EsHttpClient.java @@ -133,6 +133,11 @@ public final ClusterClient cluster() { return clusterClient; } + @Override + public String version() throws IOException { + return client.info(EXTENDED_DEFAULT).getVersion().getNumber(); + } + @Override protected boolean ping() throws IOException { return client.ping(EXTENDED_DEFAULT); diff --git a/commons/com.b2international.index/src/com/b2international/index/es/client/tcp/EsTcpClient.java b/commons/com.b2international.index/src/com/b2international/index/es/client/tcp/EsTcpClient.java index ae5e0949bdc..41f1fdc5fc2 100644 --- a/commons/com.b2international.index/src/com/b2international/index/es/client/tcp/EsTcpClient.java +++ b/commons/com.b2international.index/src/com/b2international/index/es/client/tcp/EsTcpClient.java @@ -56,6 +56,13 @@ public EsTcpClient(Client client) { this.clusterClient = new ClusterTcpClient(client.admin().cluster()); } + @Override + public String version() throws IOException { + // fake version to mimic 7.x behavior, TCP support has been removed in 8.x of ES, no need to ask the connected ES what version it has + // replace this with actual logic if we need an actual 7.x ES version for anything + return "7.x"; + } + @Override protected boolean ping() throws IOException { return true; // always returns true diff --git a/commons/com.b2international.index/src/com/b2international/index/es/query/Es8QueryBuilder.java b/commons/com.b2international.index/src/com/b2international/index/es/query/Es8QueryBuilder.java new file mode 100644 index 00000000000..7b22f3bed0e --- /dev/null +++ b/commons/com.b2international.index/src/com/b2international/index/es/query/Es8QueryBuilder.java @@ -0,0 +1,540 @@ +/* + * Copyright 2022 B2i Healthcare Pte Ltd, http://b2i.sg + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.b2international.index.es.query; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.elasticsearch.script.Script; +import org.slf4j.Logger; + +import com.b2international.commons.exceptions.FormattedRuntimeException; +import com.b2international.index.IndexClientFactory; +import com.b2international.index.compat.TextConstants; +import com.b2international.index.mapping.DocumentMapping; +import com.b2international.index.query.*; +import com.b2international.index.query.TextPredicate.MatchType; +import com.b2international.index.util.DecimalUtils; +import com.google.common.collect.*; + +import co.elastic.clients.elasticsearch._types.FieldValue; +import co.elastic.clients.elasticsearch._types.query_dsl.*; +import co.elastic.clients.elasticsearch._types.query_dsl.Query; +import co.elastic.clients.json.JsonData; + +/** + * @since 8.5 + */ +public class Es8QueryBuilder { + + private static final Query MATCH_NONE = QueryBuilders.matchNone(m -> m); + private static String ILLEGAL_STACK_STATE_MESSAGE = "Illegal internal stack state: %s"; + + private final Deque deque = Queues.newLinkedBlockingDeque(); + private final Map settings; + private final DocumentMapping mapping; + private final Logger log; + private final String path; + + private boolean needsScoring; + private Float boost = null; // null means no boost + + public Es8QueryBuilder(DocumentMapping mapping, Map settings, Logger log) { + this(mapping, settings, log, ""); + } + + private Es8QueryBuilder(DocumentMapping mapping, Map settings, Logger log, String path) { + this.mapping = mapping; + this.settings = settings; + this.log = log; + this.path = path; + } + + private FormattedRuntimeException newIllegalStateException() { + return new FormattedRuntimeException(ILLEGAL_STACK_STATE_MESSAGE, deque); + } + + public boolean needsScoring() { + return needsScoring; + } + + public Query build(Expression expression) { + checkNotNull(expression, "expression"); + visit(expression); + if (deque.size() == 1) { + return deque.pop(); + } else { + throw newIllegalStateException(); + } + } + + private void visit(Expression expression) { + if (expression instanceof BoostPredicate) { + // in case of boost predicate, store the boost value until visit of the inner expression happens, and let that set the boost value + BoostPredicate boostPredicate = (BoostPredicate) expression; + this.boost = boostPredicate.boost(); + try { + visit(boostPredicate.expression()); + } finally { + this.boost = null; + } + } else if (expression instanceof MatchAll) { + deque.push(QueryBuilders.matchAll(m -> m)); + } else if (expression instanceof MatchNone) { + // XXX executing a term query on a probably nonexistent field and value, should return zero docs + deque.push(MATCH_NONE); + } else if (expression instanceof StringPredicate) { + visit((StringPredicate) expression); + } else if (expression instanceof LongPredicate) { + visit((LongPredicate) expression); + } else if (expression instanceof LongRangePredicate) { + visit((LongRangePredicate) expression); + } else if (expression instanceof StringRangePredicate) { + visit((StringRangePredicate) expression); + } else if (expression instanceof NestedPredicate) { + visit((NestedPredicate) expression); + } else if (expression instanceof HasParentPredicate) { + visit((HasParentPredicate) expression); + } else if (expression instanceof PrefixPredicate) { + visit((PrefixPredicate) expression); + } else if (expression instanceof RegexpPredicate) { + visit((RegexpPredicate) expression); + } else if (expression instanceof StringSetPredicate) { + visit((StringSetPredicate) expression); + } else if (expression instanceof LongSetPredicate) { + visit((LongSetPredicate) expression); + } else if (expression instanceof IntPredicate) { + visit((IntPredicate) expression); + } else if (expression instanceof IntSetPredicate) { + visit((IntSetPredicate) expression); + } else if (expression instanceof BoolExpression) { + visit((BoolExpression) expression); + } else if (expression instanceof BooleanPredicate) { + visit((BooleanPredicate) expression); + } else if (expression instanceof IntRangePredicate) { + visit((IntRangePredicate) expression); + } else if (expression instanceof TextPredicate) { + visit((TextPredicate) expression); + } else if (expression instanceof DisMaxPredicate) { + visit((DisMaxPredicate) expression); + } else if (expression instanceof ScriptScoreExpression) { + visit((ScriptScoreExpression) expression); + } else if (expression instanceof DecimalPredicate) { + visit((DecimalPredicate) expression); + } else if (expression instanceof DecimalRangePredicate) { + visit((DecimalRangePredicate) expression); + } else if (expression instanceof DecimalSetPredicate) { + visit((DecimalSetPredicate) expression); + } else if (expression instanceof DoublePredicate) { + visit((DoublePredicate) expression); + } else if (expression instanceof DoubleRangePredicate) { + visit((DoubleRangePredicate) expression); + } else if (expression instanceof DoubleSetPredicate) { + visit((DoubleSetPredicate) expression); + } else if (expression instanceof ScriptQueryExpression){ + visit((ScriptQueryExpression)expression); + } else { + throw new IllegalArgumentException("Unexpected expression: " + expression); + } + } + + private void visit(ScriptQueryExpression expression) { + Script esScript = expression.toEsScript(mapping); + deque.push(QueryBuilders.script(script -> script.boost(this.boost).script(s -> s.inline(in -> in + .lang(esScript.getLang()) + .source(esScript.getIdOrCode()) + .options(esScript.getOptions()) + .params(Maps.transformValues(esScript.getParams(), JsonData::of)) + )))); + } + + private void visit(ScriptScoreExpression expression) { + final Expression inner = expression.expression(); + visit(inner); + final Query innerQuery = deque.pop(); + + Script esScript = expression.toEsScript(mapping); + + needsScoring = true; + deque.push(QueryBuilders + .functionScore(q -> q + .boost(this.boost) + .boostMode(FunctionBoostMode.Replace) + .query(innerQuery) + .functions( + FunctionScoreBuilders.scriptScore( + scriptScore -> scriptScore.script( + s -> s.inline(in -> in + .lang(esScript.getLang()) + .source(esScript.getIdOrCode()) + .options(esScript.getOptions()) + .params(Maps.transformValues(esScript.getParams(), JsonData::of)) + ) + ) + ) + ) + ) + ); + } + + private void visit(BoolExpression bool) { + // Assumes that BoolExpression clauses are stored in writable array lists + reduceTermFilters(bool.mustClauses()); + reduceTermFilters(bool.filterClauses()); + + deque.push(QueryBuilders.bool(query -> { + query.boost(this.boost); + for (Expression must : bool.mustClauses()) { + // visit the item and immediately pop the deque item back + final Es8QueryBuilder innerQueryBuilder = new Es8QueryBuilder(mapping, settings, log, path); + innerQueryBuilder.visit(must); + if (innerQueryBuilder.needsScoring) { + needsScoring = innerQueryBuilder.needsScoring; + query.must(innerQueryBuilder.deque.pop()); + } else { + query.filter(innerQueryBuilder.deque.pop()); + } + } + + for (Expression mustNot : bool.mustNotClauses()) { + visit(mustNot); + query.mustNot(deque.pop()); + } + + for (Expression should : bool.shouldClauses()) { + visit(should); + query.should(deque.pop()); + } + + for (Expression filter : bool.filterClauses()) { + visit(filter); + query.filter(deque.pop()); + } + + if (!bool.shouldClauses().isEmpty()) { + query.minimumShouldMatch(String.valueOf(bool.minShouldMatch())); + } + + return query; + })); + } + + private void reduceTermFilters(List clauses) { + Multimap termExpressionsByField = HashMultimap.create(); + for (Expression expression : List.copyOf(clauses)) { + if (shouldMergeSingleArgumentPredicate(expression)) { + termExpressionsByField.put(((SingleArgumentPredicate) expression).getField(), expression); + } else if (shouldMergeSetPredicate(expression)) { + termExpressionsByField.put(((SetPredicate) expression).getField(), expression); + } + } + + for (String field : Set.copyOf(termExpressionsByField.keySet())) { + Collection termExpressions = termExpressionsByField.removeAll(field); + if (termExpressions.size() > 1) { + Set values = null; + for (Expression expression : termExpressions) { + if (values != null && values.isEmpty()) { + break; + } + Set expressionValues; + if (expression instanceof SingleArgumentPredicate) { + expressionValues = Set.of(((SingleArgumentPredicate) expression).getArgument()); + } else if (expression instanceof SetPredicate) { + expressionValues = Set.copyOf(((SetPredicate) expression).values()); + } else { + throw new IllegalStateException("Invalid clause detected when processing term/terms clauses: " + expression); + } + values = values == null ? expressionValues : Set.copyOf(Sets.intersection(values, expressionValues)); + } + // remove all matching clauses first + clauses.removeAll(termExpressions); + // add the new merged expression + clauses.add(Expressions.matchAnyObject(field, values)); + } + } + + } + + private boolean shouldMergeSingleArgumentPredicate(Expression expression) { + return AbstractExpressionBuilder.shouldMergeSingleArgumentPredicate(expression) && referencesScalarField(expression); + } + + private boolean shouldMergeSetPredicate(Expression expression) { + return AbstractExpressionBuilder.shouldMergeSetPredicate(expression) && referencesScalarField(expression); + } + + // Predicates should not be eliminated if the field is a collection type + private boolean referencesScalarField(Expression expression) { + final String fieldName = ((Predicate) expression).getField(); + return mapping.getSelectableFields().contains(fieldName) && !mapping.isCollection(fieldName); + } + + private void visit(NestedPredicate predicate) { + final String nestedPath = toFieldPath(predicate); + final DocumentMapping nestedMapping = mapping.getNestedMapping(predicate.getField()); + final Es8QueryBuilder nestedQueryBuilder = new Es8QueryBuilder(nestedMapping, settings, log, nestedPath); + nestedQueryBuilder.visit(predicate.getExpression()); + needsScoring = nestedQueryBuilder.needsScoring; + final Query nestedQuery = nestedQueryBuilder.deque.pop(); + deque.push(QueryBuilders.nested(n -> n + .boost(this.boost) + .path(nestedPath) + .query(nestedQuery) + .scoreMode(ChildScoreMode.None) + )); + } + + private String toFieldPath(Predicate predicate) { + return toFieldPath(predicate.getField()); + } + + private String toFieldPath(final String subPath) { + return path.isEmpty() ? subPath : String.join(".", path, subPath); + } + + private void visit(HasParentPredicate predicate) { + throw new UnsupportedOperationException(); + } + + private void visit(TextPredicate predicate) { + final String field = toFieldPath(predicate); + final String term = predicate.term(); + final MatchType type = predicate.type(); + final int minShouldMatch = predicate.minShouldMatch(); + Query query; + switch (type) { + case BOOLEAN_PREFIX: + query = QueryBuilders.matchBoolPrefix(mbp -> mbp + .boost(this.boost) + .field(field) + .query(term) + .analyzer(predicate.analyzer()) + .operator(Operator.And) + ); + break; + case PHRASE: + query = QueryBuilders.matchPhrase(mp -> mp + .boost(this.boost) + .field(field) + .query(term) + .analyzer(predicate.analyzer()) + ); + break; + case ALL: + query = QueryBuilders.match(m -> m + .boost(this.boost) + .field(field) + .query(term) + .analyzer(predicate.analyzer()) + .operator(Operator.And) + ); + break; + case ANY: + query = QueryBuilders.match(m -> m + .boost(this.boost) + .field(field) + .query(term) + .analyzer(predicate.analyzer()) + .operator(Operator.Or) + .minimumShouldMatch(Integer.toString(minShouldMatch)) + ); + break; + case FUZZY: + query = QueryBuilders.match(m -> m + .boost(this.boost) + .field(field) + .query(term) + .analyzer(predicate.analyzer()) + .fuzziness("1") + .prefixLength(1) + .operator(Operator.And) + .maxExpansions(10) + ); + break; + case PARSED: + query = QueryBuilders.queryString(qs -> qs + .boost(this.boost) + .fields(field) + .query(TextConstants.escape(term)) + .analyzer(predicate.analyzer()) + .escape(false) + .allowLeadingWildcard(true) + .defaultOperator(Operator.And) + ); + break; + default: throw new UnsupportedOperationException("Unexpected text match type: " + type); + } + if (query == null) { + query = MATCH_NONE; + } else { + needsScoring = true; + } + deque.push(query); + } + + private void visit(SingleArgumentPredicate predicate) { + deque.push(QueryBuilders.term(query -> query + .boost(this.boost) + .field(toFieldPath(predicate)) + .value(String.valueOf(predicate.getArgument())) + )); + } + + private void visit(DecimalPredicate predicate) { + deque.push(QueryBuilders.term(query -> query + .boost(this.boost) + .field(toFieldPath(predicate)) + .value(DecimalUtils.encode(predicate.getArgument())) + )); + } + + private void visit(SetPredicate predicate) { + toTermsQuery(predicate, predicate.values(), null); + } + + private void visit(DecimalSetPredicate predicate) { + toTermsQuery(predicate, predicate.values(), DecimalUtils::encode); + } + + // consider max terms count and break into multiple terms queries if number of terms are greater than that value + private void toTermsQuery(SetPredicate predicate, final Set terms, final Function valueConverter) { + + final Function _valueConverter; + if (valueConverter == null) { + _valueConverter = a -> a; + } else { + _valueConverter = valueConverter; + } + + final int maxTermsCount = Integer.parseInt((String) settings.get(IndexClientFactory.MAX_TERMS_COUNT_KEY)); + if (terms.size() > maxTermsCount) { + log.warn("More ({}) than currently configured max_terms_count ({}) filter values on field query: {}.{}", terms.size(), maxTermsCount, mapping.typeAsString(), toFieldPath(predicate)); + final Query boolQuery = QueryBuilders.bool(bool -> { + bool.boost(this.boost); + + Iterables.partition(terms, maxTermsCount).forEach(partition -> { + bool.should(QueryBuilders.terms(t -> t + .field(toFieldPath(predicate)) + .terms(TermsQueryField.of(values -> values.value(partition.stream().map(_valueConverter).map(String::valueOf).map(FieldValue::of).collect(Collectors.toList())))) + )); + }); + + return bool + .minimumShouldMatch("1"); + }); + deque.push(boolQuery); + } else { + // push the terms query directly + deque.push(QueryBuilders.terms(t -> t + .boost(this.boost) + .field(toFieldPath(predicate)) + .terms(TermsQueryField.of(values -> values.value(terms.stream().map(_valueConverter).map(String::valueOf).map(FieldValue::of).collect(Collectors.toList())))) + )); + } + } + + private void visit(PrefixPredicate predicate) { + if (predicate.values().size() == 0) { + deque.push(MATCH_NONE); + } else if (predicate.values().size() == 1) { + deque.push(QueryBuilders.prefix(p -> + p + .boost(this.boost) + .field(toFieldPath(predicate)) + .value(Iterables.getOnlyElement(predicate.values())) + )); + } else { + deque.push(QueryBuilders.bool(bool -> { + bool.boost(this.boost); + for (String prefixMatch : predicate.values()) { + bool.should(QueryBuilders.prefix(p -> + p + .field(toFieldPath(predicate)) + .value(prefixMatch) + )); + } + return bool; + })); + } + } + + private void visit(RegexpPredicate regexp) { + deque.push(QueryBuilders.regexp(r -> r.boost(this.boost).field(toFieldPath(regexp)).value(regexp.getArgument()))); + } + + private void visit(RangePredicate range) { + deque.push(QueryBuilders.range(r -> { + r.boost(this.boost); + if (range.lower() != null) { + if (range.isIncludeLower()) { + r.gte(JsonData.of(range.lower())); + } else { + r.gt(JsonData.of(range.lower())); + } + } + if (range.upper() != null) { + if (range.isIncludeUpper()) { + r.lte(JsonData.of(range.upper())); + } else { + r.lt(JsonData.of(range.upper())); + } + } + return r.field(toFieldPath(range)); + })); + } + + private void visit(DecimalRangePredicate range) { + deque.push(QueryBuilders.range(r -> { + r.boost(this.boost); + if (range.lower() != null) { + final String lower = DecimalUtils.encode(range.lower()); + if (range.isIncludeLower()) { + r.gte(JsonData.of(lower)); + } else { + r.gt(JsonData.of(lower)); + } + } + if (range.upper() != null) { + final String upper = DecimalUtils.encode(range.upper()); + if (range.isIncludeUpper()) { + r.lte(JsonData.of(upper)); + } else { + r.lt(JsonData.of(upper)); + } + } + return r.field(toFieldPath(range)); + })); + } + + private void visit(DisMaxPredicate dismax) { + deque.push(QueryBuilders.disMax(dm -> { + dm.boost(this.boost); + List disjunctQueries = new ArrayList<>(); + for (Expression disjunct : dismax.disjuncts()) { + visit(disjunct); + disjunctQueries.add(deque.pop()); + } + return dm + .queries(disjunctQueries) + .tieBreaker((double) dismax.tieBreaker()); + })); + } + +} diff --git a/commons/com.b2international.index/src/com/b2international/index/query/Knn.java b/commons/com.b2international.index/src/com/b2international/index/query/Knn.java new file mode 100644 index 00000000000..176936c0e2d --- /dev/null +++ b/commons/com.b2international.index/src/com/b2international/index/query/Knn.java @@ -0,0 +1,132 @@ +/* + * Copyright 2022 B2i Healthcare Pte Ltd, http://b2i.sg + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.b2international.index.query; + +import com.b2international.collections.PrimitiveLists; +import com.b2international.collections.floats.FloatList; + +/** + * @since 8.5 + */ +public final class Knn { + + private Class from; + private String field; + private Expression filter; + private int k; + private int numCandidates; + private FloatList queryVector; + + public Knn() { + } + + public Class getFrom() { + return from; + } + + public String getField() { + return field; + } + + public Expression getFilter() { + return filter; + } + + public int getK() { + return k; + } + + public int getNumCandidates() { + return numCandidates; + } + + public FloatList getQueryVector() { + return queryVector; + } + + public static KnnBuilder select(Class from) { + return new KnnBuilder<>(from); + } + + public Knn withFilter(Expression filter) { + return Knn.select(from) + .field(field) + .filter(filter) + .k(k) + .numCandidates(numCandidates) + .queryVector(queryVector) + .build(); + } + + public static final class KnnBuilder implements Buildable> { + + private final Class from; + private String field; + private Expression filter; + private int k; + private int numCandidates; + private FloatList queryVector; + + public KnnBuilder(Class from) { + this.from = from; + } + + public KnnBuilder field(String field) { + this.field = field; + return this; + } + + public KnnBuilder filter(Expression filter) { + this.filter = filter; + return this; + } + + public KnnBuilder k(int k) { + this.k = k; + return this; + } + + public KnnBuilder numCandidates(int numCandidates) { + this.numCandidates = numCandidates; + return this; + } + + public KnnBuilder queryVector(float...values) { + return queryVector(PrimitiveLists.newFloatArrayList(values)); + } + + public KnnBuilder queryVector(FloatList queryVector) { + this.queryVector = queryVector; + return this; + } + + @Override + public Knn build() { + Knn knn = new Knn<>(); + knn.from = from; + knn.field = field; + knn.filter = filter; + knn.k = k; + knn.numCandidates = numCandidates; + knn.queryVector = queryVector; + return knn; + } + + + + } + +} diff --git a/commons/com.b2international.index/src/com/b2international/index/revision/DefaultRevisionSearcher.java b/commons/com.b2international.index/src/com/b2international/index/revision/DefaultRevisionSearcher.java index 4ac4aa71a24..7a56d544472 100644 --- a/commons/com.b2international.index/src/com/b2international/index/revision/DefaultRevisionSearcher.java +++ b/commons/com.b2international.index/src/com/b2international/index/revision/DefaultRevisionSearcher.java @@ -26,7 +26,9 @@ import com.b2international.index.aggregations.Aggregation; import com.b2international.index.aggregations.AggregationBuilder; import com.b2international.index.es.EsDocumentSearcher; +import com.b2international.index.query.Expression; import com.b2international.index.query.Expressions; +import com.b2international.index.query.Knn; import com.b2international.index.query.Query; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; @@ -35,7 +37,7 @@ /** * @since 4.7 */ -public class DefaultRevisionSearcher implements RevisionSearcher { +public final class DefaultRevisionSearcher implements RevisionSearcher { private final RevisionBranchRef branch; private final Searcher searcher; @@ -87,28 +89,40 @@ public Hits search(Query query) throws IOException { if (query.isRevisionQuery()) { if (query.getSelection().getParentScope() == null) { // rewrite query if we are looking for revision, otherwise if we are looking for unversioned nested use it as is - query = query.withFilter(branch.toRevisionFilter()).build(); + query = query.withFilter(getRevisionFilter()).build(); } else { checkArgument(Revision.class.isAssignableFrom(query.getSelection().getParentScope()), "Searching non-revision documents require a revision parent type: %s", query); // run a query on the parent documents with nested match on the children - query = query.withFilter(Expressions.hasParent(query.getSelection().getParentScope(), branch.toRevisionFilter())).build(); + query = query.withFilter(Expressions.hasParent(query.getSelection().getParentScope(), getRevisionFilter())).build(); } } return searcher.search(query); } - + @Override public Aggregation aggregate(AggregationBuilder aggregation) throws IOException { aggregation.query(Expressions.bool() .filter(aggregation.getQuery()) - .filter(branch.toRevisionFilter()) + .filter(getRevisionFilter()) .build()); return searcher.aggregate(aggregation); } + @Override + public Hits knn(Knn knn) throws IOException { + return searcher.knn(knn.withFilter(Expressions.bool() + .filter(knn.getFilter()) + .filter(getRevisionFilter()) + .build())); + } + @Override public String branch() { return branch.path(); } - + + public final Expression getRevisionFilter() { + return branch.toRevisionFilter(); + } + } diff --git a/commons/com.b2international.index/src/com/b2international/index/revision/RevisionIndexAdmin.java b/commons/com.b2international.index/src/com/b2international/index/revision/RevisionIndexAdmin.java index 99f817343dd..7abfac25866 100644 --- a/commons/com.b2international.index/src/com/b2international/index/revision/RevisionIndexAdmin.java +++ b/commons/com.b2international.index/src/com/b2international/index/revision/RevisionIndexAdmin.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 B2i Healthcare Pte Ltd, http://b2i.sg + * Copyright 2018-2022 B2i Healthcare Pte Ltd, http://b2i.sg * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ import com.b2international.index.admin.IndexAdmin; import com.b2international.index.es.client.EsClient; +import com.b2international.index.es8.Es8Client; import com.b2international.index.mapping.DocumentMapping; import com.b2international.index.mapping.Mappings; @@ -44,6 +45,11 @@ public EsClient client() { return rawIndexAdmin.client(); } + @Override + public Es8Client es8Client() throws UnsupportedOperationException { + return rawIndexAdmin.es8Client(); + } + @Override public Logger log() { return rawIndexAdmin.log(); diff --git a/commons/pom.xml b/commons/pom.xml index af63c6ef3d1..4d036cb8296 100644 --- a/commons/pom.xml +++ b/commons/pom.xml @@ -20,6 +20,7 @@ com.b2international.commons.test com.b2international.groovy com.b2international.index + com.b2international.index.es8 com.b2international.index.tests com.b2international.index.test.tools com.b2international.mapdb diff --git a/core/com.b2international.snowowl.core.dependencies.feature/feature.xml b/core/com.b2international.snowowl.core.dependencies.feature/feature.xml index e9f911007cb..f2545337f9f 100644 --- a/core/com.b2international.snowowl.core.dependencies.feature/feature.xml +++ b/core/com.b2international.snowowl.core.dependencies.feature/feature.xml @@ -396,14 +396,14 @@ Visit us at http://b2i.sg id="org.apache.httpcomponents.httpclient" download-size="0" install-size="0" - version="4.5.3" + version="4.5.10" unpack="false"/> T get(Class type, String key) throws IOException { return index.read(branchPath, searcher -> searcher.get(type, key)); } + @Override + public Hits knn(Knn knn) throws IOException { + return index.read(branchPath, searcher -> searcher.knn(knn)); + } + @Override public String branch() { return branchPath; diff --git a/releng/target-platform/target-platform.target b/releng/target-platform/target-platform.target index 253319c64cc..4b1ac7d53ed 100644 --- a/releng/target-platform/target-platform.target +++ b/releng/target-platform/target-platform.target @@ -351,13 +351,13 @@ org.apache.httpcomponents httpclient-osgi - 4.5.3 + 4.5.10 jar org.apache.httpcomponents httpcore-osgi - 4.4.10 + 4.4.12 jar diff --git a/tests/com.b2international.snowowl.test.dependencies/.classpath b/tests/com.b2international.snowowl.test.dependencies/.classpath index 15612333ffd..5d7ff6628b4 100644 --- a/tests/com.b2international.snowowl.test.dependencies/.classpath +++ b/tests/com.b2international.snowowl.test.dependencies/.classpath @@ -6,10 +6,10 @@ - + - + diff --git a/tests/com.b2international.snowowl.test.dependencies/META-INF/MANIFEST.MF b/tests/com.b2international.snowowl.test.dependencies/META-INF/MANIFEST.MF index 0c82e7c652d..babac91a26c 100644 --- a/tests/com.b2international.snowowl.test.dependencies/META-INF/MANIFEST.MF +++ b/tests/com.b2international.snowowl.test.dependencies/META-INF/MANIFEST.MF @@ -14,10 +14,10 @@ Bundle-ClassPath: ., lib/docker-java-transport-3.2.13.jar, lib/docker-java-transport-zerodep-3.2.13.jar, lib/duct-tape-1.0.8.jar, - lib/elasticsearch-1.17.1.jar, + lib/elasticsearch-1.17.3.jar, lib/jna-5.8.0.jar, lib/system-rules-1.18.0-sources.jar, - lib/testcontainers-1.17.1.jar + lib/testcontainers-1.17.3.jar Export-Package: com.b2international.snowowl.rules, org.databene.contiperf, org.databene.contiperf.clock, diff --git a/tests/com.b2international.snowowl.test.dependencies/pom.xml b/tests/com.b2international.snowowl.test.dependencies/pom.xml index 02a3212b033..9db68133443 100644 --- a/tests/com.b2international.snowowl.test.dependencies/pom.xml +++ b/tests/com.b2international.snowowl.test.dependencies/pom.xml @@ -16,7 +16,7 @@ org.testcontainers elasticsearch - 1.17.1 + 1.17.3 test