From 09f71b93e492f379218caf8ca7d99c2e9578077d Mon Sep 17 00:00:00 2001 From: Alexander Birkner Date: Fri, 20 Jan 2023 15:44:36 +0100 Subject: [PATCH] doc type is deprecated on es7 see: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/removal-of-types.html --- community-server/build.gradle | 2 +- community-server/dependencies.lock | 115 +++++++++++++++++- .../es7/dao/index/ElasticSearchRestDAOV7.java | 37 ++---- 3 files changed, 119 insertions(+), 35 deletions(-) diff --git a/community-server/build.gradle b/community-server/build.gradle index 0d30a58b6..642bb502e 100644 --- a/community-server/build.gradle +++ b/community-server/build.gradle @@ -22,7 +22,7 @@ dependencies { implementation project(':event-queue:conductor-amqp') implementation project(':event-queue:conductor-nats') - + implementation project(':index:conductor-es7-persistence') implementation project(':external-payload-storage:conductor-azureblob-storage') implementation project(':external-payload-storage:conductor-postgres-external-storage') diff --git a/community-server/dependencies.lock b/community-server/dependencies.lock index 8632fb5de..7f29d0b26 100644 --- a/community-server/dependencies.lock +++ b/community-server/dependencies.lock @@ -29,6 +29,9 @@ "com.netflix.conductor:conductor-es6-persistence": { "locked": "3.13.2" }, + "com.netflix.conductor:conductor-es7-persistence": { + "project": true + }, "com.netflix.conductor:conductor-grpc-server": { "locked": "3.13.2" }, @@ -90,7 +93,7 @@ "locked": "2.17.1" }, "org.springdoc:springdoc-openapi-ui": { - "locked": "1.6.12" + "locked": "1.6.14" }, "org.springframework.boot:spring-boot-starter": { "locked": "2.6.7" @@ -121,6 +124,7 @@ "com.fasterxml.jackson.core:jackson-core": { "firstLevelTransitive": [ "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-mysql-persistence", "com.netflix.conductor:conductor-postgres-persistence" ], @@ -129,6 +133,7 @@ "com.fasterxml.jackson.core:jackson-databind": { "firstLevelTransitive": [ "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-mysql-persistence", "com.netflix.conductor:conductor-postgres-persistence" ], @@ -137,6 +142,7 @@ "com.google.guava:guava": { "firstLevelTransitive": [ "com.netflix.conductor:conductor-amqp", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -165,6 +171,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -177,6 +184,7 @@ }, "com.netflix.conductor:conductor-common-persistence": { "firstLevelTransitive": [ + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-mysql-persistence", "com.netflix.conductor:conductor-postgres-persistence" ], @@ -187,6 +195,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -201,6 +210,9 @@ "com.netflix.conductor:conductor-es6-persistence": { "locked": "3.13.2" }, + "com.netflix.conductor:conductor-es7-persistence": { + "project": true + }, "com.netflix.conductor:conductor-grpc-server": { "locked": "3.13.2" }, @@ -264,6 +276,12 @@ ], "locked": "5.13.1" }, + "commons-io:commons-io": { + "firstLevelTransitive": [ + "com.netflix.conductor:conductor-es7-persistence" + ], + "locked": "2.7" + }, "io.micrometer:micrometer-registry-datadog": { "firstLevelTransitive": [ "com.netflix.conductor:conductor-metrics" @@ -315,6 +333,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -341,6 +360,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -357,6 +377,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -373,6 +394,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -389,6 +411,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -405,6 +428,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -416,6 +440,18 @@ ], "locked": "2.17.1" }, + "org.elasticsearch.client:elasticsearch-rest-client": { + "firstLevelTransitive": [ + "com.netflix.conductor:conductor-es7-persistence" + ], + "locked": "7.15.2" + }, + "org.elasticsearch.client:elasticsearch-rest-high-level-client": { + "firstLevelTransitive": [ + "com.netflix.conductor:conductor-es7-persistence" + ], + "locked": "7.15.2" + }, "org.flywaydb:flyway-core": { "firstLevelTransitive": [ "com.netflix.conductor:conductor-mysql-persistence", @@ -438,7 +474,7 @@ "firstLevelTransitive": [ "com.netflix.conductor:conductor-postgres-external-storage" ], - "locked": "1.6.12" + "locked": "1.6.14" }, "org.springframework.boot:spring-boot-starter": { "locked": "2.6.7" @@ -477,6 +513,7 @@ "com.fasterxml.jackson.core:jackson-core": { "firstLevelTransitive": [ "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-mysql-persistence", "com.netflix.conductor:conductor-postgres-persistence" ], @@ -485,6 +522,7 @@ "com.fasterxml.jackson.core:jackson-databind": { "firstLevelTransitive": [ "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-mysql-persistence", "com.netflix.conductor:conductor-postgres-persistence" ], @@ -493,6 +531,7 @@ "com.google.guava:guava": { "firstLevelTransitive": [ "com.netflix.conductor:conductor-amqp", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -521,6 +560,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -533,6 +573,7 @@ }, "com.netflix.conductor:conductor-common-persistence": { "firstLevelTransitive": [ + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-mysql-persistence", "com.netflix.conductor:conductor-postgres-persistence" ], @@ -543,6 +584,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -557,6 +599,9 @@ "com.netflix.conductor:conductor-es6-persistence": { "locked": "3.13.2" }, + "com.netflix.conductor:conductor-es7-persistence": { + "project": true + }, "com.netflix.conductor:conductor-grpc-server": { "locked": "3.13.2" }, @@ -620,6 +665,12 @@ ], "locked": "5.13.1" }, + "commons-io:commons-io": { + "firstLevelTransitive": [ + "com.netflix.conductor:conductor-es7-persistence" + ], + "locked": "2.7" + }, "io.micrometer:micrometer-registry-datadog": { "firstLevelTransitive": [ "com.netflix.conductor:conductor-metrics" @@ -671,6 +722,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -697,6 +749,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -713,6 +766,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -729,6 +783,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -745,6 +800,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -761,6 +817,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -772,6 +829,18 @@ ], "locked": "2.17.1" }, + "org.elasticsearch.client:elasticsearch-rest-client": { + "firstLevelTransitive": [ + "com.netflix.conductor:conductor-es7-persistence" + ], + "locked": "7.15.2" + }, + "org.elasticsearch.client:elasticsearch-rest-high-level-client": { + "firstLevelTransitive": [ + "com.netflix.conductor:conductor-es7-persistence" + ], + "locked": "7.15.2" + }, "org.flywaydb:flyway-core": { "firstLevelTransitive": [ "com.netflix.conductor:conductor-mysql-persistence", @@ -794,7 +863,7 @@ "firstLevelTransitive": [ "com.netflix.conductor:conductor-postgres-external-storage" ], - "locked": "1.6.12" + "locked": "1.6.14" }, "org.springframework.boot:spring-boot-starter": { "locked": "2.6.7" @@ -854,6 +923,9 @@ "com.netflix.conductor:conductor-es6-persistence": { "locked": "3.13.2" }, + "com.netflix.conductor:conductor-es7-persistence": { + "project": true + }, "com.netflix.conductor:conductor-grpc-server": { "locked": "3.13.2" }, @@ -927,7 +999,7 @@ "locked": "5.8.2" }, "org.springdoc:springdoc-openapi-ui": { - "locked": "1.6.12" + "locked": "1.6.14" }, "org.springframework.boot:spring-boot-starter": { "locked": "2.6.7" @@ -961,6 +1033,7 @@ "com.fasterxml.jackson.core:jackson-core": { "firstLevelTransitive": [ "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-mysql-persistence", "com.netflix.conductor:conductor-postgres-persistence" ], @@ -969,6 +1042,7 @@ "com.fasterxml.jackson.core:jackson-databind": { "firstLevelTransitive": [ "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-mysql-persistence", "com.netflix.conductor:conductor-postgres-persistence" ], @@ -977,6 +1051,7 @@ "com.google.guava:guava": { "firstLevelTransitive": [ "com.netflix.conductor:conductor-amqp", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -1008,6 +1083,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -1020,6 +1096,7 @@ }, "com.netflix.conductor:conductor-common-persistence": { "firstLevelTransitive": [ + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-mysql-persistence", "com.netflix.conductor:conductor-postgres-persistence" ], @@ -1030,6 +1107,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -1044,6 +1122,9 @@ "com.netflix.conductor:conductor-es6-persistence": { "locked": "3.13.2" }, + "com.netflix.conductor:conductor-es7-persistence": { + "project": true + }, "com.netflix.conductor:conductor-grpc-server": { "locked": "3.13.2" }, @@ -1107,6 +1188,12 @@ ], "locked": "5.13.1" }, + "commons-io:commons-io": { + "firstLevelTransitive": [ + "com.netflix.conductor:conductor-es7-persistence" + ], + "locked": "2.7" + }, "io.grpc:grpc-protobuf": { "locked": "1.50.0" }, @@ -1167,6 +1254,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -1193,6 +1281,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -1209,6 +1298,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -1225,6 +1315,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -1241,6 +1332,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -1257,6 +1349,7 @@ "com.netflix.conductor:conductor-amqp", "com.netflix.conductor:conductor-azureblob-storage", "com.netflix.conductor:conductor-common-persistence", + "com.netflix.conductor:conductor-es7-persistence", "com.netflix.conductor:conductor-kafka", "com.netflix.conductor:conductor-metrics", "com.netflix.conductor:conductor-mysql-persistence", @@ -1268,6 +1361,18 @@ ], "locked": "2.17.1" }, + "org.elasticsearch.client:elasticsearch-rest-client": { + "firstLevelTransitive": [ + "com.netflix.conductor:conductor-es7-persistence" + ], + "locked": "7.15.2" + }, + "org.elasticsearch.client:elasticsearch-rest-high-level-client": { + "firstLevelTransitive": [ + "com.netflix.conductor:conductor-es7-persistence" + ], + "locked": "7.15.2" + }, "org.flywaydb:flyway-core": { "firstLevelTransitive": [ "com.netflix.conductor:conductor-mysql-persistence", @@ -1293,7 +1398,7 @@ "firstLevelTransitive": [ "com.netflix.conductor:conductor-postgres-external-storage" ], - "locked": "1.6.12" + "locked": "1.6.14" }, "org.springframework.boot:spring-boot-starter": { "locked": "2.6.7" diff --git a/index/es7-persistence/src/main/java/com/netflix/conductor/es7/dao/index/ElasticSearchRestDAOV7.java b/index/es7-persistence/src/main/java/com/netflix/conductor/es7/dao/index/ElasticSearchRestDAOV7.java index 8e23341d6..752072a51 100644 --- a/index/es7-persistence/src/main/java/com/netflix/conductor/es7/dao/index/ElasticSearchRestDAOV7.java +++ b/index/es7-persistence/src/main/java/com/netflix/conductor/es7/dao/index/ElasticSearchRestDAOV7.java @@ -790,14 +790,7 @@ private SearchResult searchObjectsViaExpression( throws ParserException, IOException { QueryBuilder queryBuilder = boolQueryBuilder(structuredQuery, freeTextQuery); return searchObjects( - getIndexName(docType), - queryBuilder, - start, - size, - sortOptions, - docType, - idOnly, - clazz); + getIndexName(docType), queryBuilder, start, size, sortOptions, idOnly, clazz); } @Override @@ -925,8 +918,7 @@ private SearchResult searchObjectIdsViaExpression( String docType) throws ParserException, IOException { QueryBuilder queryBuilder = boolQueryBuilder(structuredQuery, freeTextQuery); - return searchObjectIds( - getIndexName(docType), queryBuilder, start, size, sortOptions, docType); + return searchObjectIds(getIndexName(docType), queryBuilder, start, size, sortOptions); } private SearchResult searchObjectIdsViaExpression( @@ -940,20 +932,12 @@ private SearchResult searchObjectIdsViaExpression( throws ParserException, IOException { QueryBuilder queryBuilder = boolQueryBuilder(structuredQuery, freeTextQuery); return searchObjects( - getIndexName(docType), - queryBuilder, - start, - size, - sortOptions, - docType, - false, - clazz); + getIndexName(docType), queryBuilder, start, size, sortOptions, false, clazz); } private SearchResult searchObjectIds( - String indexName, QueryBuilder queryBuilder, int start, int size, String docType) - throws IOException { - return searchObjectIds(indexName, queryBuilder, start, size, null, docType); + String indexName, QueryBuilder queryBuilder, int start, int size) throws IOException { + return searchObjectIds(indexName, queryBuilder, start, size, null); } /** @@ -965,7 +949,6 @@ private SearchResult searchObjectIds( * @param size The total return size. * @param sortOptions A list of string options to sort in the form VALUE:ORDER; where ORDER is * optional and can be either ASC OR DESC. - * @param docType The document type to searchObjectIdsViaExpression for. * @return The SearchResults which includes the count and IDs that were found. * @throws IOException If we cannot communicate with ES. */ @@ -974,8 +957,7 @@ private SearchResult searchObjectIds( QueryBuilder queryBuilder, int start, int size, - List sortOptions, - String docType) + List sortOptions) throws IOException { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(queryBuilder); @@ -1014,7 +996,6 @@ private SearchResult searchObjects( int start, int size, List sortOptions, - String docType, boolean idOnly, Class clazz) throws IOException { @@ -1042,7 +1023,6 @@ private SearchResult searchObjects( // Generate the actual request to send to ES. SearchRequest searchRequest = new SearchRequest(indexName); - searchRequest.types(docType); searchRequest.source(searchSourceBuilder); SearchResponse response = elasticSearchClient.search(searchRequest, RequestOptions.DEFAULT); @@ -1101,7 +1081,7 @@ public List searchArchivableWorkflows(String indexName, long archiveTtlD SearchResult workflowIds; try { - workflowIds = searchObjectIds(indexName, q, 0, 1000, WORKFLOW_DOC_TYPE); + workflowIds = searchObjectIds(indexName, q, 0, 1000); } catch (IOException e) { logger.error("Unable to communicate with ES to find archivable workflows", e); return Collections.emptyList(); @@ -1151,8 +1131,7 @@ public List searchRecentRunningWorkflows( q, 0, 5000, - Collections.singletonList("updateTime:ASC"), - WORKFLOW_DOC_TYPE); + Collections.singletonList("updateTime:ASC")); } catch (IOException e) { logger.error("Unable to communicate with ES to find recent running workflows", e); return Collections.emptyList();