From 5d57ce42216eae58ebfc19644de2cf763f8ae4dc Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 24 May 2021 13:47:44 +0100 Subject: [PATCH 01/67] Upgrade to lucene-9.0.0-snapshot-a12260eb950 --- buildSrc/version.properties | 2 +- .../common/settings/KeyStoreWrapperTests.java | 15 +- .../analysis/common/ClassicFilterFactory.java | 2 +- .../common/ClassicTokenizerFactory.java | 2 +- .../analysis/common/CommonAnalysisPlugin.java | 6 +- .../common/UAX29URLEmailTokenizerFactory.java | 2 +- .../common/CommonAnalysisFactoryTests.java | 2 +- .../common/DisableGraphQueryTests.java | 18 +- .../expression/ExpressionScriptEngine.java | 7 +- .../index/query/SourceConfirmedTextQuery.java | 10 +- .../join/query/HasChildQueryBuilder.java | 6 + .../percolator/PercolateQuery.java | 17 +- .../percolator/PercolatorFieldMapper.java | 4 +- .../percolator/CandidateQueryTests.java | 19 +- .../PercolatorFieldMapperTests.java | 4 +- .../percolator/QueryAnalyzerTests.java | 9 +- plugins/analysis-icu/build.gradle | 4 +- .../index/analysis/ICUCollationKeyFilter.java | 2 +- plugins/analysis-kuromoji/build.gradle | 2 +- plugins/analysis-nori/build.gradle | 2 +- plugins/analysis-phonetic/build.gradle | 2 +- plugins/analysis-smartcn/build.gradle | 2 +- plugins/analysis-stempel/build.gradle | 2 +- plugins/analysis-ukrainian/build.gradle | 2 +- .../SmbSimpleFsDirectoryFactory.java | 4 +- .../store/SmbSimpleFSDirectoryTests.java | 7 +- server/build.gradle | 2 +- .../cluster/routing/AllocationIdIT.java | 6 +- .../queries/BinaryDocValuesRangeQuery.java | 8 + .../apache/lucene/queries/MinDocQuery.java | 5 + .../queries/SearchAfterSortedDocQuery.java | 6 + .../lucene/queries/SpanMatchNoDocsQuery.java | 10 +- .../lucene/search/XCombinedFieldQuery.java | 466 ------------------ .../search/XMultiNormsLeafSimScorer.java | 161 ------ .../vectorhighlight/CustomFieldQuery.java | 6 +- .../main/java/org/elasticsearch/Version.java | 4 +- .../action/search/SearchPhaseController.java | 7 +- .../action/search/TransportSearchHelper.java | 8 +- .../elasticsearch/common/geo/GeoUtils.java | 4 +- .../elasticsearch/common/lucene/Lucene.java | 18 +- .../lucene/search/MoreLikeThisQuery.java | 6 + .../lucene/search/MultiPhrasePrefixQuery.java | 8 + .../search/function/FunctionScoreQuery.java | 7 - .../search/function/ScriptScoreQuery.java | 15 +- .../common/settings/KeyStoreWrapper.java | 6 +- .../common/util/CuckooFilter.java | 9 +- .../elasticsearch/env/NodeEnvironment.java | 4 +- .../gateway/MetadataStateFormat.java | 4 +- .../gateway/PersistedClusterStateService.java | 12 +- .../index/codec/CodecService.java | 11 +- .../PerFieldMappingPostingFormatCodec.java | 10 +- .../elasticsearch/index/engine/Engine.java | 12 - .../engine/PrunePostingsMergePolicy.java | 5 - .../engine/RamAccountingRefreshListener.java | 2 +- .../RecoverySourcePruneMergePolicy.java | 12 +- .../engine/SingleDocDirectoryReader.java | 12 + .../index/engine/TranslogLeafReader.java | 21 +- .../plain/PagedBytesIndexFieldData.java | 4 +- .../plain/SortedNumericIndexFieldData.java | 2 +- .../index/fieldvisitor/FieldsVisitor.java | 4 +- .../fieldvisitor/SingleFieldsVisitor.java | 7 +- .../index/get/ShardGetService.java | 3 +- .../index/mapper/DateFieldMapper.java | 2 +- .../index/mapper/DocumentLeafReader.java | 17 +- .../index/mapper/NumberFieldMapper.java | 4 +- .../query/CombinedFieldsQueryBuilder.java | 6 +- .../index/query/ScriptQueryBuilder.java | 6 + .../index/query/TermsSetQueryBuilder.java | 2 +- .../index/search/MatchQueryParser.java | 5 - .../index/search/QueryStringQueryParser.java | 9 +- .../search/SimpleQueryStringQueryParser.java | 9 +- .../index/shard/ShardSplittingQuery.java | 13 +- .../index/shard/StoreRecovery.java | 3 +- .../index/similarity/SimilarityProviders.java | 6 +- .../index/similarity/SimilarityService.java | 2 +- .../index/store/FsDirectoryFactory.java | 3 +- .../org/elasticsearch/index/store/Store.java | 14 +- .../index/translog/Checkpoint.java | 4 +- .../indices/IndicesQueryCache.java | 6 - .../indices/analysis/HunspellService.java | 6 +- .../indices/analysis/PreBuiltAnalyzers.java | 2 +- .../elasticsearch/plugins/PluginsService.java | 18 +- .../composite/PointsSortedDocsProducer.java | 6 +- .../bucket/filter/MergedPointRangeQuery.java | 8 + .../bucket/filter/QueryToFilterAdapter.java | 2 +- ...DiversifiedBytesHashSamplerAggregator.java | 5 +- .../DiversifiedMapSamplerAggregator.java | 5 +- .../DiversifiedNumericSamplerAggregator.java | 5 +- .../DiversifiedOrdinalsSamplerAggregator.java | 5 +- .../bucket/sampler/SamplerAggregator.java | 2 +- .../aggregations/metrics/InternalTopHits.java | 4 +- .../aggregations/metrics/MaxAggregator.java | 4 +- .../search/internal/ContextIndexSearcher.java | 5 - .../search/profile/query/ProfileWeight.java | 7 - .../runtime/AbstractScriptFieldQuery.java | 8 + ...oPointScriptFieldDistanceFeatureQuery.java | 5 - .../LongScriptFieldDistanceFeatureQuery.java | 5 - .../search/slice/SliceQuery.java | 7 + .../lucene/queries/BlendedTermQueryTests.java | 6 +- ...sRandomBinaryDocValuesRangeQueryTests.java | 19 +- .../search/XCombinedFieldQueryTests.java | 343 ------------- .../common/lucene/LuceneTests.java | 11 +- .../search/function/MinScoreScorerTests.java | 6 - .../gateway/MetadataStateFormatTests.java | 4 +- .../PersistedClusterStateServiceTests.java | 12 +- .../elasticsearch/index/codec/CodecTests.java | 42 +- .../engine/CompletionStatsCacheTests.java | 4 +- .../index/engine/SegmentTests.java | 2 +- .../index/mapper/DateFieldTypeTests.java | 2 +- .../index/mapper/NumberFieldTypeTests.java | 4 +- .../CombinedFieldsQueryBuilderTests.java | 4 +- .../CombinedFieldsQueryParsingTests.java | 32 +- .../index/query/DisMaxQueryBuilderTests.java | 33 +- .../MatchBoolPrefixQueryBuilderTests.java | 5 +- .../query/MultiMatchQueryBuilderTests.java | 53 +- .../query/QueryStringQueryBuilderTests.java | 73 +-- .../query/SimpleQueryStringBuilderTests.java | 24 +- .../query/TermsSetQueryBuilderTests.java | 2 +- .../query/plugin/DummyQueryParserPlugin.java | 6 + .../search/MultiMatchQueryParserTests.java | 10 +- .../similarity/ScriptedSimilarityTests.java | 3 +- .../similarity/SimilarityServiceTests.java | 2 +- .../index/similarity/SimilarityTests.java | 2 +- .../index/store/FsDirectoryFactoryTests.java | 5 +- .../elasticsearch/index/store/StoreTests.java | 4 +- .../indices/IndicesQueryCacheTests.java | 15 +- .../indices/analysis/AnalysisModuleTests.java | 4 +- .../metrics/MaxAggregatorTests.java | 3 +- .../internal/ContextIndexSearcherTests.java | 11 +- .../lookup/LeafStoredFieldsLookupTests.java | 3 +- .../profile/query/QueryProfilerTests.java | 12 +- .../search/sort/FieldSortBuilderTests.java | 4 +- .../analysis/AnalysisFactoryTestCase.java | 10 +- .../aggregations/AggregatorTestCase.java | 2 +- .../hamcrest/ElasticsearchAssertions.java | 9 - .../xpack/search/BlockingQueryBuilder.java | 6 + .../xpack/search/ThrowingQueryBuilder.java | 6 + .../engine/RewriteCachingDirectoryReader.java | 12 + .../snapshots/SourceOnlySnapshot.java | 9 +- .../SourceOnlySnapshotRepository.java | 4 +- .../accesscontrol/FieldSubsetReader.java | 7 +- .../accesscontrol/FieldSubsetReaderTests.java | 2 +- .../unsignedlong/UnsignedLongFieldMapper.java | 2 +- .../xpack/rollup/v2/RollupShardIndexer.java | 3 +- .../lucene/search/CappedScoreWeight.java | 7 - .../store/SearchableSnapshotDirectory.java | 22 +- .../input/DirectBlobContainerIndexInput.java | 15 +- .../DirectBlobContainerIndexInputTests.java | 3 +- .../authz/accesscontrol/FieldExtractor.java | 7 +- .../accesscontrol/FieldExtractorTests.java | 7 +- .../xpack/spatial/SpatialUtils.java | 8 +- .../org/apache/lucene/geo/XShapeTestUtil.java | 11 +- .../xpack/sql/qa/mixed_node/SqlSearchIT.java | 2 +- .../mapper/AutomatonQueryOnBinaryDv.java | 7 + 154 files changed, 618 insertions(+), 1569 deletions(-) delete mode 100644 server/src/main/java/org/apache/lucene/search/XCombinedFieldQuery.java delete mode 100644 server/src/main/java/org/apache/lucene/search/XMultiNormsLeafSimScorer.java delete mode 100644 server/src/test/java/org/apache/lucene/search/XCombinedFieldQueryTests.java diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 07445fe501554..d37b5ce02939a 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ elasticsearch = 8.0.0 -lucene = 8.9.0-snapshot-efdc43fee18 +lucene = 9.0.0-snapshot-a12260eb950 bundled_jdk_vendor = adoptopenjdk bundled_jdk = 16.0.1+9 diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java index a2fdd902338f5..5097c21ef93ce 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java @@ -11,7 +11,7 @@ import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.store.NIOFSDirectory; import org.elasticsearch.common.Randomness; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.Environment; @@ -27,7 +27,6 @@ import javax.crypto.spec.GCMParameterSpec; import javax.crypto.spec.PBEKeySpec; import javax.crypto.spec.SecretKeySpec; - import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.EOFException; @@ -198,7 +197,7 @@ public void testUpgradeNoop() throws Exception { public void testFailWhenCannotConsumeSecretStream() throws Exception { assumeFalse("Cannot open unprotected keystore on FIPS JVM", inFipsJvm()); Path configDir = env.configFile(); - SimpleFSDirectory directory = new SimpleFSDirectory(configDir); + NIOFSDirectory directory = new NIOFSDirectory(configDir); try (IndexOutput indexOutput = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)) { CodecUtil.writeHeader(indexOutput, "elasticsearch.keystore", 3); indexOutput.writeByte((byte) 0); // No password @@ -227,7 +226,7 @@ public void testFailWhenCannotConsumeSecretStream() throws Exception { public void testFailWhenCannotConsumeEncryptedBytesStream() throws Exception { assumeFalse("Cannot open unprotected keystore on FIPS JVM", inFipsJvm()); Path configDir = env.configFile(); - SimpleFSDirectory directory = new SimpleFSDirectory(configDir); + NIOFSDirectory directory = new NIOFSDirectory(configDir); try (IndexOutput indexOutput = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)) { CodecUtil.writeHeader(indexOutput, "elasticsearch.keystore", 3); indexOutput.writeByte((byte) 0); // No password @@ -257,7 +256,7 @@ public void testFailWhenCannotConsumeEncryptedBytesStream() throws Exception { public void testFailWhenSecretStreamNotConsumed() throws Exception { assumeFalse("Cannot open unprotected keystore on FIPS JVM", inFipsJvm()); Path configDir = env.configFile(); - SimpleFSDirectory directory = new SimpleFSDirectory(configDir); + NIOFSDirectory directory = new NIOFSDirectory(configDir); try (IndexOutput indexOutput = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)) { CodecUtil.writeHeader(indexOutput, "elasticsearch.keystore", 3); indexOutput.writeByte((byte) 0); // No password @@ -285,7 +284,7 @@ public void testFailWhenSecretStreamNotConsumed() throws Exception { public void testFailWhenEncryptedBytesStreamIsNotConsumed() throws Exception { assumeFalse("Cannot open unprotected keystore on FIPS JVM", inFipsJvm()); Path configDir = env.configFile(); - SimpleFSDirectory directory = new SimpleFSDirectory(configDir); + NIOFSDirectory directory = new NIOFSDirectory(configDir); try (IndexOutput indexOutput = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)) { CodecUtil.writeHeader(indexOutput, "elasticsearch.keystore", 3); indexOutput.writeByte((byte) 0); // No password @@ -372,7 +371,7 @@ public void testIllegalSettingName() throws Exception { public void testBackcompatV1() throws Exception { assumeFalse("Can't run in a FIPS JVM as PBE is not available", inFipsJvm()); Path configDir = env.configFile(); - SimpleFSDirectory directory = new SimpleFSDirectory(configDir); + NIOFSDirectory directory = new NIOFSDirectory(configDir); try (IndexOutput output = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)) { CodecUtil.writeHeader(output, "elasticsearch.keystore", 1); output.writeByte((byte) 0); // hasPassword = false @@ -403,7 +402,7 @@ public void testBackcompatV1() throws Exception { public void testBackcompatV2() throws Exception { assumeFalse("Can't run in a FIPS JVM as PBE is not available", inFipsJvm()); Path configDir = env.configFile(); - SimpleFSDirectory directory = new SimpleFSDirectory(configDir); + NIOFSDirectory directory = new NIOFSDirectory(configDir); byte[] fileBytes = new byte[20]; random().nextBytes(fileBytes); try (IndexOutput output = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)) { diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicFilterFactory.java index 5ff0134f7807f..eb7658fed1274 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicFilterFactory.java @@ -8,7 +8,7 @@ package org.elasticsearch.analysis.common; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.standard.ClassicFilter; +import org.apache.lucene.analysis.classic.ClassicFilter; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicTokenizerFactory.java index c9b997f03cebf..580597bbc2442 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicTokenizerFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicTokenizerFactory.java @@ -9,7 +9,7 @@ package org.elasticsearch.analysis.common; import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.analysis.standard.ClassicTokenizer; +import org.apache.lucene.analysis.classic.ClassicTokenizer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java index 9e544ae0b1e51..5978e013d2be1 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java @@ -27,6 +27,8 @@ import org.apache.lucene.analysis.cjk.CJKWidthFilter; import org.apache.lucene.analysis.ckb.SoraniAnalyzer; import org.apache.lucene.analysis.ckb.SoraniNormalizationFilter; +import org.apache.lucene.analysis.classic.ClassicFilter; +import org.apache.lucene.analysis.classic.ClassicTokenizer; import org.apache.lucene.analysis.commongrams.CommonGramsFilter; import org.apache.lucene.analysis.core.DecimalDigitFilter; import org.apache.lucene.analysis.core.KeywordTokenizer; @@ -40,6 +42,7 @@ import org.apache.lucene.analysis.de.GermanNormalizationFilter; import org.apache.lucene.analysis.de.GermanStemFilter; import org.apache.lucene.analysis.el.GreekAnalyzer; +import org.apache.lucene.analysis.email.UAX29URLEmailTokenizer; import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.apache.lucene.analysis.en.KStemFilter; import org.apache.lucene.analysis.en.PorterStemFilter; @@ -89,10 +92,7 @@ import org.apache.lucene.analysis.ru.RussianAnalyzer; import org.apache.lucene.analysis.shingle.ShingleFilter; import org.apache.lucene.analysis.snowball.SnowballFilter; -import org.apache.lucene.analysis.standard.ClassicFilter; -import org.apache.lucene.analysis.standard.ClassicTokenizer; import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer; import org.apache.lucene.analysis.sv.SwedishAnalyzer; import org.apache.lucene.analysis.th.ThaiAnalyzer; import org.apache.lucene.analysis.th.ThaiTokenizer; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UAX29URLEmailTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UAX29URLEmailTokenizerFactory.java index 3f5335da5906e..2391c36b90a29 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UAX29URLEmailTokenizerFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UAX29URLEmailTokenizerFactory.java @@ -9,8 +9,8 @@ package org.elasticsearch.analysis.common; import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.email.UAX29URLEmailTokenizer; import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java index d001a89e10f25..35f8b705ff5bf 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java @@ -212,7 +212,7 @@ protected Map> getPreConfiguredTokenizers() { tokenizers.put("keyword", null); tokenizers.put("lowercase", Void.class); tokenizers.put("classic", null); - tokenizers.put("uax_url_email", org.apache.lucene.analysis.standard.UAX29URLEmailTokenizerFactory.class); + tokenizers.put("uax_url_email", org.apache.lucene.analysis.email.UAX29URLEmailTokenizerFactory.class); tokenizers.put("path_hierarchy", null); tokenizers.put("letter", null); tokenizers.put("whitespace", null); diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java index 8efde7e6b1e03..776e53f4edee2 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java @@ -80,15 +80,17 @@ public void setup() { // that ignores position length attribute expectedQueryWithUnigram= new BooleanQuery.Builder() .add( - new SynonymQuery( - new Term("text_shingle_unigram", "foo"), - new Term("text_shingle_unigram", "foo bar") - ), BooleanClause.Occur.SHOULD) + new SynonymQuery.Builder("text_shingle_unigram") + .addTerm(new Term("text_shingle_unigram", "foo")) + .addTerm(new Term("text_shingle_unigram", "foo bar")) + .build(), + BooleanClause.Occur.SHOULD) .add( - new SynonymQuery( - new Term("text_shingle_unigram", "bar"), - new Term("text_shingle_unigram", "bar baz") - ), BooleanClause.Occur.SHOULD) + new SynonymQuery.Builder("text_shingle_unigram") + .addTerm(new Term("text_shingle_unigram", "bar")) + .addTerm(new Term("text_shingle_unigram", "bar baz")) + .build(), + BooleanClause.Occur.SHOULD) .add( new TermQuery( new Term("text_shingle_unigram", "baz") diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java index 9a00334f6b911..d74bc6f1d6a7d 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java @@ -13,7 +13,6 @@ import org.apache.lucene.expressions.js.JavascriptCompiler; import org.apache.lucene.expressions.js.VariableContext; import org.apache.lucene.search.DoubleValuesSource; -import org.apache.lucene.search.SortField; import org.elasticsearch.SpecialPermission; import org.elasticsearch.common.Nullable; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -232,7 +231,7 @@ private static NumberSortScript.LeafFactory newSortScript(Expression expr, Searc for (String variable : expr.variables) { try { if (variable.equals("_score")) { - bindings.add(new SortField("_score", SortField.Type.SCORE)); + bindings.add("_score", DoubleValuesSource.SCORES); needsScores = true; } else if (vars != null && vars.containsKey(variable)) { bindFromParams(vars, bindings, variable); @@ -283,7 +282,7 @@ private static AggregationScript.LeafFactory newAggregationScript(Expression exp for (String variable : expr.variables) { try { if (variable.equals("_score")) { - bindings.add(new SortField("_score", SortField.Type.SCORE)); + bindings.add("_score", DoubleValuesSource.SCORES); needsScores = true; } else if (variable.equals("_value")) { specialValue = new ReplaceableConstDoubleValueSource(); @@ -355,7 +354,7 @@ private static ScoreScript.LeafFactory newScoreScript(Expression expr, SearchLoo for (String variable : expr.variables) { try { if (variable.equals("_score")) { - bindings.add(new SortField("_score", SortField.Type.SCORE)); + bindings.add("_score", DoubleValuesSource.SCORES); needsScores = true; } else if (variable.equals("_value")) { specialValue = new ReplaceableConstDoubleValueSource(); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/SourceConfirmedTextQuery.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/SourceConfirmedTextQuery.java index baba8b0345f1f..180d5c7faf73a 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/SourceConfirmedTextQuery.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/SourceConfirmedTextQuery.java @@ -183,6 +183,11 @@ public int hashCode() { return 31 * Objects.hash(in, valueFetcherProvider, indexAnalyzer) + classHash(); } + @Override + public void visit(QueryVisitor visitor) { + in.visit(visitor.getSubVisitor(Occur.MUST, this)); + } + @Override public Query rewrite(IndexReader reader) throws IOException { Query inRewritten = in.rewrite(reader); @@ -244,11 +249,6 @@ public boolean isCacheable(LeafReaderContext ctx) { return false; } - @Override - public void extractTerms(Set termSet) { - termSet.addAll(terms); - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { RuntimePhraseScorer scorer = scorer(context); diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java index 95266e8ddd562..623cd540cadf6 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java @@ -13,6 +13,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.join.JoinUtil; import org.apache.lucene.search.join.ScoreMode; import org.apache.lucene.search.similarities.Similarity; @@ -411,6 +412,11 @@ public String toString(String s) { return "LateParsingQuery {joinField=" + joinField + "}"; } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + public int getMinChildren() { return minChildren; } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java index 06bdb1e845230..eedb451d8bdd2 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java @@ -10,19 +10,19 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.ScorerSupplier; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; -import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Bits; import org.elasticsearch.common.CheckedFunction; @@ -32,7 +32,6 @@ import java.io.IOException; import java.util.List; import java.util.Objects; -import java.util.Set; final class PercolateQuery extends Query implements Accountable { @@ -75,10 +74,6 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo final Weight verifiedMatchesWeight = verifiedMatchesQuery.createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, boost); final Weight candidateMatchesWeight = candidateMatchesQuery.createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, boost); return new Weight(this) { - @Override - public void extractTerms(Set set) { - } - @Override public Explanation explain(LeafReaderContext leafReaderContext, int docId) throws IOException { Scorer scorer = scorer(leafReaderContext); @@ -239,6 +234,12 @@ public String toString(String s) { candidateMatchesQuery.toString(s) + "}}"; } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + + @Override public long ramBytesUsed() { long ramUsed = 0L; diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index a5174998ff345..0f89b267874d2 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -19,9 +19,9 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.sandbox.search.CoveringQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.CoveringQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LongValuesSource; import org.apache.lucene.search.MatchNoDocsQuery; @@ -61,9 +61,9 @@ import org.elasticsearch.index.query.ConstantScoreQueryBuilder; import org.elasticsearch.index.query.DisMaxQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.Rewriteable; +import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; import java.io.ByteArrayOutputStream; diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java index 6de96ac9c5f07..ede6e9cb39f6f 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java @@ -13,7 +13,6 @@ import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; import org.apache.lucene.document.FloatPoint; -import org.apache.lucene.document.HalfFloatPoint; import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; @@ -35,10 +34,11 @@ import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.memory.MemoryIndex; import org.apache.lucene.queries.BlendedTermQuery; +import org.apache.lucene.sandbox.document.HalfFloatPoint; +import org.apache.lucene.sandbox.search.CoveringQuery; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.CoveringQuery; import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; @@ -49,6 +49,7 @@ import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Sort; @@ -96,7 +97,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -1148,6 +1148,11 @@ public boolean equals(Object obj) { public int hashCode() { return classHash(); } + + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } } private static final class ControlQuery extends Query { @@ -1165,9 +1170,6 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo final IndexSearcher percolatorIndexSearcher = memoryIndex.createSearcher(); return new Weight(this) { - @Override - public void extractTerms(Set terms) {} - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { Scorer scorer = scorer(context); @@ -1256,6 +1258,11 @@ public int hashCode() { return classHash(); } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + } } diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java index 09c08ba2fda98..322228ceba784 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java @@ -11,7 +11,6 @@ import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.FloatPoint; -import org.apache.lucene.document.HalfFloatPoint; import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; @@ -19,9 +18,10 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; import org.apache.lucene.index.memory.MemoryIndex; +import org.apache.lucene.sandbox.document.HalfFloatPoint; +import org.apache.lucene.sandbox.search.CoveringQuery; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.CoveringQuery; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermInSetQuery; diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java index 3160c8eaf9a83..8818cbff32573 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java @@ -9,7 +9,6 @@ import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.FloatPoint; -import org.apache.lucene.document.HalfFloatPoint; import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LatLonPoint; @@ -20,6 +19,7 @@ import org.apache.lucene.queries.intervals.IntervalQuery; import org.apache.lucene.queries.intervals.Intervals; import org.apache.lucene.queries.intervals.IntervalsSource; +import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; @@ -799,13 +799,16 @@ public void testExtractQueryMetadata_disjunctionMaxQuery() { } public void testSynonymQuery() { - SynonymQuery query = new SynonymQuery(); + SynonymQuery query = new SynonymQuery.Builder("field").build(); Result result = analyze(query); assertThat(result.verified, is(true)); assertThat(result.minimumShouldMatch, equalTo(0)); assertThat(result.extractions.isEmpty(), is(true)); - query = new SynonymQuery(new Term("_field", "_value1"), new Term("_field", "_value2")); + query = new SynonymQuery.Builder("_field") + .addTerm(new Term("_field", "_value1")) + .addTerm(new Term("_field", "_value2")) + .build(); result = analyze(query); assertThat(result.verified, is(true)); assertThat(result.minimumShouldMatch, equalTo(1)); diff --git a/plugins/analysis-icu/build.gradle b/plugins/analysis-icu/build.gradle index 489d6a84f3898..7521233e065c3 100644 --- a/plugins/analysis-icu/build.gradle +++ b/plugins/analysis-icu/build.gradle @@ -1,5 +1,3 @@ -import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis - /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License @@ -23,7 +21,7 @@ tasks.named("forbiddenApisMain").configure { } dependencies { - api "org.apache.lucene:lucene-analyzers-icu:${versions.lucene}" + api "org.apache.lucene:lucene-analysis-icu:${versions.lucene}" api "com.ibm.icu:icu4j:${versions.icu4j}" } diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/ICUCollationKeyFilter.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/ICUCollationKeyFilter.java index 4c83862edba3b..ab89157b923b0 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/ICUCollationKeyFilter.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/ICUCollationKeyFilter.java @@ -22,7 +22,7 @@ import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.collation.ICUCollationDocValuesField; +import org.apache.lucene.analysis.icu.ICUCollationDocValuesField; import java.io.IOException; diff --git a/plugins/analysis-kuromoji/build.gradle b/plugins/analysis-kuromoji/build.gradle index a21f7bda30775..fc58b919c727a 100644 --- a/plugins/analysis-kuromoji/build.gradle +++ b/plugins/analysis-kuromoji/build.gradle @@ -14,7 +14,7 @@ esplugin { } dependencies { - api "org.apache.lucene:lucene-analyzers-kuromoji:${versions.lucene}" + api "org.apache.lucene:lucene-analysis-kuromoji:${versions.lucene}" } restResources { diff --git a/plugins/analysis-nori/build.gradle b/plugins/analysis-nori/build.gradle index 046875bdea7c4..161e46f4a6958 100644 --- a/plugins/analysis-nori/build.gradle +++ b/plugins/analysis-nori/build.gradle @@ -14,7 +14,7 @@ esplugin { } dependencies { - api "org.apache.lucene:lucene-analyzers-nori:${versions.lucene}" + api "org.apache.lucene:lucene-analysis-nori:${versions.lucene}" } restResources { diff --git a/plugins/analysis-phonetic/build.gradle b/plugins/analysis-phonetic/build.gradle index 0479b1215d142..874b4e3d9af95 100644 --- a/plugins/analysis-phonetic/build.gradle +++ b/plugins/analysis-phonetic/build.gradle @@ -14,7 +14,7 @@ esplugin { } dependencies { - api "org.apache.lucene:lucene-analyzers-phonetic:${versions.lucene}" + api "org.apache.lucene:lucene-analysis-phonetic:${versions.lucene}" api "commons-codec:commons-codec:${versions.commonscodec}" } diff --git a/plugins/analysis-smartcn/build.gradle b/plugins/analysis-smartcn/build.gradle index e01b3e8347fae..1f2b7aae79ce8 100644 --- a/plugins/analysis-smartcn/build.gradle +++ b/plugins/analysis-smartcn/build.gradle @@ -14,7 +14,7 @@ esplugin { } dependencies { - api "org.apache.lucene:lucene-analyzers-smartcn:${versions.lucene}" + api "org.apache.lucene:lucene-analysis-smartcn:${versions.lucene}" } restResources { diff --git a/plugins/analysis-stempel/build.gradle b/plugins/analysis-stempel/build.gradle index bc5e973b9d980..b9307ddfd52aa 100644 --- a/plugins/analysis-stempel/build.gradle +++ b/plugins/analysis-stempel/build.gradle @@ -14,7 +14,7 @@ esplugin { } dependencies { - api "org.apache.lucene:lucene-analyzers-stempel:${versions.lucene}" + api "org.apache.lucene:lucene-analysis-stempel:${versions.lucene}" } restResources { diff --git a/plugins/analysis-ukrainian/build.gradle b/plugins/analysis-ukrainian/build.gradle index a8f0a65df1b47..cbadd2d37b0cb 100644 --- a/plugins/analysis-ukrainian/build.gradle +++ b/plugins/analysis-ukrainian/build.gradle @@ -14,7 +14,7 @@ esplugin { } dependencies { - api "org.apache.lucene:lucene-analyzers-morfologik:${versions.lucene}" + api "org.apache.lucene:lucene-analysis-morfologik:${versions.lucene}" api "org.carrot2:morfologik-stemming:2.1.1" api "org.carrot2:morfologik-fsa:2.1.1" api "ua.net.nlp:morfologik-ukrainian-search:3.7.5" diff --git a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbsimplefs/SmbSimpleFsDirectoryFactory.java b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbsimplefs/SmbSimpleFsDirectoryFactory.java index 83d0772c99380..b9053c5c9f8f8 100644 --- a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbsimplefs/SmbSimpleFsDirectoryFactory.java +++ b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbsimplefs/SmbSimpleFsDirectoryFactory.java @@ -10,7 +10,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.LockFactory; -import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.store.NIOFSDirectory; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.store.FsDirectoryFactory; import org.elasticsearch.index.store.SmbDirectoryWrapper; @@ -22,6 +22,6 @@ public final class SmbSimpleFsDirectoryFactory extends FsDirectoryFactory { @Override protected Directory newFSDirectory(Path location, LockFactory lockFactory, IndexSettings indexSettings) throws IOException { - return new SmbDirectoryWrapper(new SimpleFSDirectory(location, lockFactory)); + return new SmbDirectoryWrapper(new NIOFSDirectory(location, lockFactory)); } } diff --git a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbSimpleFSDirectoryTests.java b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbSimpleFSDirectoryTests.java index 2b09a9c5ae729..2f81a2021495b 100644 --- a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbSimpleFSDirectoryTests.java +++ b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbSimpleFSDirectoryTests.java @@ -8,16 +8,17 @@ package org.elasticsearch.index.store; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.NIOFSDirectory; + import java.io.IOException; import java.nio.file.Path; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.SimpleFSDirectory; public class SmbSimpleFSDirectoryTests extends EsBaseDirectoryTestCase { @Override protected Directory getDirectory(Path file) throws IOException { - return new SmbDirectoryWrapper(new SimpleFSDirectory(file)); + return new SmbDirectoryWrapper(new NIOFSDirectory(file)); } @Override diff --git a/server/build.gradle b/server/build.gradle index 7917d74da9abf..7e6d52155808f 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -32,7 +32,7 @@ dependencies { // lucene api "org.apache.lucene:lucene-core:${versions.lucene}" - api "org.apache.lucene:lucene-analyzers-common:${versions.lucene}" + api "org.apache.lucene:lucene-analysis-common:${versions.lucene}" api "org.apache.lucene:lucene-backward-codecs:${versions.lucene}" api "org.apache.lucene:lucene-grouping:${versions.lucene}" api "org.apache.lucene:lucene-highlighter:${versions.lucene}" diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java index f483fe58f4626..77e832d6e21fd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java @@ -8,7 +8,7 @@ package org.elasticsearch.cluster.routing; -import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.store.NIOFSDirectory; import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanation; import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.index.IndexRequestBuilder; @@ -126,7 +126,7 @@ public void testFailedRecoveryOnAllocateStalePrimaryRequiresAnotherAllocateStale }); internalCluster().stopNode(node1); - try(Store store = new Store(shardId, indexSettings, new SimpleFSDirectory(indexPath), new DummyShardLock(shardId))) { + try(Store store = new Store(shardId, indexSettings, new NIOFSDirectory(indexPath), new DummyShardLock(shardId))) { store.removeCorruptionMarker(); } node1 = internalCluster().startNode(node1DataPathSettings); @@ -204,7 +204,7 @@ private String historyUUID(String node, String indexName) { } private void putFakeCorruptionMarker(IndexSettings indexSettings, ShardId shardId, Path indexPath) throws IOException { - try(Store store = new Store(shardId, indexSettings, new SimpleFSDirectory(indexPath), new DummyShardLock(shardId))) { + try(Store store = new Store(shardId, indexSettings, new NIOFSDirectory(indexPath), new DummyShardLock(shardId))) { store.markStoreCorrupted(new IOException("fake ioexception")); } } diff --git a/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java b/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java index 0303c5cd7157a..804342cbe9326 100644 --- a/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java +++ b/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java @@ -15,6 +15,7 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TwoPhaseIterator; @@ -108,6 +109,13 @@ public boolean isCacheable(LeafReaderContext ctx) { }; } + @Override + public void visit(QueryVisitor visitor) { + if (visitor.acceptField(fieldName)) { + visitor.visitLeaf(this); + } + } + @Override public String toString(String field) { return "BinaryDocValuesRangeQuery(fieldName=" + field + ",from=" + originalFrom + ",to=" + originalTo + ")"; diff --git a/server/src/main/java/org/apache/lucene/queries/MinDocQuery.java b/server/src/main/java/org/apache/lucene/queries/MinDocQuery.java index b2c1100fb73d0..38d19b3b58277 100644 --- a/server/src/main/java/org/apache/lucene/queries/MinDocQuery.java +++ b/server/src/main/java/org/apache/lucene/queries/MinDocQuery.java @@ -15,6 +15,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; @@ -136,6 +137,10 @@ public long cost() { } } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } @Override public String toString(String field) { diff --git a/server/src/main/java/org/apache/lucene/queries/SearchAfterSortedDocQuery.java b/server/src/main/java/org/apache/lucene/queries/SearchAfterSortedDocQuery.java index cd8248229972e..b8ee252a49d90 100644 --- a/server/src/main/java/org/apache/lucene/queries/SearchAfterSortedDocQuery.java +++ b/server/src/main/java/org/apache/lucene/queries/SearchAfterSortedDocQuery.java @@ -17,6 +17,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafFieldComparator; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Sort; @@ -112,6 +113,11 @@ public int hashCode() { return Objects.hash(classHash(), sort, after.doc, after.score, Arrays.hashCode(after.fields)); } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + interface TopComparator { boolean lessThanTop(int doc) throws IOException; } diff --git a/server/src/main/java/org/apache/lucene/queries/SpanMatchNoDocsQuery.java b/server/src/main/java/org/apache/lucene/queries/SpanMatchNoDocsQuery.java index acae5073afa39..a7d11cc9357dc 100644 --- a/server/src/main/java/org/apache/lucene/queries/SpanMatchNoDocsQuery.java +++ b/server/src/main/java/org/apache/lucene/queries/SpanMatchNoDocsQuery.java @@ -11,6 +11,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.TermStates; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.search.spans.SpanWeight; @@ -19,7 +20,6 @@ import java.io.IOException; import java.util.Collections; import java.util.Map; -import java.util.Set; /** * A {@link SpanQuery} that matches no documents. @@ -53,6 +53,11 @@ public int hashCode() { return classHash(); } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + @Override public SpanWeight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new SpanWeight(this, searcher, Collections.emptyMap(), boost) { @@ -64,9 +69,6 @@ public Spans getSpans(LeafReaderContext ctx, Postings requiredPostings) { return null; } - @Override - public void extractTerms(Set terms) {} - @Override public boolean isCacheable(LeafReaderContext ctx) { return true; diff --git a/server/src/main/java/org/apache/lucene/search/XCombinedFieldQuery.java b/server/src/main/java/org/apache/lucene/search/XCombinedFieldQuery.java deleted file mode 100644 index 03daa65223950..0000000000000 --- a/server/src/main/java/org/apache/lucene/search/XCombinedFieldQuery.java +++ /dev/null @@ -1,466 +0,0 @@ -/* @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.lucene.search; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermState; -import org.apache.lucene.index.TermStates; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.search.similarities.BM25Similarity; -import org.apache.lucene.search.similarities.DFRSimilarity; -import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.search.similarities.SimilarityBase; -import org.apache.lucene.util.Accountable; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.RamUsageEstimator; -import org.apache.lucene.util.SmallFloat; - -/** - * A {@link Query} that treats multiple fields as a single stream and scores terms as if you had - * indexed them as a single term in a single field. - * - *

The query works as follows: - * - *

    - *
  1. Given a list of fields and weights, it pretends there is a synthetic combined field where - * all terms have been indexed. It computes new term and collection statistics for this - * combined field. - *
  2. It uses a disjunction iterator and {@link IndexSearcher#getSimilarity} to score documents. - *
- * - *

In order for a similarity to be compatible, {@link Similarity#computeNorm} must be additive: - * the norm of the combined field is the sum of norms for each individual field. The norms must also - * be encoded using {@link SmallFloat#intToByte4}. These requirements hold for all similarities that - * compute norms the same way as {@link SimilarityBase#computeNorm}, which includes {@link - * BM25Similarity} and {@link DFRSimilarity}. Per-field similarities are not supported. - * - *

The scoring is based on BM25F's simple formula described in: - * http://www.staff.city.ac.uk/~sb317/papers/foundations_bm25_review.pdf. This query implements the - * same approach but allows other similarities besides {@link - * org.apache.lucene.search.similarities.BM25Similarity}. - * - * TODO: this is temporarily copied from Lucene, remove once we update to Lucene 8.9. - */ -public final class XCombinedFieldQuery extends Query implements Accountable { - private static final long BASE_RAM_BYTES = - RamUsageEstimator.shallowSizeOfInstance(XCombinedFieldQuery.class); - - /** A builder for {@link XCombinedFieldQuery}. */ - public static class Builder { - private final Map fieldAndWeights = new HashMap<>(); - private final Set termsSet = new HashSet<>(); - - /** - * Adds a field to this builder. - * - * @param field The field name. - */ - public Builder addField(String field) { - return addField(field, 1f); - } - - /** - * Adds a field to this builder. - * - * @param field The field name. - * @param weight The weight associated to this field. - */ - public Builder addField(String field, float weight) { - if (weight < 1) { - throw new IllegalArgumentException("weight must be greater or equal to 1"); - } - fieldAndWeights.put(field, new FieldAndWeight(field, weight)); - return this; - } - - /** Adds a term to this builder. */ - public Builder addTerm(BytesRef term) { - if (termsSet.size() > BooleanQuery.getMaxClauseCount()) { - throw new BooleanQuery.TooManyClauses(); - } - termsSet.add(term); - return this; - } - - /** Builds the {@link XCombinedFieldQuery}. */ - public XCombinedFieldQuery build() { - int size = fieldAndWeights.size() * termsSet.size(); - if (size > BooleanQuery.getMaxClauseCount()) { - throw new BooleanQuery.TooManyClauses(); - } - BytesRef[] terms = termsSet.toArray(new BytesRef[0]); - return new XCombinedFieldQuery(new TreeMap<>(fieldAndWeights), terms); - } - } - - static class FieldAndWeight { - final String field; - final float weight; - - FieldAndWeight(String field, float weight) { - this.field = field; - this.weight = weight; - } - } - - // sorted map for fields. - private final TreeMap fieldAndWeights; - // array of terms, sorted. - private final BytesRef terms[]; - // array of terms per field, sorted - private final Term fieldTerms[]; - - private final long ramBytesUsed; - - private XCombinedFieldQuery(TreeMap fieldAndWeights, BytesRef[] terms) { - this.fieldAndWeights = fieldAndWeights; - this.terms = terms; - int numFieldTerms = fieldAndWeights.size() * terms.length; - if (numFieldTerms > BooleanQuery.getMaxClauseCount()) { - throw new BooleanQuery.TooManyClauses(); - } - this.fieldTerms = new Term[numFieldTerms]; - Arrays.sort(terms); - int pos = 0; - for (String field : fieldAndWeights.keySet()) { - for (BytesRef term : terms) { - fieldTerms[pos++] = new Term(field, term); - } - } - - this.ramBytesUsed = - BASE_RAM_BYTES - + RamUsageEstimator.sizeOfObject(fieldAndWeights) - + RamUsageEstimator.sizeOfObject(fieldTerms) - + RamUsageEstimator.sizeOfObject(terms); - } - - public List getTerms() { - return Collections.unmodifiableList(Arrays.asList(fieldTerms)); - } - - @Override - public String toString(String field) { - StringBuilder builder = new StringBuilder("CombinedFieldQuery(("); - int pos = 0; - for (FieldAndWeight fieldWeight : fieldAndWeights.values()) { - if (pos++ != 0) { - builder.append(" "); - } - builder.append(fieldWeight.field); - if (fieldWeight.weight != 1f) { - builder.append("^"); - builder.append(fieldWeight.weight); - } - } - builder.append(")("); - pos = 0; - for (BytesRef term : terms) { - if (pos++ != 0) { - builder.append(" "); - } - builder.append(term.utf8ToString()); - } - builder.append("))"); - return builder.toString(); - } - - @Override - public int hashCode() { - return 31 * classHash() + Arrays.hashCode(terms); - } - - @Override - public boolean equals(Object other) { - return sameClassAs(other) && Arrays.equals(terms, ((XCombinedFieldQuery) other).terms); - } - - @Override - public long ramBytesUsed() { - return ramBytesUsed; - } - - @Override - public Query rewrite(IndexReader reader) throws IOException { - // optimize zero and single field cases - if (terms.length == 0) { - return new BooleanQuery.Builder().build(); - } - // single field and one term - if (fieldTerms.length == 1) { - return new TermQuery(fieldTerms[0]); - } - // single field and multiple terms - if (fieldAndWeights.size() == 1) { - SynonymQuery.Builder builder = new SynonymQuery.Builder(fieldTerms[0].field()); - for (Term term : fieldTerms) { - builder.addTerm(term); - } - return builder.build(); - } - return this; - } - - @Override - public void visit(QueryVisitor visitor) { - Term[] selectedTerms = - Arrays.stream(fieldTerms).filter(t -> visitor.acceptField(t.field())).toArray(Term[]::new); - if (selectedTerms.length > 0) { - QueryVisitor v = visitor.getSubVisitor(BooleanClause.Occur.SHOULD, this); - v.consumeTerms(this, selectedTerms); - } - } - - private BooleanQuery rewriteToBoolean() { - // rewrite to a simple disjunction if the score is not needed. - BooleanQuery.Builder bq = new BooleanQuery.Builder(); - for (Term term : fieldTerms) { - bq.add(new TermQuery(term), BooleanClause.Occur.SHOULD); - } - return bq.build(); - } - - @Override - public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) - throws IOException { - if (scoreMode.needsScores()) { - return new CombinedFieldWeight(this, searcher, scoreMode, boost); - } else { - // rewrite to a simple disjunction if the score is not needed. - Query bq = rewriteToBoolean(); - return searcher.rewrite(bq).createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, boost); - } - } - - class CombinedFieldWeight extends Weight { - private final IndexSearcher searcher; - private final TermStates termStates[]; - private final Similarity.SimScorer simWeight; - - CombinedFieldWeight(Query query, IndexSearcher searcher, ScoreMode scoreMode, float boost) - throws IOException { - super(query); - assert scoreMode.needsScores(); - this.searcher = searcher; - long docFreq = 0; - long totalTermFreq = 0; - termStates = new TermStates[fieldTerms.length]; - for (int i = 0; i < termStates.length; i++) { - FieldAndWeight field = fieldAndWeights.get(fieldTerms[i].field()); - TermStates ts = TermStates.build(searcher.getTopReaderContext(), fieldTerms[i], true); - termStates[i] = ts; - if (ts.docFreq() > 0) { - TermStatistics termStats = - searcher.termStatistics(fieldTerms[i], ts.docFreq(), ts.totalTermFreq()); - docFreq = Math.max(termStats.docFreq(), docFreq); - totalTermFreq += (double) field.weight * termStats.totalTermFreq(); - } - } - if (docFreq > 0) { - CollectionStatistics pseudoCollectionStats = mergeCollectionStatistics(searcher); - TermStatistics pseudoTermStatistics = - new TermStatistics(new BytesRef("pseudo_term"), docFreq, Math.max(1, totalTermFreq)); - this.simWeight = - searcher.getSimilarity().scorer(boost, pseudoCollectionStats, pseudoTermStatistics); - } else { - this.simWeight = null; - } - } - - private CollectionStatistics mergeCollectionStatistics(IndexSearcher searcher) - throws IOException { - long maxDoc = searcher.getIndexReader().maxDoc(); - long docCount = 0; - long sumTotalTermFreq = 0; - long sumDocFreq = 0; - for (FieldAndWeight fieldWeight : fieldAndWeights.values()) { - CollectionStatistics collectionStats = searcher.collectionStatistics(fieldWeight.field); - if (collectionStats != null) { - docCount = Math.max(collectionStats.docCount(), docCount); - sumDocFreq = Math.max(collectionStats.sumDocFreq(), sumDocFreq); - sumTotalTermFreq += (double) fieldWeight.weight * collectionStats.sumTotalTermFreq(); - } - } - - return new CollectionStatistics( - "pseudo_field", maxDoc, docCount, sumTotalTermFreq, sumDocFreq); - } - - @Override - public void extractTerms(Set termSet) { - termSet.addAll(Arrays.asList(fieldTerms)); - } - - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - Weight weight = - searcher.rewrite(rewriteToBoolean()).createWeight(searcher, ScoreMode.COMPLETE, 1f); - return weight.matches(context, doc); - } - - @Override - public Explanation explain(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer != null) { - int newDoc = scorer.iterator().advance(doc); - if (newDoc == doc) { - final float freq; - if (scorer instanceof CombinedFieldScorer) { - freq = ((CombinedFieldScorer) scorer).freq(); - } else { - assert scorer instanceof TermScorer; - freq = ((TermScorer) scorer).freq(); - } - final XMultiNormsLeafSimScorer docScorer = - new XMultiNormsLeafSimScorer( - simWeight, context.reader(), fieldAndWeights.values(), true); - Explanation freqExplanation = Explanation.match(freq, "termFreq=" + freq); - Explanation scoreExplanation = docScorer.explain(doc, freqExplanation); - return Explanation.match( - scoreExplanation.getValue(), - "weight(" + getQuery() + " in " + doc + "), result of:", - scoreExplanation); - } - } - return Explanation.noMatch("no matching term"); - } - - @Override - public Scorer scorer(LeafReaderContext context) throws IOException { - List iterators = new ArrayList<>(); - List fields = new ArrayList<>(); - for (int i = 0; i < fieldTerms.length; i++) { - TermState state = termStates[i].get(context); - if (state != null) { - TermsEnum termsEnum = context.reader().terms(fieldTerms[i].field()).iterator(); - termsEnum.seekExact(fieldTerms[i].bytes(), state); - PostingsEnum postingsEnum = termsEnum.postings(null, PostingsEnum.FREQS); - iterators.add(postingsEnum); - fields.add(fieldAndWeights.get(fieldTerms[i].field())); - } - } - - if (iterators.isEmpty()) { - return null; - } - - // we must optimize this case (term not in segment), disjunctions require >= 2 subs - if (iterators.size() == 1) { - final LeafSimScorer scoringSimScorer = - new LeafSimScorer(simWeight, context.reader(), fields.get(0).field, true); - return new TermScorer(this, iterators.get(0), scoringSimScorer); - } - final XMultiNormsLeafSimScorer scoringSimScorer = - new XMultiNormsLeafSimScorer(simWeight, context.reader(), fields, true); - LeafSimScorer nonScoringSimScorer = - new LeafSimScorer(simWeight, context.reader(), "pseudo_field", false); - // we use termscorers + disjunction as an impl detail - DisiPriorityQueue queue = new DisiPriorityQueue(iterators.size()); - for (int i = 0; i < iterators.size(); i++) { - float weight = fields.get(i).weight; - queue.add( - new WeightedDisiWrapper( - new TermScorer(this, iterators.get(i), nonScoringSimScorer), weight)); - } - // Even though it is called approximation, it is accurate since none of - // the sub iterators are two-phase iterators. - DocIdSetIterator iterator = new DisjunctionDISIApproximation(queue); - return new CombinedFieldScorer(this, queue, iterator, scoringSimScorer); - } - - @Override - public boolean isCacheable(LeafReaderContext ctx) { - return false; - } - } - - private static class WeightedDisiWrapper extends DisiWrapper { - final float weight; - - WeightedDisiWrapper(Scorer scorer, float weight) { - super(scorer); - this.weight = weight; - } - - float freq() throws IOException { - return weight * ((PostingsEnum) iterator).freq(); - } - } - - private static class CombinedFieldScorer extends Scorer { - private final DisiPriorityQueue queue; - private final DocIdSetIterator iterator; - private final XMultiNormsLeafSimScorer simScorer; - - CombinedFieldScorer( - Weight weight, - DisiPriorityQueue queue, - DocIdSetIterator iterator, - XMultiNormsLeafSimScorer simScorer) { - super(weight); - this.queue = queue; - this.iterator = iterator; - this.simScorer = simScorer; - } - - @Override - public int docID() { - return iterator.docID(); - } - - float freq() throws IOException { - DisiWrapper w = queue.topList(); - float freq = ((WeightedDisiWrapper) w).freq(); - for (w = w.next; w != null; w = w.next) { - freq += ((WeightedDisiWrapper) w).freq(); - if (freq < 0) { // overflow - return Integer.MAX_VALUE; - } - } - return freq; - } - - @Override - public float score() throws IOException { - return simScorer.score(iterator.docID(), freq()); - } - - @Override - public DocIdSetIterator iterator() { - return iterator; - } - - @Override - public float getMaxScore(int upTo) throws IOException { - return Float.POSITIVE_INFINITY; - } - } -} diff --git a/server/src/main/java/org/apache/lucene/search/XMultiNormsLeafSimScorer.java b/server/src/main/java/org/apache/lucene/search/XMultiNormsLeafSimScorer.java deleted file mode 100644 index bb3e34b4e410b..0000000000000 --- a/server/src/main/java/org/apache/lucene/search/XMultiNormsLeafSimScorer.java +++ /dev/null @@ -1,161 +0,0 @@ -/* @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.lucene.search; - -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.search.similarities.Similarity.SimScorer; -import org.apache.lucene.util.SmallFloat; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Objects; - -import static org.apache.lucene.search.XCombinedFieldQuery.FieldAndWeight; - -/** - * Copy of {@link LeafSimScorer} that sums document's norms from multiple fields. - * - * TODO: this is temporarily copied from Lucene, remove once we update to Lucene 8.9. - */ -final class XMultiNormsLeafSimScorer { - /** - * Cache of decoded norms. - */ - private static final float[] LENGTH_TABLE = new float[256]; - - static { - for (int i = 0; i < 256; i++) { - LENGTH_TABLE[i] = SmallFloat.byte4ToInt((byte) i); - } - } - - private final SimScorer scorer; - private final NumericDocValues norms; - - /** - * Sole constructor: Score documents of {@code reader} with {@code scorer}. - * - */ - XMultiNormsLeafSimScorer(SimScorer scorer, - LeafReader reader, - Collection normFields, - boolean needsScores) throws IOException { - this.scorer = Objects.requireNonNull(scorer); - if (needsScores) { - final List normsList = new ArrayList<>(); - final List weightList = new ArrayList<>(); - for (FieldAndWeight field : normFields) { - NumericDocValues norms = reader.getNormValues(field.field); - if (norms != null) { - normsList.add(norms); - weightList.add(field.weight); - } - } - if (normsList.isEmpty()) { - norms = null; - } else if (normsList.size() == 1) { - norms = normsList.get(0); - } else { - final NumericDocValues[] normsArr = normsList.toArray(new NumericDocValues[0]); - final float[] weightArr = new float[normsList.size()]; - for (int i = 0; i < weightList.size(); i++) { - weightArr[i] = weightList.get(i); - } - norms = new XMultiNormsLeafSimScorer.MultiFieldNormValues(normsArr, weightArr); - } - } else { - norms = null; - } - } - - private long getNormValue(int doc) throws IOException { - if (norms != null) { - boolean found = norms.advanceExact(doc); - assert found; - return norms.longValue(); - } else { - return 1L; // default norm - } - } - - /** Score the provided document assuming the given term document frequency. - * This method must be called on non-decreasing sequences of doc ids. - * @see SimScorer#score(float, long) */ - public float score(int doc, float freq) throws IOException { - return scorer.score(freq, getNormValue(doc)); - } - - /** Explain the score for the provided document assuming the given term document frequency. - * This method must be called on non-decreasing sequences of doc ids. - * @see SimScorer#explain(Explanation, long) */ - public Explanation explain(int doc, Explanation freqExpl) throws IOException { - return scorer.explain(freqExpl, getNormValue(doc)); - } - - private static class MultiFieldNormValues extends NumericDocValues { - private final NumericDocValues[] normsArr; - private final float[] weightArr; - private long current; - private int docID = -1; - - MultiFieldNormValues(NumericDocValues[] normsArr, float[] weightArr) { - this.normsArr = normsArr; - this.weightArr = weightArr; - } - - @Override - public long longValue() { - return current; - } - - @Override - public boolean advanceExact(int target) throws IOException { - float normValue = 0; - for (int i = 0; i < normsArr.length; i++) { - boolean found = normsArr[i].advanceExact(target); - assert found; - normValue += weightArr[i] * LENGTH_TABLE[Byte.toUnsignedInt((byte) normsArr[i].longValue())]; - } - current = SmallFloat.intToByte4(Math.round(normValue)); - return true; - } - - @Override - public int docID() { - return docID; - } - - @Override - public int nextDoc() { - throw new UnsupportedOperationException(); - } - - @Override - public int advance(int target) { - throw new UnsupportedOperationException(); - } - - @Override - public long cost() { - throw new UnsupportedOperationException(); - } - } -} diff --git a/server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java b/server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java index dd12837fce8c0..a36d06f3e01e8 100644 --- a/server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java +++ b/server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java @@ -11,6 +11,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.queries.BlendedTermQuery; +import org.apache.lucene.sandbox.search.CombinedFieldQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.MultiPhraseQuery; @@ -18,7 +19,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.XCombinedFieldQuery; import org.apache.lucene.search.spans.SpanTermQuery; import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery; import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery; @@ -75,8 +75,8 @@ protected void flatten(Query sourceQuery, IndexReader reader, Collection for (Term term : synQuery.getTerms()) { flatten(new TermQuery(term), reader, flatQueries, boost); } - } else if (sourceQuery instanceof XCombinedFieldQuery) { - XCombinedFieldQuery combinedFieldQuery = (XCombinedFieldQuery) sourceQuery; + } else if (sourceQuery instanceof CombinedFieldQuery) { + CombinedFieldQuery combinedFieldQuery = (CombinedFieldQuery) sourceQuery; for (Term term : combinedFieldQuery.getTerms()) { flatten(new TermQuery(term), reader, flatQueries, boost); } diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 6b62d420b9ade..b6140d5d29251 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -82,8 +82,8 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_7_12_1 = new Version(7120199, org.apache.lucene.util.Version.LUCENE_8_8_0); public static final Version V_7_12_2 = new Version(7120299, org.apache.lucene.util.Version.LUCENE_8_8_0); public static final Version V_7_13_0 = new Version(7130099, org.apache.lucene.util.Version.LUCENE_8_8_2); - public static final Version V_7_14_0 = new Version(7140099, org.apache.lucene.util.Version.LUCENE_8_9_0); - public static final Version V_8_0_0 = new Version(8000099, org.apache.lucene.util.Version.LUCENE_8_9_0); + public static final Version V_7_14_0 = new Version(7140099, org.apache.lucene.util.Version.LUCENE_8_8_2); // TODO upgrade + public static final Version V_8_0_0 = new Version(8000099, org.apache.lucene.util.Version.LUCENE_9_0_0); public static final Version CURRENT = V_8_0_0; private static final ImmutableOpenIntMap idToVersion; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index 17aa182be56bb..3ff4375266241 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -187,7 +187,6 @@ static TopDocs mergeTopDocs(Collection results, int topN, int from) { if (results.isEmpty()) { return null; } - final boolean setShardIndex = false; final TopDocs topDocs = results.stream().findFirst().get(); final TopDocs mergedTopDocs; final int numShards = results.size(); @@ -197,15 +196,15 @@ static TopDocs mergeTopDocs(Collection results, int topN, int from) { CollapseTopFieldDocs firstTopDocs = (CollapseTopFieldDocs) topDocs; final Sort sort = new Sort(firstTopDocs.fields); final CollapseTopFieldDocs[] shardTopDocs = results.toArray(new CollapseTopFieldDocs[numShards]); - mergedTopDocs = CollapseTopFieldDocs.merge(sort, from, topN, shardTopDocs, setShardIndex); + mergedTopDocs = CollapseTopFieldDocs.merge(sort, from, topN, shardTopDocs); } else if (topDocs instanceof TopFieldDocs) { TopFieldDocs firstTopDocs = (TopFieldDocs) topDocs; final Sort sort = new Sort(firstTopDocs.fields); final TopFieldDocs[] shardTopDocs = results.toArray(new TopFieldDocs[numShards]); - mergedTopDocs = TopDocs.merge(sort, from, topN, shardTopDocs, setShardIndex); + mergedTopDocs = TopDocs.merge(sort, from, topN, shardTopDocs); } else { final TopDocs[] shardTopDocs = results.toArray(new TopDocs[numShards]); - mergedTopDocs = TopDocs.merge(from, topN, shardTopDocs, setShardIndex); + mergedTopDocs = TopDocs.merge(from, topN, shardTopDocs); } return mergedTopDocs; } diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java index de79cb0bf29ff..f02d8dc999581 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java @@ -9,7 +9,7 @@ package org.elasticsearch.action.search; import org.apache.lucene.store.ByteArrayDataInput; -import org.apache.lucene.store.RAMOutputStream; +import org.apache.lucene.store.ByteBuffersDataOutput; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; @@ -30,7 +30,8 @@ static InternalScrollSearchRequest internalScrollSearchRequest(ShardSearchContex } static String buildScrollId(AtomicArray searchPhaseResults) { - try (RAMOutputStream out = new RAMOutputStream()) { + try { + ByteBuffersDataOutput out = new ByteBuffersDataOutput(); out.writeString(INCLUDE_CONTEXT_UUID); out.writeString(searchPhaseResults.length() == 1 ? ParsedScrollId.QUERY_AND_FETCH_TYPE : ParsedScrollId.QUERY_THEN_FETCH_TYPE); out.writeVInt(searchPhaseResults.asList().size()); @@ -45,8 +46,7 @@ static String buildScrollId(AtomicArray searchPhase out.writeString(searchShardTarget.getNodeId()); } } - byte[] bytes = new byte[(int) out.getFilePointer()]; - out.writeTo(bytes, 0); + byte[] bytes = out.toArrayCopy(); return Base64.getUrlEncoder().encodeToString(bytes); } catch (IOException e) { throw new UncheckedIOException(e); diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java b/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java index 6709134407336..cda0f41e71962 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java @@ -599,8 +599,8 @@ public static double arcDistance(double lat1, double lon1, double lat2, double l * 4 decimal degrees */ public static double planeDistance(double lat1, double lon1, double lat2, double lon2) { - double x = (lon2 - lon1) * SloppyMath.TO_RADIANS * Math.cos((lat2 + lat1) / 2.0 * SloppyMath.TO_RADIANS); - double y = (lat2 - lat1) * SloppyMath.TO_RADIANS; + double x = Math.toRadians(lon2 - lon1) * Math.cos((lat2 + lat1) / Math.toRadians(2.0)); + double y = Math.toRadians(lat2 - lat1); return Math.sqrt(x * x + y * y) * EARTH_MEAN_RADIUS; } diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java index 9750ea1f53825..56b9f51ef22b4 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -49,6 +49,7 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.Terms; +import org.apache.lucene.index.VectorValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.FieldDoc; @@ -74,7 +75,6 @@ import org.apache.lucene.store.Lock; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.Version; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.Nullable; @@ -197,7 +197,7 @@ public static SegmentInfos pruneUnreferencedFiles(String segmentsFileName, Direc * since checksums don's match anymore. that's why we prune the name here directly. * We also want the caller to know if we were not able to remove a segments_N file. */ - if (file.startsWith(IndexFileNames.SEGMENTS) || file.equals(IndexFileNames.OLD_SEGMENTS_GEN)) { + if (file.startsWith(IndexFileNames.SEGMENTS)) { foundSegmentFiles++; if (file.equals(si.getSegmentsFileName()) == false) { directory.deleteFile(file); // remove all segment_N files except of the one we wanna keep @@ -235,7 +235,7 @@ public static IndexCommit getIndexCommit(SegmentInfos si, Directory directory) t public static void cleanLuceneIndex(Directory directory) throws IOException { try (Lock writeLock = directory.obtainLock(IndexWriter.WRITE_LOCK_NAME)) { for (final String file : directory.listAll()) { - if (file.startsWith(IndexFileNames.SEGMENTS) || file.equals(IndexFileNames.OLD_SEGMENTS_GEN)) { + if (file.startsWith(IndexFileNames.SEGMENTS)) { directory.deleteFile(file); // remove all segment_N files } } @@ -1024,6 +1024,16 @@ public NumericDocValues getNormValues(String field) { return null; } + @Override + public VectorValues getVectorValues(String field) throws IOException { + return null; + } + + @Override + public TopDocs searchNearestVectors(String field, float[] target, int k, int fanout) throws IOException { + return null; + } + public FieldInfos getFieldInfos() { return new FieldInfos(new FieldInfo[0]); } @@ -1074,7 +1084,7 @@ public CacheHelper getReaderCacheHelper() { /** * Prepares a new {@link IndexWriterConfig} that does not do any merges, by setting both the merge policy and the merge scheduler. * Setting just the merge policy means that constructing the index writer will create a {@link ConcurrentMergeScheduler} by default, - * which is quite heavyweight and in particular it can unnecessarily block on {@link IOUtils#spins}. + * which is quite heavyweight. */ @SuppressForbidden(reason = "NoMergePolicy#INSTANCE is safe to use since we also set NoMergeScheduler#INSTANCE") public static IndexWriterConfig indexWriterConfigWithNoMerging(Analyzer analyzer) { diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java b/server/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java index 1f0674f059e0c..14bdccc4d512c 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java @@ -19,6 +19,7 @@ import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.similarities.ClassicSimilarity; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.search.similarities.TFIDFSimilarity; @@ -148,6 +149,11 @@ public Query rewrite(IndexReader reader) throws IOException { return createQuery(mlt); } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + private Query createQuery(XMoreLikeThis mlt) throws IOException { BooleanQuery.Builder bqBuilder = new BooleanQuery.Builder(); if (this.likeFields != null) { diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java b/server/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java index 87895bc91129c..b985048f5fb09 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java @@ -19,6 +19,7 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiPhraseQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.StringHelper; @@ -303,4 +304,11 @@ private boolean termArraysEquals(List termArrays1, List termArra public String getField() { return field; } + + @Override + public void visit(QueryVisitor visitor) { + if (visitor.acceptField(field)) { + visitor.visitLeaf(this); // TODO implement term visiting + } + } } diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java b/server/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java index e8e858b483521..4ff013797061e 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java @@ -10,7 +10,6 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.FilterScorer; @@ -34,7 +33,6 @@ import java.util.List; import java.util.Locale; import java.util.Objects; -import java.util.Set; /** * A query that allows for a pluggable boost function / filter. If it matches @@ -248,11 +246,6 @@ class CustomBoostFactorWeight extends Weight { this.needsScores = needsScores; } - @Override - public void extractTerms(Set terms) { - subQueryWeight.extractTerms(terms); - } - private FunctionFactorScorer functionScorer(LeafReaderContext context) throws IOException { Scorer subQueryScorer = subQueryWeight.scorer(context); if (subQueryScorer == null) { diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreQuery.java b/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreQuery.java index c33f588ac3670..28cf45fb7c5a8 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreQuery.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreQuery.java @@ -10,20 +10,19 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.FilterLeafCollector; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.Query; -import org.apache.lucene.search.Scorable; -import org.apache.lucene.search.Weight; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.Scorable; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.BulkScorer; +import org.apache.lucene.search.Weight; import org.apache.lucene.util.Bits; import org.elasticsearch.Version; import org.elasticsearch.script.ScoreScript; @@ -32,7 +31,6 @@ import java.io.IOException; import java.util.Objects; -import java.util.Set; /** * A query that uses a script to compute documents' scores. @@ -89,11 +87,6 @@ public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { } } - @Override - public void extractTerms(Set terms) { - subQueryWeight.extractTerms(terms); - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { Scorer subQueryScorer = subQueryWeight.scorer(context); diff --git a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java index 75d08a3427192..f4c2f24c3fbf7 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java +++ b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java @@ -16,7 +16,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.util.SetOnce; import org.elasticsearch.cli.ExitCodes; import org.elasticsearch.cli.UserException; @@ -205,7 +205,7 @@ public static KeyStoreWrapper load(Path configDir) throws IOException { return null; } - SimpleFSDirectory directory = new SimpleFSDirectory(configDir); + NIOFSDirectory directory = new NIOFSDirectory(configDir); try (IndexInput indexInput = directory.openInput(KEYSTORE_FILENAME, IOContext.READONCE)) { ChecksumIndexInput input = new BufferedChecksumIndexInput(indexInput); final int formatVersion; @@ -478,7 +478,7 @@ private void decryptLegacyEntries() throws GeneralSecurityException, IOException public synchronized void save(Path configDir, char[] password) throws Exception { ensureOpen(); - SimpleFSDirectory directory = new SimpleFSDirectory(configDir); + NIOFSDirectory directory = new NIOFSDirectory(configDir); // write to tmp file first, then overwrite String tmpFile = KEYSTORE_FILENAME + ".tmp"; try (IndexOutput output = directory.createOutput(tmpFile, IOContext.DEFAULT)) { diff --git a/server/src/main/java/org/elasticsearch/common/util/CuckooFilter.java b/server/src/main/java/org/elasticsearch/common/util/CuckooFilter.java index a7c9673582f02..bebc783d20a97 100644 --- a/server/src/main/java/org/elasticsearch/common/util/CuckooFilter.java +++ b/server/src/main/java/org/elasticsearch/common/util/CuckooFilter.java @@ -7,8 +7,6 @@ */ package org.elasticsearch.common.util; -import org.apache.lucene.store.DataInput; -import org.apache.lucene.store.DataOutput; import org.apache.lucene.util.packed.PackedInts; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -121,7 +119,8 @@ public class CuckooFilter implements Writeable { this.rng = rng; this.fingerprintMask = (0x80000000 >> (bitsPerEntry - 1)) >>> (Integer.SIZE - bitsPerEntry); - + this.data = null; // nocommit + /* data = (PackedInts.Mutable) PackedInts.getReader(new DataInput() { @Override public byte readByte() throws IOException { @@ -133,6 +132,7 @@ public void readBytes(byte[] b, int offset, int len) throws IOException { in.readBytes(b, offset, len); } }); + */ } @Override @@ -143,6 +143,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVInt(count); out.writeVInt(evictedFingerprint); + // nocommit + /* data.save(new DataOutput() { @Override public void writeByte(byte b) throws IOException { @@ -154,6 +156,7 @@ public void writeBytes(byte[] b, int offset, int length) throws IOException { out.writeBytes(b, offset, length); } }); + */ } /** diff --git a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java index e3c90240b5e8d..323a17e7bb2a5 100644 --- a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -18,8 +18,8 @@ import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.store.NativeFSLockFactory; -import org.apache.lucene.store.SimpleFSDirectory; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -534,7 +534,7 @@ public static void acquireFSLockForPaths(IndexSettings indexSettings, Path... sh // resolve the directory the shard actually lives in Path p = shardPaths[i].resolve("index"); // open a directory (will be immediately closed) on the shard's location - dirs[i] = new SimpleFSDirectory(p, indexSettings.getValue(FsDirectoryFactory.INDEX_LOCK_FACTOR_SETTING)); + dirs[i] = new NIOFSDirectory(p, indexSettings.getValue(FsDirectoryFactory.INDEX_LOCK_FACTOR_SETTING)); // create a lock for the "write.lock" file try { locks[i] = dirs[i].obtainLock(IndexWriter.WRITE_LOCK_NAME); diff --git a/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java b/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java index bf7b0a1d9eaf1..09897bcbe5e62 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java @@ -18,7 +18,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.store.NIOFSDirectory; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.lucene.store.IndexOutputOutputStream; @@ -292,7 +292,7 @@ public final T read(NamedXContentRegistry namedXContentRegistry, Path file) thro } protected Directory newDirectory(Path dir) throws IOException { - return new SimpleFSDirectory(dir); + return new NIOFSDirectory(dir); } diff --git a/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java b/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java index 4b8a04a099e0b..a9abb0491d532 100644 --- a/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java @@ -32,7 +32,7 @@ import org.apache.lucene.search.Weight; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; @@ -201,7 +201,7 @@ private static IndexWriter createIndexWriter(Directory directory, boolean openEx * {@link Writer} on this path. */ public static void delete(Path dataPath) throws IOException { - Lucene.cleanLuceneIndex(new SimpleFSDirectory(dataPath.resolve(METADATA_DIRECTORY_NAME))); + Lucene.cleanLuceneIndex(new NIOFSDirectory(dataPath.resolve(METADATA_DIRECTORY_NAME))); } // exposed for tests @@ -209,7 +209,7 @@ Directory createDirectory(Path path) throws IOException { // it is possible to disable the use of MMapDirectory for indices, and it may be surprising to users that have done so if we still // use a MMapDirectory here, which might happen with FSDirectory.open(path). Concurrency is of no concern here so a // SimpleFSDirectory is fine: - return new SimpleFSDirectory(path); + return new NIOFSDirectory(path); } public Path getDataPath() { @@ -246,7 +246,7 @@ public boolean empty() { public static NodeMetadata nodeMetadata(Path dataPath) throws IOException { final Path indexPath = dataPath.resolve(METADATA_DIRECTORY_NAME); if (Files.exists(indexPath)) { - try (DirectoryReader reader = DirectoryReader.open(new SimpleFSDirectory(dataPath.resolve(METADATA_DIRECTORY_NAME)))) { + try (DirectoryReader reader = DirectoryReader.open(new NIOFSDirectory(dataPath.resolve(METADATA_DIRECTORY_NAME)))) { final Map userData = reader.getIndexCommit().getUserData(); assert userData.get(NODE_VERSION_KEY) != null; @@ -269,12 +269,12 @@ public static void overrideVersion(Version newVersion, Path... dataPaths) throws for (final Path dataPath : dataPaths) { final Path indexPath = dataPath.resolve(METADATA_DIRECTORY_NAME); if (Files.exists(indexPath)) { - try (DirectoryReader reader = DirectoryReader.open(new SimpleFSDirectory(dataPath.resolve(METADATA_DIRECTORY_NAME)))) { + try (DirectoryReader reader = DirectoryReader.open(new NIOFSDirectory(dataPath.resolve(METADATA_DIRECTORY_NAME)))) { final Map userData = reader.getIndexCommit().getUserData(); assert userData.get(NODE_VERSION_KEY) != null; try (IndexWriter indexWriter = - createIndexWriter(new SimpleFSDirectory(dataPath.resolve(METADATA_DIRECTORY_NAME)), true)) { + createIndexWriter(new NIOFSDirectory(dataPath.resolve(METADATA_DIRECTORY_NAME)), true)) { final Map commitData = new HashMap<>(userData); commitData.put(NODE_VERSION_KEY, Integer.toString(newVersion.id)); indexWriter.setLiveCommitData(commitData.entrySet()); diff --git a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java index f413f068eebf6..5a72663088e3d 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java +++ b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java @@ -10,8 +10,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene87.Lucene87Codec; -import org.apache.lucene.codecs.lucene87.Lucene87Codec.Mode; +import org.apache.lucene.codecs.lucene90.Lucene90Codec; import org.elasticsearch.common.Nullable; import org.elasticsearch.index.mapper.MapperService; @@ -36,13 +35,13 @@ public class CodecService { public CodecService(@Nullable MapperService mapperService, Logger logger) { final var codecs = new HashMap(); if (mapperService == null) { - codecs.put(DEFAULT_CODEC, new Lucene87Codec()); - codecs.put(BEST_COMPRESSION_CODEC, new Lucene87Codec(Mode.BEST_COMPRESSION)); + codecs.put(DEFAULT_CODEC, new Lucene90Codec()); + codecs.put(BEST_COMPRESSION_CODEC, new Lucene90Codec(Lucene90Codec.Mode.BEST_COMPRESSION)); } else { codecs.put(DEFAULT_CODEC, - new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); + new PerFieldMappingPostingFormatCodec(Lucene90Codec.Mode.BEST_SPEED, mapperService, logger)); codecs.put(BEST_COMPRESSION_CODEC, - new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger)); + new PerFieldMappingPostingFormatCodec(Lucene90Codec.Mode.BEST_COMPRESSION, mapperService, logger)); } codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault()); for (String codec : Codec.availableCodecs()) { diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java index a08a05ee5f223..6490bdb20ab8b 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java +++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java @@ -12,8 +12,8 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat; -import org.apache.lucene.codecs.lucene87.Lucene87Codec; +import org.apache.lucene.codecs.lucene90.Lucene90Codec; +import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.index.mapper.CompletionFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; @@ -27,11 +27,11 @@ * per index in real time via the mapping API. If no specific postings format is * configured for a specific field the default postings format is used. */ -public class PerFieldMappingPostingFormatCodec extends Lucene87Codec { +public class PerFieldMappingPostingFormatCodec extends Lucene90Codec { private final Logger logger; private final MapperService mapperService; - // Always enable compression on binary doc values - private final DocValuesFormat docValuesFormat = new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION); + + private final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat(); static { assert Codec.forName(Lucene.LATEST_CODEC).getClass().isAssignableFrom(PerFieldMappingPostingFormatCodec.class) : diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index 4407c5720a718..60e6360351800 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -26,7 +26,6 @@ import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.util.Accountable; -import org.apache.lucene.util.Accountables; import org.apache.lucene.util.SetOnce; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.index.IndexRequest; @@ -778,13 +777,6 @@ public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean incl } protected void fillSegmentStats(SegmentReader segmentReader, boolean includeSegmentFileSizes, SegmentsStats stats) { - stats.add(1, segmentReader.ramBytesUsed()); - stats.addTermsMemoryInBytes(guardedRamBytesUsed(segmentReader.getPostingsReader())); - stats.addStoredFieldsMemoryInBytes(guardedRamBytesUsed(segmentReader.getFieldsReader())); - stats.addTermVectorsMemoryInBytes(guardedRamBytesUsed(segmentReader.getTermVectorsReader())); - stats.addNormsMemoryInBytes(guardedRamBytesUsed(segmentReader.getNormsReader())); - stats.addPointsMemoryInBytes(guardedRamBytesUsed(segmentReader.getPointsReader())); - stats.addDocValuesMemoryInBytes(guardedRamBytesUsed(segmentReader.getDocValuesReader())); if (includeSegmentFileSizes) { stats.addFiles(getSegmentFileSizes(segmentReader)); } @@ -892,11 +884,7 @@ private void fillSegmentInfo(SegmentReader segmentReader, boolean verbose, boole } catch (IOException e) { logger.trace(() -> new ParameterizedMessage("failed to get size for [{}]", info.info.name), e); } - segment.memoryInBytes = segmentReader.ramBytesUsed(); segment.segmentSort = info.info.getIndexSort(); - if (verbose) { - segment.ramTree = Accountables.namedAccountable("root", segmentReader); - } segment.attributes = info.info.getAttributes(); // TODO: add more fine grained mem stats values to per segment info here segments.put(info.info.name, segment); diff --git a/server/src/main/java/org/elasticsearch/index/engine/PrunePostingsMergePolicy.java b/server/src/main/java/org/elasticsearch/index/engine/PrunePostingsMergePolicy.java index 52978412f8c74..fb578e91c3f3a 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/PrunePostingsMergePolicy.java +++ b/server/src/main/java/org/elasticsearch/index/engine/PrunePostingsMergePolicy.java @@ -127,11 +127,6 @@ public ImpactsEnum impacts(int flags) throws IOException { public int size() { return postingsReader.size(); } - - @Override - public long ramBytesUsed() { - return postingsReader.ramBytesUsed(); - } }; } diff --git a/server/src/main/java/org/elasticsearch/index/engine/RamAccountingRefreshListener.java b/server/src/main/java/org/elasticsearch/index/engine/RamAccountingRefreshListener.java index 0d5c89a7479a2..a8d944d8af87f 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/RamAccountingRefreshListener.java +++ b/server/src/main/java/org/elasticsearch/index/engine/RamAccountingRefreshListener.java @@ -58,7 +58,7 @@ public void accept(ElasticsearchDirectoryReader reader, ElasticsearchDirectoryRe // don't add the segment's memory unless it is not referenced by the previous reader // (only new segments) if (prevReaders.contains(segmentReader.getCoreCacheHelper().getKey()) == false) { - final long ramBytesUsed = segmentReader.ramBytesUsed(); + final long ramBytesUsed = 0; // add the segment memory to the breaker (non-breaking) breaker.addWithoutBreaking(ramBytesUsed); // and register a listener for when the segment is closed to decrement the diff --git a/server/src/main/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicy.java b/server/src/main/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicy.java index 750dabf187bf1..ff82cf0900d71 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicy.java +++ b/server/src/main/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicy.java @@ -178,11 +178,6 @@ public void checkIntegrity() throws IOException { public void close() throws IOException { in.close(); } - - @Override - public long ramBytesUsed() { - return in.ramBytesUsed(); - } } private abstract static class FilterStoredFieldsReader extends StoredFieldsReader { @@ -193,11 +188,6 @@ private abstract static class FilterStoredFieldsReader extends StoredFieldsReade this.in = fieldsReader; } - @Override - public long ramBytesUsed() { - return in.ramBytesUsed(); - } - @Override public void close() throws IOException { in.close(); @@ -270,7 +260,7 @@ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { } @Override - public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { + public void stringField(FieldInfo fieldInfo, String value) throws IOException { visitor.stringField(fieldInfo, value); } diff --git a/server/src/main/java/org/elasticsearch/index/engine/SingleDocDirectoryReader.java b/server/src/main/java/org/elasticsearch/index/engine/SingleDocDirectoryReader.java index 00262b9684e71..d64fd83cd0380 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/SingleDocDirectoryReader.java +++ b/server/src/main/java/org/elasticsearch/index/engine/SingleDocDirectoryReader.java @@ -25,6 +25,8 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.Terms; +import org.apache.lucene.index.VectorValues; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; @@ -210,6 +212,16 @@ public NumericDocValues getNormValues(String field) throws IOException { return getDelegate().getNormValues(field); } + @Override + public VectorValues getVectorValues(String field) throws IOException { + return getDelegate().getVectorValues(field); + } + + @Override + public TopDocs searchNearestVectors(String field, float[] target, int k, int fanout) throws IOException { + return getDelegate().searchNearestVectors(field, target, k, fanout); + } + @Override public FieldInfos getFieldInfos() { return getDelegate().getFieldInfos(); diff --git a/server/src/main/java/org/elasticsearch/index/engine/TranslogLeafReader.java b/server/src/main/java/org/elasticsearch/index/engine/TranslogLeafReader.java index 548b25b29e4cd..db2acc6bf94be 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/TranslogLeafReader.java +++ b/server/src/main/java/org/elasticsearch/index/engine/TranslogLeafReader.java @@ -22,6 +22,8 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.Terms; +import org.apache.lucene.index.VectorValues; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.util.set.Sets; @@ -32,7 +34,6 @@ import org.elasticsearch.index.translog.Translog; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.Set; @@ -44,13 +45,13 @@ public final class TranslogLeafReader extends LeafReader { private final Translog.Index operation; private static final FieldInfo FAKE_SOURCE_FIELD = new FieldInfo(SourceFieldMapper.NAME, 1, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(), - 0, 0, 0, false); + 0, 0, 0, 0, VectorValues.SimilarityFunction.NONE, false); private static final FieldInfo FAKE_ROUTING_FIELD = new FieldInfo(RoutingFieldMapper.NAME, 2, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(), - 0, 0, 0, false); + 0, 0, 0, 0, VectorValues.SimilarityFunction.NONE, false); private static final FieldInfo FAKE_ID_FIELD = new FieldInfo(IdFieldMapper.NAME, 3, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(), - 0, 0, 0, false); + 0, 0, 0, 0, VectorValues.SimilarityFunction.NONE, false); public static Set ALL_FIELD_NAMES = Sets.newHashSet(FAKE_SOURCE_FIELD.name, FAKE_ROUTING_FIELD.name, FAKE_ID_FIELD.name); TranslogLeafReader(Translog.Index operation) { @@ -96,6 +97,16 @@ public NumericDocValues getNormValues(String field) { throw new UnsupportedOperationException(); } + @Override + public VectorValues getVectorValues(String field) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public TopDocs searchNearestVectors(String field, float[] target, int k, int fanout) throws IOException { + throw new UnsupportedOperationException(); + } + @Override public FieldInfos getFieldInfos() { throw new UnsupportedOperationException(); @@ -147,7 +158,7 @@ public void document(int docID, StoredFieldVisitor visitor) throws IOException { visitor.binaryField(FAKE_SOURCE_FIELD, operation.source().toBytesRef().bytes); } if (operation.routing() != null && visitor.needsField(FAKE_ROUTING_FIELD) == StoredFieldVisitor.Status.YES) { - visitor.stringField(FAKE_ROUTING_FIELD, operation.routing().getBytes(StandardCharsets.UTF_8)); + visitor.stringField(FAKE_ROUTING_FIELD, operation.routing()); } if (visitor.needsField(FAKE_ID_FIELD) == StoredFieldVisitor.Status.YES) { BytesRef bytesRef = Uid.encodeId(operation.id()); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java index 5fc4751045c0f..715ffe02297a5 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java @@ -9,8 +9,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.lucene.codecs.blocktree.FieldReader; -import org.apache.lucene.codecs.blocktree.Stats; +import org.apache.lucene.codecs.lucene90.blocktree.FieldReader; +import org.apache.lucene.codecs.lucene90.blocktree.Stats; import org.apache.lucene.index.FilteredTermsEnum; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java index af4c3e9197be6..52a0f7a4a0788 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java @@ -8,13 +8,13 @@ package org.elasticsearch.index.fielddata.plain; -import org.apache.lucene.document.HalfFloatPoint; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.time.DateUtils; diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java index 7a9baa57a6bb4..9999e89a1ad5c 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java @@ -19,7 +19,6 @@ import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.Uid; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -102,10 +101,9 @@ public void binaryField(FieldInfo fieldInfo, BytesRef value) { } @Override - public void stringField(FieldInfo fieldInfo, byte[] bytes) { + public void stringField(FieldInfo fieldInfo, String value) { assert IdFieldMapper.NAME.equals(fieldInfo.name) == false : "_id field must go through binaryField"; assert sourceFieldName.equals(fieldInfo.name) == false : "source field must go through binaryField"; - final String value = new String(bytes, StandardCharsets.UTF_8); addValue(fieldInfo.name, value); } diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java index 0b2bcc134a1e2..febb0decd97bb 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java @@ -9,12 +9,11 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Uid; -import org.apache.lucene.util.BytesRef; -import java.nio.charset.StandardCharsets; import java.util.List; /** @@ -60,8 +59,8 @@ public void binaryField(FieldInfo fieldInfo, byte[] value) { } @Override - public void stringField(FieldInfo fieldInfo, byte[] bytes) { - addValue(new String(bytes, StandardCharsets.UTF_8)); + public void stringField(FieldInfo fieldInfo, String value) { + addValue(value); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java index 32de2ba44f1b2..e3daf94692313 100644 --- a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -14,6 +14,7 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.VectorValues; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; @@ -238,7 +239,7 @@ private GetResult innerGetLoadFromStoredFields(String id, String[] storedFields, IndexableFieldType fieldType = indexableField.fieldType(); if (fieldType.stored()) { FieldInfo fieldInfo = new FieldInfo(indexableField.name(), 0, false, false, false, IndexOptions.NONE, - DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, false); + DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, 0, VectorValues.SimilarityFunction.NONE, false); StoredFieldVisitor.Status status = fieldVisitor.needsField(fieldInfo); if (status == StoredFieldVisitor.Status.YES) { if (indexableField.numericValue() != null) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 3eeb8a0809182..67dc5194efcf7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -14,8 +14,8 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PointValues; +import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; -import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.Query; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java index 8832fbe08acec..18dc434f06228 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java @@ -25,13 +25,14 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.Terms; +import org.apache.lucene.index.VectorValues; import org.apache.lucene.index.memory.MemoryIndex; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; @@ -161,7 +162,7 @@ public void document(int docID, StoredFieldVisitor visitor) throws IOException { visitor.doubleField(fieldInfo, v.doubleValue()); } } else if (field.stringValue() != null) { - visitor.stringField(fieldInfo, field.stringValue().getBytes(StandardCharsets.UTF_8)); + visitor.stringField(fieldInfo, field.stringValue()); } else if (field.binaryValue() != null) { // We can't just pass field.binaryValue().bytes here as there may be offset/length // considerations @@ -187,6 +188,16 @@ public NumericDocValues getNormValues(String field) throws IOException { throw new UnsupportedOperationException(); } + @Override + public VectorValues getVectorValues(String field) throws IOException { + return null; + } + + @Override + public TopDocs searchNearestVectors(String field, float[] target, int k, int fanout) throws IOException { + return null; + } + @Override public Bits getLiveDocs() { throw new UnsupportedOperationException(); @@ -248,6 +259,8 @@ private static FieldInfo fieldInfo(String name) { 0, 0, 0, + 0, + VectorValues.SimilarityFunction.NONE, false ); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 99ac9d3ca446b..4104a9301afc4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -13,14 +13,14 @@ import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; import org.apache.lucene.document.FloatPoint; -import org.apache.lucene.document.HalfFloatPoint; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.sandbox.document.HalfFloatPoint; +import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; -import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; diff --git a/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java index e2cf9a82e2d13..f23eecf51ea3f 100644 --- a/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java @@ -12,15 +12,15 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute; import org.apache.lucene.index.Term; +import org.apache.lucene.misc.search.similarity.LegacyBM25Similarity; +import org.apache.lucene.sandbox.search.CombinedFieldQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostAttribute; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.XCombinedFieldQuery; import org.apache.lucene.search.similarities.BM25Similarity; import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.search.similarity.LegacyBM25Similarity; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.QueryBuilder; import org.elasticsearch.common.ParseField; @@ -395,7 +395,7 @@ public Query createPhraseQuery(String field, String queryText, int phraseSlop) { @Override protected Query newSynonymQuery(TermAndBoost[] terms) { - XCombinedFieldQuery.Builder query = new XCombinedFieldQuery.Builder(); + CombinedFieldQuery.Builder query = new CombinedFieldQuery.Builder(); for (TermAndBoost termAndBoost : terms) { assert termAndBoost.boost == BoostAttribute.DEFAULT_BOOST; BytesRef bytes = termAndBoost.term.bytes(); diff --git a/server/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java index 541742ae81e97..50ca1682f3dc4 100644 --- a/server/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java @@ -14,6 +14,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TwoPhaseIterator; @@ -165,6 +166,11 @@ public int hashCode() { return h; } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { diff --git a/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java index c2d67e560d6eb..5503de1348912 100644 --- a/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java @@ -10,8 +10,8 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.Term; +import org.apache.lucene.sandbox.search.CoveringQuery; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.CoveringQuery; import org.apache.lucene.search.DoubleValues; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LongValues; diff --git a/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java b/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java index abe470563271b..b5f637b15a4c1 100644 --- a/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java +++ b/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java @@ -425,11 +425,6 @@ private SpanQuery newSpanQuery(Term[] terms, boolean isPrefix) { return new SpanOrQuery(spanQueries); } - @Override - protected SpanQuery createSpanQuery(TokenStream in, String field) throws IOException { - return createSpanQuery(in, field, false); - } - private SpanQuery createSpanQuery(TokenStream in, String field, boolean isPrefix) throws IOException { TermToBytesRefAttribute termAtt = in.getAttribute(TermToBytesRefAttribute.class); PositionIncrementAttribute posIncAtt = in.getAttribute(PositionIncrementAttribute.class); diff --git a/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java b/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java index 126f21bfcfc23..69fb5130dae64 100644 --- a/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java +++ b/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java @@ -602,11 +602,12 @@ private Query getPossiblyAnalyzedPrefixQuery(String field, String termStr, Mappe } } else if (isLastPos == false) { // build a synonym query for terms in the same position. - Term[] terms = new Term[plist.size()]; - for (int i = 0; i < plist.size(); i++) { - terms[i] = new Term(field, plist.get(i)); + SynonymQuery.Builder sb = new SynonymQuery.Builder(field); + for (String synonym : plist) { + sb.addTerm(new Term(field, synonym)); + } - posQuery = new SynonymQuery(terms); + posQuery = sb.build(); } else { List innerClauses = new ArrayList<>(); for (String token : plist) { diff --git a/server/src/main/java/org/elasticsearch/index/search/SimpleQueryStringQueryParser.java b/server/src/main/java/org/elasticsearch/index/search/SimpleQueryStringQueryParser.java index da844a4e63a3c..4ecaf22938211 100644 --- a/server/src/main/java/org/elasticsearch/index/search/SimpleQueryStringQueryParser.java +++ b/server/src/main/java/org/elasticsearch/index/search/SimpleQueryStringQueryParser.java @@ -254,11 +254,12 @@ private Query newPossiblyAnalyzedQuery(String field, String termStr, Analyzer an } } else if (isLastPos == false) { // build a synonym query for terms in the same position. - Term[] terms = new Term[plist.size()]; - for (int i = 0; i < plist.size(); i++) { - terms[i] = new Term(field, plist.get(i)); + SynonymQuery.Builder sb = new SynonymQuery.Builder(field); + for (BytesRef bytesRef : plist) { + sb.addTerm(new Term(field, bytesRef)); + } - posQuery = new SynonymQuery(terms); + posQuery = sb.build(); } else { BooleanQuery.Builder innerBuilder = new BooleanQuery.Builder(); for (BytesRef token : plist) { diff --git a/server/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java b/server/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java index c21eb9274c418..699d6a4d96162 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java +++ b/server/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java @@ -19,6 +19,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TwoPhaseIterator; @@ -185,6 +186,11 @@ public int hashCode() { return classHash() ^ result; } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + private static void findSplitDocs(String idField, Predicate includeInShard, LeafReader leafReader, IntConsumer consumer) throws IOException { Terms terms = leafReader.terms(idField); @@ -227,13 +233,10 @@ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { } @Override - public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { - spare.bytes = value; - spare.offset = 0; - spare.length = value.length; + public void stringField(FieldInfo fieldInfo, String value) throws IOException { switch (fieldInfo.name) { case RoutingFieldMapper.NAME: - routing = spare.utf8ToString(); + routing = value; break; default: throw new IllegalStateException("Unexpected field: " + fieldInfo.name); diff --git a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java index b4d5347830c45..2ed13460e84bd 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.misc.store.HardlinkCopyDirectoryWrapper; import org.apache.lucene.search.Sort; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FilterDirectory; @@ -142,7 +143,7 @@ void addIndices(final RecoveryState.Index indexRecoveryStats, final Directory ta assert sources.length > 0; final int luceneIndexCreatedVersionMajor = Lucene.readSegmentInfos(sources[0]).getIndexCreatedVersionMajor(); - final Directory hardLinkOrCopyTarget = new org.apache.lucene.store.HardlinkCopyDirectoryWrapper(target); + final Directory hardLinkOrCopyTarget = new HardlinkCopyDirectoryWrapper(target); IndexWriterConfig iwc = indexWriterConfigWithNoMerging(null) .setSoftDeletesField(Lucene.SOFT_DELETES_FIELD) diff --git a/server/src/main/java/org/elasticsearch/index/similarity/SimilarityProviders.java b/server/src/main/java/org/elasticsearch/index/similarity/SimilarityProviders.java index 8c22339e1d572..fe38d36f9cd96 100644 --- a/server/src/main/java/org/elasticsearch/index/similarity/SimilarityProviders.java +++ b/server/src/main/java/org/elasticsearch/index/similarity/SimilarityProviders.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.similarity; +import org.apache.lucene.misc.search.similarity.LegacyBM25Similarity; import org.apache.lucene.search.similarities.AfterEffect; import org.apache.lucene.search.similarities.AfterEffectB; import org.apache.lucene.search.similarities.AfterEffectL; @@ -37,7 +38,6 @@ import org.apache.lucene.search.similarities.NormalizationH2; import org.apache.lucene.search.similarities.NormalizationH3; import org.apache.lucene.search.similarities.NormalizationZ; -import org.apache.lucene.search.similarity.LegacyBM25Similarity; import org.elasticsearch.Version; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; @@ -238,9 +238,7 @@ public static LegacyBM25Similarity createBM25Similarity(Settings settings, Versi float b = settings.getAsFloat("b", 0.75f); boolean discountOverlaps = settings.getAsBoolean(DISCOUNT_OVERLAPS, true); - LegacyBM25Similarity similarity = new LegacyBM25Similarity(k1, b); - similarity.setDiscountOverlaps(discountOverlaps); - return similarity; + return new LegacyBM25Similarity(k1, b, discountOverlaps); } public static BooleanSimilarity createBooleanSimilarity(Settings settings, Version indexCreatedVersion) { diff --git a/server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java b/server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java index 4020169ed668f..216d4419d9f45 100644 --- a/server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java +++ b/server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java @@ -10,6 +10,7 @@ import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.misc.search.similarity.LegacyBM25Similarity; import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.TermStatistics; @@ -17,7 +18,6 @@ import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.search.similarities.Similarity.SimScorer; -import org.apache.lucene.search.similarity.LegacyBM25Similarity; import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; diff --git a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java index 88251f0872765..9ce8bc2821f66 100644 --- a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java +++ b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java @@ -18,7 +18,6 @@ import org.apache.lucene.store.MMapDirectory; import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.store.NativeFSLockFactory; -import org.apache.lucene.store.SimpleFSDirectory; import org.apache.lucene.store.SimpleFSLockFactory; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; @@ -80,7 +79,7 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index case MMAPFS: return setPreload(new MMapDirectory(location, lockFactory), lockFactory, preLoadExtensions); case SIMPLEFS: - return new SimpleFSDirectory(location, lockFactory); + // TODO nocommit throw an error here or just fall through? case NIOFS: return new NIOFSDirectory(location, lockFactory); default: diff --git a/server/src/main/java/org/elasticsearch/index/store/Store.java b/server/src/main/java/org/elasticsearch/index/store/Store.java index 44668de95c111..d4acad14ed5d6 100644 --- a/server/src/main/java/org/elasticsearch/index/store/Store.java +++ b/server/src/main/java/org/elasticsearch/index/store/Store.java @@ -33,7 +33,7 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.Lock; -import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; @@ -432,7 +432,7 @@ private void closeInternal() { public static MetadataSnapshot readMetadataSnapshot(Path indexLocation, ShardId shardId, NodeEnvironment.ShardLocker shardLocker, Logger logger) throws IOException { try (ShardLock lock = shardLocker.lock(shardId, "read metadata snapshot", TimeUnit.SECONDS.toMillis(5)); - Directory dir = new SimpleFSDirectory(indexLocation)) { + Directory dir = new NIOFSDirectory(indexLocation)) { failIfCorrupted(dir); return new MetadataSnapshot(null, dir, logger); } catch (IndexNotFoundException ex) { @@ -453,7 +453,7 @@ public static MetadataSnapshot readMetadataSnapshot(Path indexLocation, ShardId public static void tryOpenIndex(Path indexLocation, ShardId shardId, NodeEnvironment.ShardLocker shardLocker, Logger logger) throws IOException, ShardLockObtainFailedException { try (ShardLock lock = shardLocker.lock(shardId, "open index", TimeUnit.SECONDS.toMillis(5)); - Directory dir = new SimpleFSDirectory(indexLocation)) { + Directory dir = new NIOFSDirectory(indexLocation)) { failIfCorrupted(dir); SegmentInfos segInfo = Lucene.readSegmentInfos(dir); logger.trace("{} loaded segment info [{}]", shardId, segInfo); @@ -623,7 +623,6 @@ public void cleanupAndVerify(String reason, MetadataSnapshot sourceMetadata) thr // FNF should not happen since we hold a write lock? } catch (IOException ex) { if (existingFile.startsWith(IndexFileNames.SEGMENTS) - || existingFile.equals(IndexFileNames.OLD_SEGMENTS_GEN) || existingFile.startsWith(CORRUPTED_MARKER_NAME_PREFIX)) { // TODO do we need to also fail this if we can't delete the pending commit file? // if one of those files can't be deleted we better fail the cleanup otherwise we might leave an old commit @@ -968,9 +967,6 @@ public RecoveryDiff recoveryDiff(MetadataSnapshot recoveryTargetSnapshot) { final List perCommitStoreFiles = new ArrayList<>(); for (StoreFileMetadata meta : this) { - if (IndexFileNames.OLD_SEGMENTS_GEN.equals(meta.name())) { // legacy - continue; // we don't need that file at all - } final String segmentId = IndexFileNames.parseSegmentName(meta.name()); final String extension = IndexFileNames.getExtension(meta.name()); if (IndexFileNames.SEGMENTS.equals(segmentId) || @@ -1006,9 +1002,9 @@ public RecoveryDiff recoveryDiff(MetadataSnapshot recoveryTargetSnapshot) { } RecoveryDiff recoveryDiff = new RecoveryDiff(Collections.unmodifiableList(identical), Collections.unmodifiableList(different), Collections.unmodifiableList(missing)); - assert recoveryDiff.size() == this.metadata.size() - (metadata.containsKey(IndexFileNames.OLD_SEGMENTS_GEN) ? 1 : 0) + assert recoveryDiff.size() == this.metadata.size() : "some files are missing recoveryDiff size: [" + recoveryDiff.size() + "] metadata size: [" + - this.metadata.size() + "] contains segments.gen: [" + metadata.containsKey(IndexFileNames.OLD_SEGMENTS_GEN) + "]"; + this.metadata.size() + "]"; return recoveryDiff; } diff --git a/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java b/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java index c7b31cbae1044..4d6865976492b 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java +++ b/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java @@ -17,8 +17,8 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.store.OutputStreamIndexOutput; -import org.apache.lucene.store.SimpleFSDirectory; import org.elasticsearch.common.io.Channels; import org.elasticsearch.index.seqno.SequenceNumbers; @@ -141,7 +141,7 @@ public String toString() { } public static Checkpoint read(Path path) throws IOException { - try (Directory dir = new SimpleFSDirectory(path.getParent())) { + try (Directory dir = new NIOFSDirectory(path.getParent())) { try (IndexInput indexInput = dir.openInput(path.getFileName().toString(), IOContext.DEFAULT)) { // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here. CodecUtil.checksumEntireFile(indexInput); diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index d4f1c3ffbfab0..fd7689d6edf2d 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -11,7 +11,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.LRUQueryCache; @@ -128,11 +127,6 @@ protected CachingWeightWrapper(Weight in) { this.in = in; } - @Override - public void extractTerms(Set terms) { - in.extractTerms(terms); - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { shardKeyMap.add(context.reader()); diff --git a/server/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java b/server/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java index 0d04c3868cd24..db44de570391f 100644 --- a/server/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java +++ b/server/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java @@ -12,13 +12,13 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.analysis.hunspell.Dictionary; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.SimpleFSDirectory; -import org.elasticsearch.core.internal.io.IOUtils; +import org.apache.lucene.store.NIOFSDirectory; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.Environment; import java.io.IOException; @@ -184,7 +184,7 @@ private Dictionary loadDictionary(String locale, Settings nodeSettings, Environm affixStream = Files.newInputStream(affixFiles[0]); - try (Directory tmp = new SimpleFSDirectory(env.tmpFile())) { + try (Directory tmp = new NIOFSDirectory(env.tmpFile())) { return new Dictionary(tmp, "hunspell", affixStream, dicStreams, ignoreCase); } diff --git a/server/src/main/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzers.java b/server/src/main/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzers.java index 652ac3a789cb6..6277de09958c4 100644 --- a/server/src/main/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzers.java +++ b/server/src/main/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzers.java @@ -9,12 +9,12 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.CharArraySet; +import org.apache.lucene.analysis.classic.ClassicAnalyzer; import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.analysis.core.SimpleAnalyzer; import org.apache.lucene.analysis.core.StopAnalyzer; import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.analysis.en.EnglishAnalyzer; -import org.apache.lucene.analysis.standard.ClassicAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.elasticsearch.Version; import org.elasticsearch.indices.analysis.PreBuiltCacheFactory.CachingStrategy; diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java index 32b46be6936cf..761bdfadc4748 100644 --- a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -10,13 +10,12 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.lucene.analysis.util.CharFilterFactory; -import org.apache.lucene.analysis.util.TokenFilterFactory; -import org.apache.lucene.analysis.util.TokenizerFactory; +import org.apache.lucene.analysis.CharFilterFactory; +import org.apache.lucene.analysis.TokenFilterFactory; +import org.apache.lucene.analysis.TokenizerFactory; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.util.SPIClassIterator; import org.elasticsearch.Build; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; @@ -52,6 +51,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.ServiceLoader; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -469,12 +469,12 @@ public List loadExtensions(Class extensionPointType) { } private static List createExtensions(Class extensionPointType, Plugin plugin) { - SPIClassIterator classIterator = SPIClassIterator.get(extensionPointType, plugin.getClass().getClassLoader()); List extensions = new ArrayList<>(); - while (classIterator.hasNext()) { - Class extensionClass = classIterator.next(); - extensions.add(createExtension(extensionClass, extensionPointType, plugin)); - } + ServiceLoader.load(extensionPointType, plugin.getClass().getClassLoader()).stream() + .map(ServiceLoader.Provider::type) + .forEachOrdered(service -> { + extensions.add(createExtension(service, extensionPointType, plugin)); + }); return extensions; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/PointsSortedDocsProducer.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/PointsSortedDocsProducer.java index 810f3d7e9fcd4..f401914cf989c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/PointsSortedDocsProducer.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/PointsSortedDocsProducer.java @@ -14,9 +14,9 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.Query; import org.apache.lucene.util.DocIdSetBuilder; -import org.apache.lucene.util.FutureArrays; import java.io.IOException; +import java.util.Arrays; import java.util.function.ToLongFunction; /** @@ -137,9 +137,9 @@ public void visit(int docID, byte[] packedValue) throws IOException { @Override public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { if ((upperPointQuery != null && - FutureArrays.compareUnsigned(minPackedValue, 0, bytesPerDim, upperPointQuery, 0, bytesPerDim) > 0) || + Arrays.compareUnsigned(minPackedValue, 0, bytesPerDim, upperPointQuery, 0, bytesPerDim) > 0) || (lowerPointQuery != null && - FutureArrays.compareUnsigned(maxPackedValue, 0, bytesPerDim, lowerPointQuery, 0, bytesPerDim) < 0)) { + Arrays.compareUnsigned(maxPackedValue, 0, bytesPerDim, lowerPointQuery, 0, bytesPerDim) < 0)) { // does not match the query return PointValues.Relation.CELL_OUTSIDE_QUERY; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQuery.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQuery.java index 5d965a3e6348d..f881a97daa5b0 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQuery.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQuery.java @@ -18,6 +18,7 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.ScorerSupplier; @@ -180,6 +181,13 @@ public boolean equals(Object obj) { && delegateForSingleValuedSegments.equals(other.delegateForSingleValuedSegments); } + @Override + public void visit(QueryVisitor visitor) { + if (visitor.acceptField(field)) { + this.delegateForMultiValuedSegments.visit(visitor); + } + } + @Override public int hashCode() { return Objects.hash(classHash(), delegateForMultiValuedSegments, delegateForSingleValuedSegments); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java index 09bdf9f7866a2..28b0c98a0f958 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PointValues; +import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BulkScorer; @@ -19,7 +20,6 @@ import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java index bf61d9e2a392a..96eae3c5923e5 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java @@ -10,8 +10,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.search.DiversifiedTopDocsCollector; -import org.apache.lucene.search.DiversifiedTopDocsCollector.ScoreDocKey; +import org.apache.lucene.misc.search.DiversifiedTopDocsCollector; import org.apache.lucene.search.TopDocsCollector; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; @@ -28,6 +27,8 @@ import java.util.Map; import java.util.function.Consumer; +import static org.apache.lucene.misc.search.DiversifiedTopDocsCollector.ScoreDocKey; + /** * Alternative, faster implementation for converting String keys to longs but * with the potential for hash collisions. diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java index 67219b1b7a58c..7fec8f471da7e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java @@ -10,8 +10,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.search.DiversifiedTopDocsCollector; -import org.apache.lucene.search.DiversifiedTopDocsCollector.ScoreDocKey; +import org.apache.lucene.misc.search.DiversifiedTopDocsCollector; import org.apache.lucene.search.TopDocsCollector; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; @@ -30,6 +29,8 @@ import java.util.Map; import java.util.function.Consumer; +import static org.apache.lucene.misc.search.DiversifiedTopDocsCollector.ScoreDocKey; + public class DiversifiedMapSamplerAggregator extends SamplerAggregator { private ValuesSource valuesSource; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java index 0f54717091b80..53cd9161409fb 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java @@ -11,8 +11,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.search.DiversifiedTopDocsCollector; -import org.apache.lucene.search.DiversifiedTopDocsCollector.ScoreDocKey; +import org.apache.lucene.misc.search.DiversifiedTopDocsCollector; import org.apache.lucene.search.TopDocsCollector; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.index.fielddata.AbstractNumericDocValues; @@ -27,6 +26,8 @@ import java.util.Map; import java.util.function.Consumer; +import static org.apache.lucene.misc.search.DiversifiedTopDocsCollector.ScoreDocKey; + public class DiversifiedNumericSamplerAggregator extends SamplerAggregator { private ValuesSource.Numeric valuesSource; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java index 08f4eeb430818..757acce70266e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java @@ -13,8 +13,7 @@ import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.search.DiversifiedTopDocsCollector; -import org.apache.lucene.search.DiversifiedTopDocsCollector.ScoreDocKey; +import org.apache.lucene.misc.search.DiversifiedTopDocsCollector; import org.apache.lucene.search.TopDocsCollector; import org.elasticsearch.index.fielddata.AbstractNumericDocValues; import org.elasticsearch.search.aggregations.Aggregator; @@ -28,6 +27,8 @@ import java.util.Map; import java.util.function.Consumer; +import static org.apache.lucene.misc.search.DiversifiedTopDocsCollector.ScoreDocKey; + public class DiversifiedOrdinalsSamplerAggregator extends SamplerAggregator { private ValuesSource.Bytes.WithOrdinals.FieldData valuesSource; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregator.java index 49b67c0af103f..11e5d2fb9a23d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregator.java @@ -8,7 +8,7 @@ package org.elasticsearch.search.aggregations.bucket.sampler; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.DiversifiedTopDocsCollector; +import org.apache.lucene.misc.search.DiversifiedTopDocsCollector; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.ParseField; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java index 7d71fbc438f62..2a6a0201ec340 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java @@ -113,7 +113,7 @@ public InternalAggregation reduce(List aggregations, Reduce shardDocs[i] = topHitsAgg.topDocs.topDocs; shardHits[i] = topHitsAgg.searchHits; } - reducedTopDocs = TopDocs.merge(sort, from, size, (TopFieldDocs[]) shardDocs, true); + reducedTopDocs = TopDocs.merge(sort, from, size, (TopFieldDocs[]) shardDocs); } else { shardDocs = new TopDocs[aggregations.size()]; for (int i = 0; i < shardDocs.length; i++) { @@ -121,7 +121,7 @@ public InternalAggregation reduce(List aggregations, Reduce shardDocs[i] = topHitsAgg.topDocs.topDocs; shardHits[i] = topHitsAgg.searchHits; } - reducedTopDocs = TopDocs.merge(from, size, shardDocs, true); + reducedTopDocs = TopDocs.merge(from, size, shardDocs); } float maxScore = Float.NaN; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregator.java index 88a7ebf9c3a81..1da09bf4bf6c4 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregator.java @@ -12,7 +12,6 @@ import org.apache.lucene.index.PointValues; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.FutureArrays; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.util.DoubleArray; import org.elasticsearch.index.fielddata.NumericDoubleValues; @@ -28,6 +27,7 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import java.io.IOException; +import java.util.Arrays; import java.util.Map; import java.util.function.Function; @@ -171,7 +171,7 @@ public void visit(int docID, byte[] packedValue) { @Override public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { - if (FutureArrays.equals(maxValue, 0, numBytes, maxPackedValue, 0, numBytes)) { + if (Arrays.equals(maxValue, 0, numBytes, maxPackedValue, 0, numBytes)) { // we only check leaves that contain the max value for the segment. return PointValues.Relation.CELL_CROSSES_QUERY; } else { diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 86cd4d44f0d39..ce6b1935e9fd4 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -208,11 +208,6 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto private Weight wrapWeight(Weight weight) { if (cancellable.isEnabled()) { return new Weight(weight.getQuery()) { - @Override - public void extractTerms(Set terms) { - throw new UnsupportedOperationException(); - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { throw new UnsupportedOperationException(); diff --git a/server/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java b/server/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java index 52a520b01fab4..eb0c4fcb887f5 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java +++ b/server/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java @@ -9,7 +9,6 @@ package org.elasticsearch.search.profile.query; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.Query; @@ -19,7 +18,6 @@ import org.elasticsearch.search.profile.Timer; import java.io.IOException; -import java.util.Set; /** * Weight wrapper that will compute how much time it takes to build the @@ -102,11 +100,6 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio return subQueryWeight.explain(context, doc); } - @Override - public void extractTerms(Set set) { - subQueryWeight.extractTerms(set); - } - @Override public boolean isCacheable(LeafReaderContext ctx) { return false; diff --git a/server/src/main/java/org/elasticsearch/search/runtime/AbstractScriptFieldQuery.java b/server/src/main/java/org/elasticsearch/search/runtime/AbstractScriptFieldQuery.java index 9acf2b6c9cc83..5c929f49a84cc 100644 --- a/server/src/main/java/org/elasticsearch/search/runtime/AbstractScriptFieldQuery.java +++ b/server/src/main/java/org/elasticsearch/search/runtime/AbstractScriptFieldQuery.java @@ -15,6 +15,7 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TwoPhaseIterator; @@ -111,6 +112,13 @@ public boolean equals(Object obj) { return script.equals(other.script) && fieldName.equals(other.fieldName); } + @Override + public void visit(QueryVisitor visitor) { + if (visitor.acceptField(fieldName)) { + visitor.visitLeaf(this); + } + } + final Explanation explainMatch(float boost, String description) { return Explanation.match( boost, diff --git a/server/src/main/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQuery.java b/server/src/main/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQuery.java index 9d8fd3768ba53..a7977c18d338c 100644 --- a/server/src/main/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQuery.java +++ b/server/src/main/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQuery.java @@ -11,7 +11,6 @@ import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.geo.GeoUtils; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; @@ -26,7 +25,6 @@ import java.io.IOException; import java.util.Objects; -import java.util.Set; import java.util.function.Function; public final class GeoPointScriptFieldDistanceFeatureQuery extends AbstractScriptFieldQuery { @@ -79,9 +77,6 @@ public boolean isCacheable(LeafReaderContext ctx) { return false; } - @Override - public void extractTerms(Set terms) {} - @Override public Scorer scorer(LeafReaderContext context) { return new DistanceScorer(this, scriptContextFunction().apply(context), context.reader().maxDoc(), boost); diff --git a/server/src/main/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQuery.java b/server/src/main/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQuery.java index f0b9a76bb65da..c5a751e4ba0e8 100644 --- a/server/src/main/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQuery.java +++ b/server/src/main/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQuery.java @@ -9,7 +9,6 @@ package org.elasticsearch.search.runtime; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; @@ -23,7 +22,6 @@ import java.io.IOException; import java.util.Objects; -import java.util.Set; import java.util.function.Function; public final class LongScriptFieldDistanceFeatureQuery extends AbstractScriptFieldQuery { @@ -56,9 +54,6 @@ public boolean isCacheable(LeafReaderContext ctx) { return false; } - @Override - public void extractTerms(Set terms) {} - @Override public Scorer scorer(LeafReaderContext context) { return new DistanceScorer(this, scriptContextFunction().apply(context), context.reader().maxDoc(), boost); diff --git a/server/src/main/java/org/elasticsearch/search/slice/SliceQuery.java b/server/src/main/java/org/elasticsearch/search/slice/SliceQuery.java index 920f0a4691f86..3a9f8cc449561 100644 --- a/server/src/main/java/org/elasticsearch/search/slice/SliceQuery.java +++ b/server/src/main/java/org/elasticsearch/search/slice/SliceQuery.java @@ -9,6 +9,7 @@ package org.elasticsearch.search.slice; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import java.util.Objects; @@ -67,4 +68,10 @@ public String toString(String f) { return getClass().getSimpleName() + "[field=" + field + ", id=" + id + ", max=" + max + "]"; } + @Override + public void visit(QueryVisitor visitor) { + if (visitor.acceptField(field)) { + visitor.visitLeaf(this); + } + } } diff --git a/server/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java b/server/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java index 64528e25a9acb..815c86d8ab476 100644 --- a/server/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java @@ -15,7 +15,6 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermStates; import org.apache.lucene.search.BooleanClause; @@ -24,8 +23,8 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryUtils; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.similarities.BM25Similarity; @@ -211,8 +210,7 @@ public void testExtractTerms() throws IOException { BlendedTermQuery blendedTermQuery = BlendedTermQuery.dismaxBlendedQuery(terms.toArray(new Term[0]), random().nextFloat()); Set extracted = new HashSet<>(); - IndexSearcher searcher = new IndexSearcher(new MultiReader()); - searcher.createWeight(searcher.rewrite(blendedTermQuery), ScoreMode.COMPLETE_NO_SCORES, 1f).extractTerms(extracted); + blendedTermQuery.visit(QueryVisitor.termCollector(extracted)); assertThat(extracted.size(), equalTo(terms.size())); assertThat(extracted, containsInAnyOrder(terms.toArray(new Term[0]))); } diff --git a/server/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java b/server/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java index 3123723b49659..cccdd89398c77 100644 --- a/server/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java @@ -8,7 +8,6 @@ package org.apache.lucene.queries; import org.apache.lucene.document.InetAddressPoint; -import org.apache.lucene.util.FutureArrays; import org.elasticsearch.index.mapper.RangeType; import java.net.InetAddress; @@ -33,7 +32,7 @@ protected Range nextRange(int dimensions) throws Exception { byte[] bMin = InetAddressPoint.encode(min); InetAddress max = nextInetaddress(); byte[] bMax = InetAddressPoint.encode(max); - if (FutureArrays.compareUnsigned(bMin, 0, bMin.length, bMax, 0, bMin.length) > 0) { + if (Arrays.compareUnsigned(bMin, 0, bMin.length, bMax, 0, bMin.length) > 0) { return new IpRange(max, min); } return new IpRange(min, max); @@ -80,7 +79,7 @@ protected void setMin(int dim, Object val) { InetAddress v = (InetAddress)val; byte[] e = InetAddressPoint.encode(v); - if (FutureArrays.compareUnsigned(min, 0, e.length, e, 0, e.length) < 0) { + if (Arrays.compareUnsigned(min, 0, e.length, e, 0, e.length) < 0) { max = e; maxAddress = v; } else { @@ -100,7 +99,7 @@ protected void setMax(int dim, Object val) { InetAddress v = (InetAddress)val; byte[] e = InetAddressPoint.encode(v); - if (FutureArrays.compareUnsigned(max, 0, e.length, e, 0, e.length) > 0) { + if (Arrays.compareUnsigned(max, 0, e.length, e, 0, e.length) > 0) { min = e; minAddress = v; } else { @@ -112,22 +111,22 @@ protected void setMax(int dim, Object val) { @Override protected boolean isDisjoint(Range o) { IpRange other = (IpRange) o; - return FutureArrays.compareUnsigned(min, 0, min.length, other.max, 0, min.length) > 0 || - FutureArrays.compareUnsigned(max, 0, max.length, other.min, 0, max.length) < 0; + return Arrays.compareUnsigned(min, 0, min.length, other.max, 0, min.length) > 0 || + Arrays.compareUnsigned(max, 0, max.length, other.min, 0, max.length) < 0; } @Override protected boolean isWithin(Range o) { IpRange other = (IpRange)o; - return FutureArrays.compareUnsigned(min, 0, min.length, other.min, 0, min.length) >= 0 && - FutureArrays.compareUnsigned(max, 0, max.length, other.max, 0, max.length) <= 0; + return Arrays.compareUnsigned(min, 0, min.length, other.min, 0, min.length) >= 0 && + Arrays.compareUnsigned(max, 0, max.length, other.max, 0, max.length) <= 0; } @Override protected boolean contains(Range o) { IpRange other = (IpRange)o; - return FutureArrays.compareUnsigned(min, 0, min.length, other.min, 0, min.length) <= 0 && - FutureArrays.compareUnsigned(max, 0, max.length, other.max, 0, max.length) >= 0; + return Arrays.compareUnsigned(min, 0, min.length, other.min, 0, min.length) <= 0 && + Arrays.compareUnsigned(max, 0, max.length, other.max, 0, max.length) >= 0; } } diff --git a/server/src/test/java/org/apache/lucene/search/XCombinedFieldQueryTests.java b/server/src/test/java/org/apache/lucene/search/XCombinedFieldQueryTests.java deleted file mode 100644 index 4f79b51b21022..0000000000000 --- a/server/src/test/java/org/apache/lucene/search/XCombinedFieldQueryTests.java +++ /dev/null @@ -1,343 +0,0 @@ -/* @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.lucene.search; - -import com.carrotsearch.randomizedtesting.generators.RandomPicks; -import java.io.IOException; -import java.util.Arrays; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field.Store; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.FieldInvertState; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.MultiReader; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.similarities.BM25Similarity; -import org.apache.lucene.search.similarities.BooleanSimilarity; -import org.apache.lucene.search.similarities.ClassicSimilarity; -import org.apache.lucene.search.similarities.LMDirichletSimilarity; -import org.apache.lucene.search.similarities.LMJelinekMercerSimilarity; -import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LuceneTestCase; - -/** - * TODO: this is temporarily copied from Lucene, remove once we update to Lucene 8.9. - */ -public class XCombinedFieldQueryTests extends LuceneTestCase { - public void testInvalid() { - XCombinedFieldQuery.Builder builder = new XCombinedFieldQuery.Builder(); - IllegalArgumentException exc = - expectThrows(IllegalArgumentException.class, () -> builder.addField("foo", 0.5f)); - assertEquals(exc.getMessage(), "weight must be greater or equal to 1"); - } - - public void testRewrite() throws IOException { - XCombinedFieldQuery.Builder builder = new XCombinedFieldQuery.Builder(); - IndexReader reader = new MultiReader(); - IndexSearcher searcher = new IndexSearcher(reader); - Query actual = searcher.rewrite(builder.build()); - assertEquals(actual, new MatchNoDocsQuery()); - builder.addField("field", 1f); - actual = searcher.rewrite(builder.build()); - assertEquals(actual, new MatchNoDocsQuery()); - builder.addTerm(new BytesRef("foo")); - actual = searcher.rewrite(builder.build()); - assertEquals(actual, new TermQuery(new Term("field", "foo"))); - builder.addTerm(new BytesRef("bar")); - actual = searcher.rewrite(builder.build()); - assertEquals( - actual, - new SynonymQuery.Builder("field") - .addTerm(new Term("field", "foo")) - .addTerm(new Term("field", "bar")) - .build()); - builder.addField("another_field", 1f); - Query query = builder.build(); - actual = searcher.rewrite(query); - assertEquals(actual, query); - } - - public void testToString() { - assertEquals("CombinedFieldQuery(()())", new XCombinedFieldQuery.Builder().build().toString()); - XCombinedFieldQuery.Builder builder = new XCombinedFieldQuery.Builder(); - builder.addField("foo", 1f); - assertEquals("CombinedFieldQuery((foo)())", builder.build().toString()); - builder.addTerm(new BytesRef("bar")); - assertEquals("CombinedFieldQuery((foo)(bar))", builder.build().toString()); - builder.addField("title", 3f); - assertEquals("CombinedFieldQuery((foo title^3.0)(bar))", builder.build().toString()); - builder.addTerm(new BytesRef("baz")); - assertEquals("CombinedFieldQuery((foo title^3.0)(bar baz))", builder.build().toString()); - } - - public void testSameScore() throws IOException { - Directory dir = newDirectory(); - Similarity similarity = randomCompatibleSimilarity(); - - IndexWriterConfig iwc = new IndexWriterConfig(); - iwc.setSimilarity(similarity); - RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); - - Document doc = new Document(); - doc.add(new StringField("f", "a", Store.NO)); - w.addDocument(doc); - - doc = new Document(); - doc.add(new StringField("g", "a", Store.NO)); - for (int i = 0; i < 10; ++i) { - w.addDocument(doc); - } - - IndexReader reader = w.getReader(); - IndexSearcher searcher = newSearcher(reader); - searcher.setSimilarity(similarity); - XCombinedFieldQuery query = - new XCombinedFieldQuery.Builder() - .addField("f", 1f) - .addField("g", 1f) - .addTerm(new BytesRef("a")) - .build(); - TopScoreDocCollector collector = - TopScoreDocCollector.create( - Math.min(reader.numDocs(), Integer.MAX_VALUE), null, Integer.MAX_VALUE); - searcher.search(query, collector); - TopDocs topDocs = collector.topDocs(); - assertEquals(new TotalHits(11, TotalHits.Relation.EQUAL_TO), topDocs.totalHits); - // All docs must have the same score - for (int i = 0; i < topDocs.scoreDocs.length; ++i) { - assertEquals(topDocs.scoreDocs[0].score, topDocs.scoreDocs[i].score, 0.0f); - } - - reader.close(); - w.close(); - dir.close(); - } - - public void testCopyField() throws IOException { - Directory dir = newDirectory(); - Similarity similarity = randomCompatibleSimilarity(); - - IndexWriterConfig iwc = new IndexWriterConfig(); - iwc.setSimilarity(similarity); - RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); - - int numMatch = atLeast(10); - int boost1 = Math.max(1, random().nextInt(5)); - int boost2 = Math.max(1, random().nextInt(5)); - for (int i = 0; i < numMatch; i++) { - Document doc = new Document(); - if (random().nextBoolean()) { - doc.add(new TextField("a", "baz", Store.NO)); - doc.add(new TextField("b", "baz", Store.NO)); - for (int k = 0; k < boost1 + boost2; k++) { - doc.add(new TextField("ab", "baz", Store.NO)); - } - w.addDocument(doc); - doc.clear(); - } - int freqA = random().nextInt(5) + 1; - for (int j = 0; j < freqA; j++) { - doc.add(new TextField("a", "foo", Store.NO)); - } - int freqB = random().nextInt(5) + 1; - for (int j = 0; j < freqB; j++) { - doc.add(new TextField("b", "foo", Store.NO)); - } - int freqAB = freqA * boost1 + freqB * boost2; - for (int j = 0; j < freqAB; j++) { - doc.add(new TextField("ab", "foo", Store.NO)); - } - w.addDocument(doc); - } - IndexReader reader = w.getReader(); - IndexSearcher searcher = newSearcher(reader); - - searcher.setSimilarity(similarity); - XCombinedFieldQuery query = - new XCombinedFieldQuery.Builder() - .addField("a", (float) boost1) - .addField("b", (float) boost2) - .addTerm(new BytesRef("foo")) - .build(); - - checkExpectedHits(searcher, numMatch, query, new TermQuery(new Term("ab", "foo"))); - - reader.close(); - w.close(); - dir.close(); - } - - public void testCopyFieldWithMultipleTerms() throws IOException { - Directory dir = newDirectory(); - Similarity similarity = randomCompatibleSimilarity(); - - IndexWriterConfig iwc = new IndexWriterConfig(); - iwc.setSimilarity(similarity); - RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); - - int numMatch = atLeast(10); - int boost1 = Math.max(1, random().nextInt(5)); - int boost2 = Math.max(1, random().nextInt(5)); - for (int i = 0; i < numMatch; i++) { - Document doc = new Document(); - - int freqA = random().nextInt(5) + 1; - for (int j = 0; j < freqA; j++) { - doc.add(new TextField("a", "foo", Store.NO)); - } - int freqB = random().nextInt(5) + 1; - for (int j = 0; j < freqB; j++) { - doc.add(new TextField("b", "bar", Store.NO)); - } - int freqAB = freqA * boost1 + freqB * boost2; - for (int j = 0; j < freqAB; j++) { - doc.add(new TextField("ab", "foo", Store.NO)); - } - w.addDocument(doc); - } - IndexReader reader = w.getReader(); - IndexSearcher searcher = newSearcher(reader); - - searcher.setSimilarity(similarity); - XCombinedFieldQuery query = - new XCombinedFieldQuery.Builder() - .addField("a", (float) boost1) - .addField("b", (float) boost2) - .addTerm(new BytesRef("foo")) - .addTerm(new BytesRef("bar")) - .build(); - - checkExpectedHits(searcher, numMatch, query, new TermQuery(new Term("ab", "foo"))); - - reader.close(); - w.close(); - dir.close(); - } - - private static Similarity randomCompatibleSimilarity() { - return RandomPicks.randomFrom( - random(), - Arrays.asList( - new BM25Similarity(), - new BooleanSimilarity(), - new ClassicSimilarity(), - new LMDirichletSimilarity(), - new LMJelinekMercerSimilarity(0.1f))); - } - - private void checkExpectedHits( - IndexSearcher searcher, int numHits, Query firstQuery, Query secondQuery) throws IOException { - TopScoreDocCollector firstCollector = - TopScoreDocCollector.create(numHits, null, Integer.MAX_VALUE); - searcher.search(firstQuery, firstCollector); - TopDocs firstTopDocs = firstCollector.topDocs(); - assertEquals(numHits, firstTopDocs.totalHits.value); - - TopScoreDocCollector secondCollector = - TopScoreDocCollector.create(numHits, null, Integer.MAX_VALUE); - searcher.search(secondQuery, secondCollector); - TopDocs secondTopDocs = secondCollector.topDocs(); - CheckHits.checkEqual(firstQuery, secondTopDocs.scoreDocs, firstTopDocs.scoreDocs); - } - - public void testDocWithNegativeNorms() throws IOException { - Directory dir = newDirectory(); - IndexWriterConfig iwc = new IndexWriterConfig(); - iwc.setSimilarity(new NegativeNormSimilarity()); - RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); - - String queryString = "foo"; - - Document doc = new Document(); - // both fields must contain tokens that match the query string "foo" - doc.add(new TextField("f", "foo", Store.NO)); - doc.add(new TextField("g", "foo baz", Store.NO)); - w.addDocument(doc); - - IndexReader reader = w.getReader(); - IndexSearcher searcher = newSearcher(reader); - searcher.setSimilarity(new BM25Similarity()); - XCombinedFieldQuery query = - new XCombinedFieldQuery.Builder() - .addField("f") - .addField("g") - .addTerm(new BytesRef(queryString)) - .build(); - TopDocs topDocs = searcher.search(query, 10); - CheckHits.checkDocIds("queried docs do not match", new int[] {0}, topDocs.scoreDocs); - - reader.close(); - w.close(); - dir.close(); - } - - public void testMultipleDocsNegativeNorms() throws IOException { - Directory dir = newDirectory(); - IndexWriterConfig iwc = new IndexWriterConfig(); - iwc.setSimilarity(new NegativeNormSimilarity()); - RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); - - String queryString = "foo"; - - Document doc0 = new Document(); - doc0.add(new TextField("f", "foo", Store.NO)); - doc0.add(new TextField("g", "foo baz", Store.NO)); - w.addDocument(doc0); - - Document doc1 = new Document(); - // add another match on the query string to the second doc - doc1.add(new TextField("f", "foo is foo", Store.NO)); - doc1.add(new TextField("g", "foo baz", Store.NO)); - w.addDocument(doc1); - - IndexReader reader = w.getReader(); - IndexSearcher searcher = newSearcher(reader); - searcher.setSimilarity(new BM25Similarity()); - XCombinedFieldQuery query = - new XCombinedFieldQuery.Builder() - .addField("f") - .addField("g") - .addTerm(new BytesRef(queryString)) - .build(); - TopDocs topDocs = searcher.search(query, 10); - // Return doc1 ahead of doc0 since its tf is higher - CheckHits.checkDocIds("queried docs do not match", new int[] {1, 0}, topDocs.scoreDocs); - - reader.close(); - w.close(); - dir.close(); - } - - private static final class NegativeNormSimilarity extends Similarity { - @Override - public long computeNorm(FieldInvertState state) { - return -128; - } - - @Override - public SimScorer scorer( - float boost, CollectionStatistics collectionStats, TermStatistics... termStats) { - return new BM25Similarity().scorer(boost, collectionStats, termStats); - } - } -} diff --git a/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java b/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java index 9dcb741ba91d7..8f99a2c7317a6 100644 --- a/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java +++ b/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java @@ -32,6 +32,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -434,6 +435,11 @@ public int hashCode() { return 42; } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new Weight(this) { @@ -443,11 +449,6 @@ public boolean isCacheable(LeafReaderContext ctx) { return true; } - @Override - public void extractTerms(Set terms) { - throw new UnsupportedOperationException(); - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { throw new UnsupportedOperationException(); diff --git a/server/src/test/java/org/elasticsearch/common/lucene/search/function/MinScoreScorerTests.java b/server/src/test/java/org/elasticsearch/common/lucene/search/function/MinScoreScorerTests.java index 0adbf9ad33e66..f21b528725808 100644 --- a/server/src/test/java/org/elasticsearch/common/lucene/search/function/MinScoreScorerTests.java +++ b/server/src/test/java/org/elasticsearch/common/lucene/search/function/MinScoreScorerTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.common.lucene.search.function; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.MatchAllDocsQuery; @@ -59,11 +58,6 @@ public int advance(int target) throws IOException { private static Weight fakeWeight() { return new Weight(new MatchAllDocsQuery()) { - @Override - public void extractTerms(Set terms) { - - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return null; diff --git a/server/src/test/java/org/elasticsearch/gateway/MetadataStateFormatTests.java b/server/src/test/java/org/elasticsearch/gateway/MetadataStateFormatTests.java index 262a0b783b0d4..b407c2f55ba35 100644 --- a/server/src/test/java/org/elasticsearch/gateway/MetadataStateFormatTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/MetadataStateFormatTests.java @@ -14,7 +14,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.MockDirectoryWrapper; -import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.metadata.Metadata; @@ -155,7 +155,7 @@ public void testCorruption() throws IOException { } public static void corruptFile(Path fileToCorrupt, Logger logger) throws IOException { - try (SimpleFSDirectory dir = new SimpleFSDirectory(fileToCorrupt.getParent())) { + try (NIOFSDirectory dir = new NIOFSDirectory(fileToCorrupt.getParent())) { long checksumBeforeCorruption; try (IndexInput input = dir.openInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) { checksumBeforeCorruption = CodecUtil.retrieveChecksum(input); diff --git a/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java index 80c0ea900e999..15703b3fda854 100644 --- a/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java @@ -17,7 +17,7 @@ import org.apache.lucene.store.FilterDirectory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.store.NIOFSDirectory; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -269,7 +269,7 @@ public void testFailsIfGlobalMetadataIsMissing() throws IOException { } final Path brokenPath = nodeEnvironment.nodeDataPath(); - try (Directory directory = new SimpleFSDirectory(brokenPath.resolve(PersistedClusterStateService.METADATA_DIRECTORY_NAME))) { + try (Directory directory = new NIOFSDirectory(brokenPath.resolve(PersistedClusterStateService.METADATA_DIRECTORY_NAME))) { final IndexWriterConfig indexWriterConfig = new IndexWriterConfig(); indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE); try (IndexWriter indexWriter = new IndexWriter(directory, indexWriterConfig)) { @@ -301,8 +301,8 @@ public void testFailsIfGlobalMetadataIsDuplicated() throws IOException { writeState(writer2, 0L, newClusterState, oldClusterState); } - try (Directory directory = new SimpleFSDirectory(brokenPath.resolve(PersistedClusterStateService.METADATA_DIRECTORY_NAME)); - Directory dupDirectory = new SimpleFSDirectory(dupPath.resolve(PersistedClusterStateService.METADATA_DIRECTORY_NAME))) { + try (Directory directory = new NIOFSDirectory(brokenPath.resolve(PersistedClusterStateService.METADATA_DIRECTORY_NAME)); + Directory dupDirectory = new NIOFSDirectory(dupPath.resolve(PersistedClusterStateService.METADATA_DIRECTORY_NAME))) { try (IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig())) { indexWriter.addIndexes(dupDirectory); indexWriter.commit(); @@ -346,8 +346,8 @@ public void testFailsIfIndexMetadataIsDuplicated() throws IOException { writeState(writer2, 0L, newClusterState, oldClusterState); } - try (Directory directory = new SimpleFSDirectory(brokenPath.resolve(PersistedClusterStateService.METADATA_DIRECTORY_NAME)); - Directory dupDirectory = new SimpleFSDirectory(dupPath.resolve(PersistedClusterStateService.METADATA_DIRECTORY_NAME))) { + try (Directory directory = new NIOFSDirectory(brokenPath.resolve(PersistedClusterStateService.METADATA_DIRECTORY_NAME)); + Directory dupDirectory = new NIOFSDirectory(dupPath.resolve(PersistedClusterStateService.METADATA_DIRECTORY_NAME))) { try (IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig())) { indexWriter.deleteDocuments(new Term("type", "global")); // do not duplicate global metadata indexWriter.addIndexes(dupDirectory); diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index 39c3a6f2a9001..b3aa9fe71995c 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -10,17 +10,14 @@ import org.apache.logging.log4j.LogManager; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat; -import org.apache.lucene.codecs.lucene87.Lucene87Codec; -import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat; -import org.apache.lucene.document.BinaryDocValuesField; +import org.apache.lucene.codecs.lucene90.Lucene90Codec; +import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.SegmentReader; import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; @@ -45,24 +42,21 @@ public class CodecTests extends ESTestCase { public void testResolveDefaultCodecs() throws Exception { CodecService codecService = createCodecService(); assertThat(codecService.codec("default"), instanceOf(PerFieldMappingPostingFormatCodec.class)); - assertThat(codecService.codec("default"), instanceOf(Lucene87Codec.class)); - assertThat(codecService.codec("Lucene87"), instanceOf(Lucene87Codec.class)); + assertThat(codecService.codec("default"), instanceOf(Lucene90Codec.class)); } public void testDefault() throws Exception { Codec codec = createCodecService().codec("default"); - assertStoredFieldsCompressionEquals(Lucene87StoredFieldsFormat.Mode.BEST_SPEED, codec); - assertDocValuesCompressionEquals(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsCompressionEquals(Lucene90Codec.Mode.BEST_SPEED, codec); } public void testBestCompression() throws Exception { Codec codec = createCodecService().codec("best_compression"); - assertStoredFieldsCompressionEquals(Lucene87StoredFieldsFormat.Mode.BEST_COMPRESSION, codec); - assertDocValuesCompressionEquals(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsCompressionEquals(Lucene90Codec.Mode.BEST_COMPRESSION, codec); } // write some docs with it, inspect .si to see this was the used compression - private void assertStoredFieldsCompressionEquals(Lucene87StoredFieldsFormat.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene90Codec.Mode expected, Codec actual) throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(null); iwc.setCodec(actual); @@ -72,29 +66,9 @@ private void assertStoredFieldsCompressionEquals(Lucene87StoredFieldsFormat.Mode iw.close(); DirectoryReader ir = DirectoryReader.open(dir); SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader(); - String v = sr.getSegmentInfo().info.getAttribute(Lucene87StoredFieldsFormat.MODE_KEY); + String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene87StoredFieldsFormat.Mode.valueOf(v)); - ir.close(); - dir.close(); - } - - // write some docs with it, inspect .si to see this was the used compression - private void assertDocValuesCompressionEquals(Lucene80DocValuesFormat.Mode expected, Codec actual) throws Exception { - Directory dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(null); - iwc.setCodec(actual); - IndexWriter iw = new IndexWriter(dir, iwc); - Document doc = new Document(); - doc.add(new BinaryDocValuesField("foo", new BytesRef("aaa"))); - iw.addDocument(doc); - iw.commit(); - iw.close(); - DirectoryReader ir = DirectoryReader.open(dir); - SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader(); - String v = sr.getFieldInfos().fieldInfo("foo").getAttribute(Lucene80DocValuesFormat.MODE_KEY); - assertNotNull(v); - assertEquals(expected, Lucene80DocValuesFormat.Mode.valueOf(v)); + assertEquals(expected, Lucene90Codec.Mode.valueOf(v)); ir.close(); dir.close(); } diff --git a/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java b/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java index d981062b678a1..c17661c3f6a94 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java @@ -8,7 +8,7 @@ package org.elasticsearch.index.engine; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene87.Lucene87Codec; +import org.apache.lucene.codecs.lucene90.Lucene90Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; @@ -46,7 +46,7 @@ public void testExceptionsAreNotCached() { public void testCompletionStatsCache() throws IOException, InterruptedException { final IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); final PostingsFormat postingsFormat = new Completion84PostingsFormat(); - indexWriterConfig.setCodec(new Lucene87Codec() { + indexWriterConfig.setCodec(new Lucene90Codec() { @Override public PostingsFormat getPostingsFormatForField(String field) { return postingsFormat; // all fields are suggest fields diff --git a/server/src/test/java/org/elasticsearch/index/engine/SegmentTests.java b/server/src/test/java/org/elasticsearch/index/engine/SegmentTests.java index e3722df957368..c3f32d4244a5f 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/SegmentTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/SegmentTests.java @@ -66,7 +66,7 @@ static Segment randomSegment() { segment.sizeInBytes = randomNonNegativeLong(); segment.docCount = randomIntBetween(1, Integer.MAX_VALUE); segment.delDocCount = randomIntBetween(0, segment.docCount); - segment.version = Version.LUCENE_7_0_0; + segment.version = Version.LUCENE_9_0_0; segment.compound = randomBoolean(); segment.mergeId = randomAlphaOfLengthBetween(1, 10); segment.memoryInBytes = randomNonNegativeLong(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index 6472fd8c124e4..93032b95bcf82 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -16,9 +16,9 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexOrDocValuesQuery; -import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.elasticsearch.Version; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index 6ada2959a1250..a6ae2ed13e1b6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -12,16 +12,16 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.FloatPoint; -import org.apache.lucene.document.HalfFloatPoint; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.sandbox.document.HalfFloatPoint; +import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.Sort; diff --git a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java index 789b214ba71e2..7947ad35ef4b8 100644 --- a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java @@ -8,12 +8,12 @@ package org.elasticsearch.index.query; +import org.apache.lucene.sandbox.search.CombinedFieldQuery; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.XCombinedFieldQuery; import org.elasticsearch.test.AbstractQueryTestCase; import java.io.IOException; @@ -64,7 +64,7 @@ protected void doAssertLuceneQuery(CombinedFieldsQueryBuilder queryBuilder, Quer instanceOf(TermQuery.class), instanceOf(MatchAllDocsQuery.class), instanceOf(MatchNoDocsQuery.class), - instanceOf(XCombinedFieldQuery.class) + instanceOf(CombinedFieldQuery.class) ))); } diff --git a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java index 8161b9917368d..995d87efc4711 100644 --- a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java @@ -13,13 +13,13 @@ import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.index.Term; +import org.apache.lucene.sandbox.search.CombinedFieldQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.XCombinedFieldQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; @@ -116,8 +116,8 @@ public void testWildcardFieldPattern() throws Exception { BooleanQuery booleanQuery = (BooleanQuery) query; assertThat(booleanQuery.clauses().size(), equalTo(2)); - assertThat(booleanQuery.clauses().get(0).getQuery(), instanceOf(XCombinedFieldQuery.class)); - assertThat(booleanQuery.clauses().get(1).getQuery(), instanceOf(XCombinedFieldQuery.class)); + assertThat(booleanQuery.clauses().get(0).getQuery(), instanceOf(CombinedFieldQuery.class)); + assertThat(booleanQuery.clauses().get(1).getQuery(), instanceOf(CombinedFieldQuery.class)); } public void testOperator() throws Exception { @@ -150,7 +150,7 @@ public void testQueryBoost() throws IOException { BoostQuery boostQuery = (BoostQuery) query; assertThat(boostQuery.getBoost(), equalTo(2.0f)); - assertThat(boostQuery.getQuery(), instanceOf(XCombinedFieldQuery.class)); + assertThat(boostQuery.getQuery(), instanceOf(CombinedFieldQuery.class)); } public void testInconsistentAnalyzers() { @@ -214,13 +214,13 @@ public void testCombinedFieldsWithSynonyms() throws IOException { .toQuery(context); Query expected = new BooleanQuery.Builder() - .add(new XCombinedFieldQuery.Builder() + .add(new CombinedFieldQuery.Builder() .addField("synonym1") .addField("synonym2") .addTerm(new BytesRef("dog")) .addTerm(new BytesRef("dogs")) .build(), BooleanClause.Occur.MUST) - .add(new XCombinedFieldQuery.Builder() + .add(new CombinedFieldQuery.Builder() .addField("synonym1") .addField("synonym2") .addTerm(new BytesRef("cats")) @@ -247,13 +247,13 @@ public void testSynonymsPhrase() throws IOException { .add(new Term("synonym2", "pig")) .build(), BooleanClause.Occur.SHOULD) .build(), BooleanClause.Occur.SHOULD) - .add(new XCombinedFieldQuery.Builder() + .add(new CombinedFieldQuery.Builder() .addField("synonym1") .addField("synonym2") .addTerm(new BytesRef("cavy")) .build(), BooleanClause.Occur.SHOULD) .build(), BooleanClause.Occur.MUST) - .add(new XCombinedFieldQuery.Builder() + .add(new CombinedFieldQuery.Builder() .addField("synonym1") .addField("synonym2") .addTerm(new BytesRef("cats")) @@ -272,24 +272,24 @@ public void testDisabledSynonymsPhrase() throws IOException { Query expected = new BooleanQuery.Builder() .add(new BooleanQuery.Builder() .add(new BooleanQuery.Builder() - .add(new XCombinedFieldQuery.Builder() + .add(new CombinedFieldQuery.Builder() .addField("synonym1") .addField("synonym2") .addTerm(new BytesRef("guinea")) .build(), BooleanClause.Occur.MUST) - .add(new XCombinedFieldQuery.Builder() + .add(new CombinedFieldQuery.Builder() .addField("synonym1") .addField("synonym2") .addTerm(new BytesRef("pig")) .build(), BooleanClause.Occur.MUST) .build(), BooleanClause.Occur.SHOULD) - .add(new XCombinedFieldQuery.Builder() + .add(new CombinedFieldQuery.Builder() .addField("synonym1") .addField("synonym2") .addTerm(new BytesRef("cavy")) .build(), BooleanClause.Occur.SHOULD) .build(), BooleanClause.Occur.MUST) - .add(new XCombinedFieldQuery.Builder() + .add(new CombinedFieldQuery.Builder() .addField("synonym1") .addField("synonym2") .addTerm(new BytesRef("cats")) @@ -312,8 +312,8 @@ public void testStopwords() throws Exception { .zeroTermsQuery(zeroTermsQuery) .toQuery(context); Query expected = new BooleanQuery.Builder() - .add(new XCombinedFieldQuery.Builder().addField("stopwords1").addTerm(quickTerm).build(), BooleanClause.Occur.SHOULD) - .add(new XCombinedFieldQuery.Builder().addField("stopwords1").addTerm(foxTerm).build(), BooleanClause.Occur.SHOULD) + .add(new CombinedFieldQuery.Builder().addField("stopwords1").addTerm(quickTerm).build(), BooleanClause.Occur.SHOULD) + .add(new CombinedFieldQuery.Builder().addField("stopwords1").addTerm(foxTerm).build(), BooleanClause.Occur.SHOULD) .build(); assertEquals(expected, query); @@ -323,12 +323,12 @@ public void testStopwords() throws Exception { .zeroTermsQuery(zeroTermsQuery) .toQuery(context); expected = new BooleanQuery.Builder() - .add(new XCombinedFieldQuery.Builder() + .add(new CombinedFieldQuery.Builder() .addField("stopwords1") .addField("stopwords2") .addTerm(quickTerm) .build(), BooleanClause.Occur.SHOULD) - .add(new XCombinedFieldQuery.Builder() + .add(new CombinedFieldQuery.Builder() .addField("stopwords1") .addField("stopwords2") .addTerm(foxTerm) diff --git a/server/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java index 451a4c729c872..f547ae25aa4ec 100644 --- a/server/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java @@ -18,14 +18,9 @@ import java.io.IOException; import java.util.Collection; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; -import static org.hamcrest.Matchers.closeTo; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; - public class DisMaxQueryBuilderTests extends AbstractQueryTestCase { /** * @return a {@link DisMaxQueryBuilder} with random inner queries @@ -46,14 +41,8 @@ protected DisMaxQueryBuilder doCreateTestQueryBuilder() { @Override protected void doAssertLuceneQuery(DisMaxQueryBuilder queryBuilder, Query query, SearchExecutionContext context) throws IOException { Collection queries = AbstractQueryBuilder.toQueries(queryBuilder.innerQueries(), context); - assertThat(query, instanceOf(DisjunctionMaxQuery.class)); - DisjunctionMaxQuery disjunctionMaxQuery = (DisjunctionMaxQuery) query; - assertThat(disjunctionMaxQuery.getTieBreakerMultiplier(), equalTo(queryBuilder.tieBreaker())); - assertThat(disjunctionMaxQuery.getDisjuncts().size(), equalTo(queries.size())); - Iterator queryIterator = queries.iterator(); - for (int i = 0; i < disjunctionMaxQuery.getDisjuncts().size(); i++) { - assertThat(disjunctionMaxQuery.getDisjuncts().get(i), equalTo(queryIterator.next())); - } + Query expected = new DisjunctionMaxQuery(queries, queryBuilder.tieBreaker()); + assertEquals(expected, query); } @Override @@ -92,20 +81,10 @@ public void testToQueryInnerPrefixQuery() throws Exception { " }\n" + "}"; Query query = parseQuery(queryAsString).toQuery(createSearchExecutionContext()); - assertThat(query, instanceOf(DisjunctionMaxQuery.class)); - DisjunctionMaxQuery disjunctionMaxQuery = (DisjunctionMaxQuery) query; - - List disjuncts = disjunctionMaxQuery.getDisjuncts(); - assertThat(disjuncts.size(), equalTo(1)); - - assertThat(disjuncts.get(0), instanceOf(BoostQuery.class)); - BoostQuery boostQuery = (BoostQuery) disjuncts.get(0); - assertThat((double) boostQuery.getBoost(), closeTo(1.2, 0.00001)); - assertThat(boostQuery.getQuery(), instanceOf(PrefixQuery.class)); - PrefixQuery firstQ = (PrefixQuery) boostQuery.getQuery(); - // since age is automatically registered in data, we encode it as numeric - assertThat(firstQ.getPrefix(), equalTo(new Term(TEXT_FIELD_NAME, "sh"))); - + Query expected = new DisjunctionMaxQuery(List.of( + new BoostQuery(new PrefixQuery(new Term(TEXT_FIELD_NAME, "sh")), 1.2f) + ), 1); + assertEquals(expected, query); } public void testFromJson() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/query/MatchBoolPrefixQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MatchBoolPrefixQueryBuilderTests.java index 57288be5cdbbf..d1b2bf106aea0 100644 --- a/server/src/test/java/org/elasticsearch/index/query/MatchBoolPrefixQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/MatchBoolPrefixQueryBuilderTests.java @@ -249,7 +249,10 @@ public void testAnalysisSynonym() throws Exception { assertBooleanQuery(query, asList( new TermQuery(new Term(TEXT_FIELD_NAME, "fox")), - new SynonymQuery(new Term(TEXT_FIELD_NAME, "dogs"), new Term(TEXT_FIELD_NAME, "dog")), + new SynonymQuery.Builder(TEXT_FIELD_NAME) + .addTerm(new Term(TEXT_FIELD_NAME, "dogs")) + .addTerm(new Term(TEXT_FIELD_NAME, "dog")) + .build(), new PrefixQuery(new Term(TEXT_FIELD_NAME, "red")) )); } diff --git a/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java index 67b1b002d8982..7c79f0d6e39ba 100644 --- a/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java @@ -39,7 +39,6 @@ import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBooleanSubQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertDisjunctionSubQuery; import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; @@ -197,34 +196,29 @@ public void testToQueryMultipleTermsBooleanQuery() throws Exception { public void testToQueryMultipleFieldsDisableDismax() throws Exception { Query query = multiMatchQuery("test").field(TEXT_FIELD_NAME).field(KEYWORD_FIELD_NAME).tieBreaker(1.0f) .toQuery(createSearchExecutionContext()); - assertThat(query, instanceOf(DisjunctionMaxQuery.class)); - DisjunctionMaxQuery dQuery = (DisjunctionMaxQuery) query; - assertThat(dQuery.getTieBreakerMultiplier(), equalTo(1.0f)); - assertThat(dQuery.getDisjuncts().size(), equalTo(2)); - assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 0).getTerm(), equalTo(new Term(TEXT_FIELD_NAME, "test"))); - assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 1).getTerm(), equalTo(new Term(KEYWORD_FIELD_NAME, "test"))); + Query expected = new DisjunctionMaxQuery( + List.of(new TermQuery(new Term(TEXT_FIELD_NAME, "test")), new TermQuery(new Term(KEYWORD_FIELD_NAME, "test"))), + 1 + ); + assertEquals(expected, query); } public void testToQueryMultipleFieldsDisMaxQuery() throws Exception { Query query = multiMatchQuery("test").field(TEXT_FIELD_NAME).field(KEYWORD_FIELD_NAME).toQuery(createSearchExecutionContext()); - assertThat(query, instanceOf(DisjunctionMaxQuery.class)); - DisjunctionMaxQuery disMaxQuery = (DisjunctionMaxQuery) query; - assertThat(disMaxQuery.getTieBreakerMultiplier(), equalTo(0.0f)); - List disjuncts = disMaxQuery.getDisjuncts(); - assertThat(disjuncts.get(0), instanceOf(TermQuery.class)); - assertThat(((TermQuery) disjuncts.get(0)).getTerm(), equalTo(new Term(TEXT_FIELD_NAME, "test"))); - assertThat(disjuncts.get(1), instanceOf(TermQuery.class)); - assertThat(((TermQuery) disjuncts.get(1)).getTerm(), equalTo(new Term(KEYWORD_FIELD_NAME, "test"))); + Query expected = new DisjunctionMaxQuery( + List.of(new TermQuery(new Term(TEXT_FIELD_NAME, "test")), new TermQuery(new Term(KEYWORD_FIELD_NAME, "test"))), + 0 + ); + assertEquals(expected, query); } public void testToQueryFieldsWildcard() throws Exception { Query query = multiMatchQuery("test").field("mapped_str*").tieBreaker(1.0f).toQuery(createSearchExecutionContext()); - assertThat(query, instanceOf(DisjunctionMaxQuery.class)); - DisjunctionMaxQuery dQuery = (DisjunctionMaxQuery) query; - assertThat(dQuery.getTieBreakerMultiplier(), equalTo(1.0f)); - assertThat(dQuery.getDisjuncts().size(), equalTo(2)); - assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 0).getTerm(), equalTo(new Term(TEXT_FIELD_NAME, "test"))); - assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 1).getTerm(), equalTo(new Term(KEYWORD_FIELD_NAME, "test"))); + Query expected = new DisjunctionMaxQuery( + List.of(new TermQuery(new Term(TEXT_FIELD_NAME, "test")), new TermQuery(new Term(KEYWORD_FIELD_NAME, "test"))), + 1 + ); + assertEquals(expected, query); } public void testToQueryFieldMissing() throws Exception { @@ -251,14 +245,15 @@ public void testToQueryBooleanPrefixMultipleFields() throws IOException { final MultiMatchQueryBuilder queryBuilder = new MultiMatchQueryBuilder("foo bar", TEXT_FIELD_NAME, KEYWORD_FIELD_NAME); queryBuilder.type(Type.BOOL_PREFIX); final Query query = queryBuilder.toQuery(createSearchExecutionContext()); - assertThat(query, instanceOf(DisjunctionMaxQuery.class)); - final DisjunctionMaxQuery disMaxQuery = (DisjunctionMaxQuery) query; - assertThat(disMaxQuery.getDisjuncts(), hasSize(2)); - final BooleanQuery firstDisjunct = assertDisjunctionSubQuery(disMaxQuery, BooleanQuery.class, 0); - assertThat(firstDisjunct.clauses(), hasSize(2)); - assertThat(assertBooleanSubQuery(firstDisjunct, TermQuery.class, 0).getTerm(), equalTo(new Term(TEXT_FIELD_NAME, "foo"))); - final PrefixQuery secondDisjunct = assertDisjunctionSubQuery(disMaxQuery, PrefixQuery.class, 1); - assertThat(secondDisjunct.getPrefix(), equalTo(new Term(KEYWORD_FIELD_NAME, "foo bar"))); + Query expected = new DisjunctionMaxQuery(List.of( + new BooleanQuery.Builder() + .add(new TermQuery(new Term(TEXT_FIELD_NAME, "foo")), BooleanClause.Occur.MUST) + .add(new PrefixQuery(new Term(TEXT_FIELD_NAME, "bar")), BooleanClause.Occur.MUST) + .build(), + new PrefixQuery(new Term(KEYWORD_FIELD_NAME, "foo bar"))), + 1 + ); + assertEquals(expected, query); } } diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java index 18aece54f34ad..35a70e01261c9 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java @@ -69,7 +69,6 @@ import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBooleanSubQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertDisjunctionSubQuery; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.hasItems; import static org.hamcrest.Matchers.containsString; @@ -489,34 +488,33 @@ public void testToQueryMultipleFieldsBooleanQuery() throws Exception { Query query = queryStringQuery("test").field(TEXT_FIELD_NAME) .field(KEYWORD_FIELD_NAME) .toQuery(createSearchExecutionContext()); - assertThat(query, instanceOf(DisjunctionMaxQuery.class)); - DisjunctionMaxQuery bQuery = (DisjunctionMaxQuery) query; - assertThat(bQuery.getDisjuncts().size(), equalTo(2)); - assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 0).getTerm(), - equalTo(new Term(TEXT_FIELD_NAME, "test"))); - assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 1).getTerm(), - equalTo(new Term(KEYWORD_FIELD_NAME, "test"))); + Query expected = new DisjunctionMaxQuery(List.of( + new TermQuery(new Term(TEXT_FIELD_NAME, "test")), + new TermQuery(new Term(KEYWORD_FIELD_NAME, "test"))), + 1 + ); + assertEquals(expected, query); } public void testToQueryMultipleFieldsDisMaxQuery() throws Exception { Query query = queryStringQuery("test").field(TEXT_FIELD_NAME).field(KEYWORD_FIELD_NAME) .toQuery(createSearchExecutionContext()); - assertThat(query, instanceOf(DisjunctionMaxQuery.class)); - DisjunctionMaxQuery disMaxQuery = (DisjunctionMaxQuery) query; - List disjuncts = disMaxQuery.getDisjuncts(); - assertThat(((TermQuery) disjuncts.get(0)).getTerm(), equalTo(new Term(TEXT_FIELD_NAME, "test"))); - assertThat(((TermQuery) disjuncts.get(1)).getTerm(), equalTo(new Term(KEYWORD_FIELD_NAME, "test"))); + Query expected = new DisjunctionMaxQuery(List.of( + new TermQuery(new Term(TEXT_FIELD_NAME, "test")), + new TermQuery(new Term(KEYWORD_FIELD_NAME, "test"))), + 1 + ); + assertEquals(expected, query); } public void testToQueryFieldsWildcard() throws Exception { Query query = queryStringQuery("test").field("mapped_str*").toQuery(createSearchExecutionContext()); - assertThat(query, instanceOf(DisjunctionMaxQuery.class)); - DisjunctionMaxQuery dQuery = (DisjunctionMaxQuery) query; - assertThat(dQuery.getDisjuncts().size(), equalTo(2)); - assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 0).getTerm(), - equalTo(new Term(TEXT_FIELD_NAME, "test"))); - assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 1).getTerm(), - equalTo(new Term(KEYWORD_FIELD_NAME, "test"))); + Query expected = new DisjunctionMaxQuery(List.of( + new TermQuery(new Term(TEXT_FIELD_NAME, "test")), + new TermQuery(new Term(KEYWORD_FIELD_NAME, "test"))), + 1 + ); + assertEquals(expected, query); } /** @@ -537,11 +535,12 @@ public void testToQueryDisMaxQuery() throws Exception { Query query = queryStringQuery("test").field(TEXT_FIELD_NAME, 2.2f) .field(KEYWORD_FIELD_NAME) .toQuery(createSearchExecutionContext()); - assertThat(query, instanceOf(DisjunctionMaxQuery.class)); - DisjunctionMaxQuery disMaxQuery = (DisjunctionMaxQuery) query; - List disjuncts = disMaxQuery.getDisjuncts(); - assertTermOrBoostQuery(disjuncts.get(0), TEXT_FIELD_NAME, "test", 2.2f); - assertTermOrBoostQuery(disjuncts.get(1), KEYWORD_FIELD_NAME, "test", 1.0f); + Query expected = new DisjunctionMaxQuery(List.of( + new BoostQuery(new TermQuery(new Term(TEXT_FIELD_NAME, "test")), 2.2f), + new TermQuery(new Term(KEYWORD_FIELD_NAME, "test"))), + 1 + ); + assertEquals(expected, query); } public void testToQueryWildcardQuery() throws Exception { @@ -592,13 +591,19 @@ public void testToQueryWilcardQueryWithSynonyms() throws Exception { Query query = queryParser.parse("first foo-bar-foobar* last"); Query expectedQuery = new BooleanQuery.Builder() - .add(new BooleanClause(new SynonymQuery(new Term(TEXT_FIELD_NAME, "first"), - new Term(TEXT_FIELD_NAME, "first")), defaultOp)) + .add(new BooleanClause(new SynonymQuery.Builder(TEXT_FIELD_NAME) + .addTerm(new Term(TEXT_FIELD_NAME, "first")) + .addTerm(new Term(TEXT_FIELD_NAME, "first")) + .build(), defaultOp)) .add(new BooleanQuery.Builder() - .add(new BooleanClause(new SynonymQuery(new Term(TEXT_FIELD_NAME, "foo"), - new Term(TEXT_FIELD_NAME, "foo")), defaultOp)) - .add(new BooleanClause(new SynonymQuery(new Term(TEXT_FIELD_NAME, "bar"), - new Term(TEXT_FIELD_NAME, "bar")), defaultOp)) + .add(new BooleanClause(new SynonymQuery.Builder(TEXT_FIELD_NAME) + .addTerm(new Term(TEXT_FIELD_NAME, "foo")) + .addTerm(new Term(TEXT_FIELD_NAME, "foo")) + .build(), defaultOp)) + .add(new BooleanClause(new SynonymQuery.Builder(TEXT_FIELD_NAME) + .addTerm(new Term(TEXT_FIELD_NAME, "bar")) + .addTerm(new Term(TEXT_FIELD_NAME, "bar")) + .build(), defaultOp)) .add(new BooleanQuery.Builder() .add(new BooleanClause(new PrefixQuery(new Term(TEXT_FIELD_NAME, "foobar")), BooleanClause.Occur.SHOULD)) @@ -606,8 +611,10 @@ public void testToQueryWilcardQueryWithSynonyms() throws Exception { BooleanClause.Occur.SHOULD)) .build(), defaultOp) .build(), defaultOp) - .add(new BooleanClause(new SynonymQuery(new Term(TEXT_FIELD_NAME, "last"), - new Term(TEXT_FIELD_NAME, "last")), defaultOp)) + .add(new BooleanClause(new SynonymQuery.Builder(TEXT_FIELD_NAME) + .addTerm(new Term(TEXT_FIELD_NAME, "last")) + .addTerm(new Term(TEXT_FIELD_NAME, "last")) + .build(), defaultOp)) .build(); assertThat(query, Matchers.equalTo(expectedQuery)); } diff --git a/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java index 157cbcddf756a..2f0127de0933c 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java @@ -461,13 +461,19 @@ public void testAnalyzerWildcardWithSynonyms() throws IOException { parser.setDefaultOperator(defaultOp); Query query = parser.parse("first foo-bar-foobar* last"); Query expectedQuery = new BooleanQuery.Builder() - .add(new BooleanClause(new SynonymQuery(new Term(TEXT_FIELD_NAME, "first"), - new Term(TEXT_FIELD_NAME, "first")), defaultOp)) + .add(new BooleanClause(new SynonymQuery.Builder(TEXT_FIELD_NAME) + .addTerm(new Term(TEXT_FIELD_NAME, "first")) + .addTerm(new Term(TEXT_FIELD_NAME, "first")) + .build(), defaultOp)) .add(new BooleanQuery.Builder() - .add(new BooleanClause(new SynonymQuery(new Term(TEXT_FIELD_NAME, "foo"), - new Term(TEXT_FIELD_NAME, "foo")), defaultOp)) - .add(new BooleanClause(new SynonymQuery(new Term(TEXT_FIELD_NAME, "bar"), - new Term(TEXT_FIELD_NAME, "bar")), defaultOp)) + .add(new BooleanClause(new SynonymQuery.Builder(TEXT_FIELD_NAME) + .addTerm(new Term(TEXT_FIELD_NAME, "foo")) + .addTerm(new Term(TEXT_FIELD_NAME, "foo")) + .build(), defaultOp)) + .add(new BooleanClause(new SynonymQuery.Builder(TEXT_FIELD_NAME) + .addTerm(new Term(TEXT_FIELD_NAME, "bar")) + .addTerm(new Term(TEXT_FIELD_NAME, "bar")) + .build(), defaultOp)) .add(new BooleanQuery.Builder() .add(new BooleanClause(new PrefixQuery(new Term(TEXT_FIELD_NAME, "foobar")), BooleanClause.Occur.SHOULD)) @@ -475,8 +481,10 @@ public void testAnalyzerWildcardWithSynonyms() throws IOException { BooleanClause.Occur.SHOULD)) .build(), defaultOp) .build(), defaultOp) - .add(new BooleanClause(new SynonymQuery(new Term(TEXT_FIELD_NAME, "last"), - new Term(TEXT_FIELD_NAME, "last")), defaultOp)) + .add(new BooleanClause(new SynonymQuery.Builder(TEXT_FIELD_NAME) + .addTerm(new Term(TEXT_FIELD_NAME, "last")) + .addTerm(new Term(TEXT_FIELD_NAME, "last")) + .build(), defaultOp)) .build(); assertThat(query, equalTo(expectedQuery)); } diff --git a/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java index b6ac9d53136eb..c97e2f2830a13 100644 --- a/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java @@ -18,7 +18,7 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.Term; -import org.apache.lucene.search.CoveringQuery; +import org.apache.lucene.sandbox.search.CoveringQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; diff --git a/server/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java b/server/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java index d6ed7f8eb573d..7d017ff996420 100644 --- a/server/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java +++ b/server/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java @@ -11,6 +11,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Weight; import org.elasticsearch.plugins.Plugin; @@ -50,5 +51,10 @@ public boolean equals(Object obj) { public int hashCode() { return classHash(); } + + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } } } diff --git a/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryParserTests.java b/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryParserTests.java index 85d4316f0fae2..f0071f6d677f3 100644 --- a/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryParserTests.java @@ -243,16 +243,16 @@ public void testMultiMatchCrossFieldsWithSynonyms() throws IOException { // check that synonym query is used for a single field Query parsedQuery = parser.parse(MultiMatchQueryBuilder.Type.CROSS_FIELDS, fieldNames, "dogs", null); - Term[] terms = new Term[2]; - terms[0] = new Term("name.first", "dog"); - terms[1] = new Term("name.first", "dogs"); - Query expectedQuery = new SynonymQuery(terms); + Query expectedQuery = new SynonymQuery.Builder("name.first") + .addTerm(new Term("name.first", "dog")) + .addTerm(new Term("name.first", "dogs")) + .build(); assertThat(parsedQuery, equalTo(expectedQuery)); // check that blended term query is used for multiple fields fieldNames.put("name.last", 1.0f); parsedQuery = parser.parse(MultiMatchQueryBuilder.Type.CROSS_FIELDS, fieldNames, "dogs", null); - terms = new Term[4]; + Term[] terms = new Term[4]; terms[0] = new Term("name.first", "dog"); terms[1] = new Term("name.first", "dogs"); terms[2] = new Term("name.last", "dog"); diff --git a/server/src/test/java/org/elasticsearch/index/similarity/ScriptedSimilarityTests.java b/server/src/test/java/org/elasticsearch/index/similarity/ScriptedSimilarityTests.java index 827c0bb9cdab8..c546dc3bcec37 100644 --- a/server/src/test/java/org/elasticsearch/index/similarity/ScriptedSimilarityTests.java +++ b/server/src/test/java/org/elasticsearch/index/similarity/ScriptedSimilarityTests.java @@ -50,8 +50,7 @@ public void testSameNormsAsBM25DiscountOverlaps() { private void doTestSameNormsAsBM25(boolean discountOverlaps) { ScriptedSimilarity sim1 = new ScriptedSimilarity("foobar", null, "foobaz", null, discountOverlaps); - BM25Similarity sim2 = new BM25Similarity(); - sim2.setDiscountOverlaps(discountOverlaps); + BM25Similarity sim2 = new BM25Similarity(discountOverlaps); for (int iter = 0; iter < 100; ++iter) { final int length = TestUtil.nextInt(random(), 1, 100); final int position = random().nextInt(length); diff --git a/server/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java b/server/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java index 8f20ad11c8f8d..72b5353574604 100644 --- a/server/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java @@ -8,11 +8,11 @@ package org.elasticsearch.index.similarity; import org.apache.lucene.index.FieldInvertState; +import org.apache.lucene.misc.search.similarity.LegacyBM25Similarity; import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.TermStatistics; import org.apache.lucene.search.similarities.BooleanSimilarity; import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.search.similarity.LegacyBM25Similarity; import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; diff --git a/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java b/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java index f71cb691a9fd6..167e08ee81588 100644 --- a/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java +++ b/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.similarity; +import org.apache.lucene.misc.search.similarity.LegacyBM25Similarity; import org.apache.lucene.search.similarities.AfterEffectL; import org.apache.lucene.search.similarities.BasicModelG; import org.apache.lucene.search.similarities.BooleanSimilarity; @@ -20,7 +21,6 @@ import org.apache.lucene.search.similarities.LMJelinekMercerSimilarity; import org.apache.lucene.search.similarities.LambdaTTF; import org.apache.lucene.search.similarities.NormalizationH2; -import org.apache.lucene.search.similarity.LegacyBM25Similarity; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; diff --git a/server/src/test/java/org/elasticsearch/index/store/FsDirectoryFactoryTests.java b/server/src/test/java/org/elasticsearch/index/store/FsDirectoryFactoryTests.java index 30da3b1f443a2..c3e7227c24eee 100644 --- a/server/src/test/java/org/elasticsearch/index/store/FsDirectoryFactoryTests.java +++ b/server/src/test/java/org/elasticsearch/index/store/FsDirectoryFactoryTests.java @@ -13,7 +13,6 @@ import org.apache.lucene.store.MMapDirectory; import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.store.NoLockFactory; -import org.apache.lucene.store.SimpleFSDirectory; import org.apache.lucene.store.SleepingLockWrapper; import org.apache.lucene.util.Constants; import org.elasticsearch.Version; @@ -138,13 +137,13 @@ private void doTestStoreDirectory(Path tempDir, String typeSettingValue, IndexMo assertTrue(type + " " + directory.toString(), directory instanceof MMapDirectory); break; case SIMPLEFS: - assertTrue(type + " " + directory.toString(), directory instanceof SimpleFSDirectory); + assertTrue(type + " " + directory.toString(), directory instanceof NIOFSDirectory); break; case FS: if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) { assertTrue(FsDirectoryFactory.isHybridFs(directory)); } else if (Constants.WINDOWS) { - assertTrue(directory.toString(), directory instanceof SimpleFSDirectory); + assertTrue(directory.toString(), directory instanceof NIOFSDirectory); } else { assertTrue(directory.toString(), directory instanceof NIOFSDirectory); } diff --git a/server/src/test/java/org/elasticsearch/index/store/StoreTests.java b/server/src/test/java/org/elasticsearch/index/store/StoreTests.java index cf50fd4f6f4dc..005e1a9c92dfb 100644 --- a/server/src/test/java/org/elasticsearch/index/store/StoreTests.java +++ b/server/src/test/java/org/elasticsearch/index/store/StoreTests.java @@ -16,7 +16,6 @@ import org.apache.lucene.document.TextField; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.index.IndexNotFoundException; @@ -459,8 +458,7 @@ public void assertDeleteContent(Store store, Directory dir) throws IOException { public static void assertConsistent(Store store, Store.MetadataSnapshot metadata) throws IOException { for (String file : store.directory().listAll()) { - if (IndexWriter.WRITE_LOCK_NAME.equals(file) == false && - IndexFileNames.OLD_SEGMENTS_GEN.equals(file) == false && file.startsWith("extra") == false) { + if (IndexWriter.WRITE_LOCK_NAME.equals(file) == false && file.startsWith("extra") == false) { assertTrue(file + " is not in the map: " + metadata.asMap().size() + " vs. " + store.directory().listAll().length, metadata.asMap().containsKey(file)); } else { diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java index 12cf61ac2e87f..3a18ee5a03893 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java @@ -12,7 +12,6 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocIdSetIterator; @@ -21,20 +20,20 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryCachingPolicy; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.ScorerSupplier; import org.apache.lucene.search.Weight; import org.apache.lucene.store.Directory; -import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.cache.query.QueryCacheStats; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; import java.io.IOException; -import java.util.Set; public class IndicesQueryCacheTests extends ESTestCase { @@ -56,6 +55,11 @@ public int hashCode() { return 31 * classHash() + id; } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + @Override public String toString(String field) { return "dummy"; @@ -352,11 +356,6 @@ private static class DummyWeight extends Weight { this.weight = weight; } - @Override - public void extractTerms(Set terms) { - weight.extractTerms(terms); - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return weight.explain(context, doc); diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java index 58ee159369a4a..264246c1d0336 100644 --- a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java @@ -18,7 +18,7 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.store.NIOFSDirectory; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.io.Streams; @@ -416,7 +416,7 @@ public void testRegisterHunspellDictionary() throws Exception { InputStream aff = getClass().getResourceAsStream("/indices/analyze/conf_dir/hunspell/en_US/en_US.aff"); InputStream dic = getClass().getResourceAsStream("/indices/analyze/conf_dir/hunspell/en_US/en_US.dic"); Dictionary dictionary; - try (Directory tmp = new SimpleFSDirectory(environment.tmpFile())) { + try (Directory tmp = new NIOFSDirectory(environment.tmpFile())) { dictionary = new Dictionary(tmp, "hunspell", aff, dic); } AnalysisModule module = new AnalysisModule(environment, singletonList(new AnalysisPlugin() { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java index 5cc4f2c162381..36b66e2054dc7 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java @@ -33,7 +33,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.FutureArrays; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; @@ -390,7 +389,7 @@ public void visit(int docID, byte[] packedValue) { @Override public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { - if (FutureArrays.equals(maxPackedValue, 0, numBytes, maxValue, 0, numBytes)) { + if (Arrays.equals(maxPackedValue, 0, numBytes, maxValue, 0, numBytes)) { return PointValues.Relation.CELL_CROSSES_QUERY; } return PointValues.Relation.CELL_OUTSIDE_QUERY; diff --git a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java index 1d5cb729a96ca..f563f1eae9927 100644 --- a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java @@ -34,6 +34,7 @@ import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.Scorable; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -392,11 +393,6 @@ private static class CreateScorerOnceWeight extends Weight { this.weight = weight; } - @Override - public void extractTerms(Set terms) { - weight.extractTerms(terms); - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return weight.explain(context, doc); @@ -457,5 +453,10 @@ public boolean equals(Object obj) { public int hashCode() { return 31 * classHash() + query.hashCode(); } + + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } } } diff --git a/server/src/test/java/org/elasticsearch/search/lookup/LeafStoredFieldsLookupTests.java b/server/src/test/java/org/elasticsearch/search/lookup/LeafStoredFieldsLookupTests.java index a744a70819400..f6acc9e5947de 100644 --- a/server/src/test/java/org/elasticsearch/search/lookup/LeafStoredFieldsLookupTests.java +++ b/server/src/test/java/org/elasticsearch/search/lookup/LeafStoredFieldsLookupTests.java @@ -10,6 +10,7 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.VectorValues; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.test.ESTestCase; import org.junit.Before; @@ -35,7 +36,7 @@ public void setUp() throws Exception { (Double) invocation.getArguments()[0] + 10); FieldInfo mockFieldInfo = new FieldInfo("field", 1, false, false, true, - IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, false); + IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, 0, VectorValues.SimilarityFunction.NONE, false); fieldsLookup = new LeafStoredFieldsLookup(field -> field.equals("field") || field.equals("alias") ? fieldType : null, (doc, visitor) -> visitor.doubleField(mockFieldInfo, 2.718)); diff --git a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java index 680a749c6fcf3..23457e3ea3c5d 100644 --- a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java +++ b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java @@ -23,6 +23,7 @@ import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryCachingPolicy; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.RandomApproximationQuery; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -44,7 +45,6 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.Set; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -230,14 +230,14 @@ public int hashCode() { return 0; } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new Weight(this) { - @Override - public void extractTerms(Set terms) { - throw new UnsupportedOperationException(); - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { throw new UnsupportedOperationException(); diff --git a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java index c6ab4d720949f..cee5c9939a959 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java @@ -13,7 +13,6 @@ import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; import org.apache.lucene.document.FloatPoint; -import org.apache.lucene.document.HalfFloatPoint; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.SortedNumericDocValuesField; @@ -21,6 +20,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; +import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.search.AssertingIndexSearcher; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.SortField; @@ -45,9 +45,9 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryRewriteContext; -import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.SearchSortValuesAndFormats; diff --git a/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java index 8bf9b018d235b..b91a420a3ebbc 100644 --- a/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java @@ -8,8 +8,8 @@ package org.elasticsearch.indices.analysis; -import org.apache.lucene.analysis.util.TokenFilterFactory; -import org.apache.lucene.analysis.util.TokenizerFactory; +import org.apache.lucene.analysis.TokenFilterFactory; +import org.apache.lucene.analysis.TokenizerFactory; import org.elasticsearch.index.analysis.HunspellTokenFilterFactory; import org.elasticsearch.index.analysis.ShingleTokenFilterFactory; import org.elasticsearch.index.analysis.StandardTokenizerFactory; @@ -261,7 +261,7 @@ public Map> getPreConfiguredCharFilters() { public void testTokenizers() { Set missing = new TreeSet(); - missing.addAll(org.apache.lucene.analysis.util.TokenizerFactory.availableTokenizers() + missing.addAll(org.apache.lucene.analysis.TokenizerFactory.availableTokenizers() .stream().map(key -> key.toLowerCase(Locale.ROOT)).collect(Collectors.toSet())); missing.removeAll(getTokenizers().keySet()); assertTrue("new tokenizers found, please update KNOWN_TOKENIZERS: " + missing.toString(), missing.isEmpty()); @@ -269,7 +269,7 @@ public void testTokenizers() { public void testCharFilters() { Set missing = new TreeSet(); - missing.addAll(org.apache.lucene.analysis.util.CharFilterFactory.availableCharFilters() + missing.addAll(org.apache.lucene.analysis.CharFilterFactory.availableCharFilters() .stream().map(key -> key.toLowerCase(Locale.ROOT)).collect(Collectors.toSet())); missing.removeAll(getCharFilters().keySet()); assertTrue("new charfilters found, please update KNOWN_CHARFILTERS: " + missing.toString(), missing.isEmpty()); @@ -277,7 +277,7 @@ public void testCharFilters() { public void testTokenFilters() { Set missing = new TreeSet(); - missing.addAll(org.apache.lucene.analysis.util.TokenFilterFactory.availableTokenFilters() + missing.addAll(org.apache.lucene.analysis.TokenFilterFactory.availableTokenFilters() .stream().map(key -> key.toLowerCase(Locale.ROOT)).collect(Collectors.toSet())); missing.removeAll(getTokenFilters().keySet()); assertTrue("new tokenfilters found, please update KNOWN_TOKENFILTERS: " + missing.toString(), missing.isEmpty()); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 242be982b0de5..e50ad0a87577f 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -10,7 +10,6 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Document; -import org.apache.lucene.document.HalfFloatPoint; import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.document.LatLonDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; @@ -24,6 +23,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.search.AssertingIndexSearcher; import org.apache.lucene.search.Collector; import org.apache.lucene.search.IndexSearcher; diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java index cdc5628285ef8..2b9a92570529a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java +++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java @@ -8,7 +8,6 @@ package org.elasticsearch.test.hamcrest; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TotalHits; import org.elasticsearch.ElasticsearchException; @@ -487,14 +486,6 @@ public static T assertBooleanSubQuery(Query query, Class su return subqueryType.cast(q.clauses().get(i).getQuery()); } - public static T assertDisjunctionSubQuery(Query query, Class subqueryType, int i) { - assertThat(query, instanceOf(DisjunctionMaxQuery.class)); - DisjunctionMaxQuery q = (DisjunctionMaxQuery) query; - assertThat(q.getDisjuncts().size(), greaterThan(i)); - assertThat(q.getDisjuncts().get(i), instanceOf(subqueryType)); - return subqueryType.cast(q.getDisjuncts().get(i)); - } - /** * Run the request from a given builder and check that it throws an exception of the right type */ diff --git a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/BlockingQueryBuilder.java b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/BlockingQueryBuilder.java index 679654a67d2b8..7fb330e40e450 100644 --- a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/BlockingQueryBuilder.java +++ b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/BlockingQueryBuilder.java @@ -8,6 +8,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Weight; import org.elasticsearch.common.io.stream.StreamInput; @@ -103,6 +104,11 @@ public boolean equals(Object obj) { public int hashCode() { return 0; } + + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } }; } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/ThrowingQueryBuilder.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/ThrowingQueryBuilder.java index b7d2d8e422ea8..04ce3d4a7c12a 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/ThrowingQueryBuilder.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/ThrowingQueryBuilder.java @@ -8,6 +8,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Weight; import org.elasticsearch.common.io.stream.StreamInput; @@ -82,6 +83,11 @@ public boolean equals(Object obj) { public int hashCode() { return 0; } + + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } }; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/RewriteCachingDirectoryReader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/RewriteCachingDirectoryReader.java index a30ba4309a7e3..c23e6182d1415 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/RewriteCachingDirectoryReader.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/RewriteCachingDirectoryReader.java @@ -23,6 +23,8 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.Terms; +import org.apache.lucene.index.VectorValues; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; @@ -213,6 +215,16 @@ public NumericDocValues getNormValues(String field) { throw new UnsupportedOperationException(); } + @Override + public VectorValues getVectorValues(String field) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public TopDocs searchNearestVectors(String field, float[] target, int k, int fanout) throws IOException { + throw new UnsupportedOperationException(); + } + @Override public FieldInfos getFieldInfos() { return fieldInfos; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java index 5a7894b62f712..9b72fe1fe424c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.index.SoftDeletesDirectoryReaderWrapper; import org.apache.lucene.index.StandardDirectoryReader; +import org.apache.lucene.index.VectorValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -61,9 +62,9 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; -import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.FIELDS_EXTENSION; -import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.INDEX_EXTENSION; -import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.META_EXTENSION; +import static org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsWriter.FIELDS_EXTENSION; +import static org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsWriter.INDEX_EXTENSION; +import static org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsWriter.META_EXTENSION; public class SourceOnlySnapshot { @@ -223,7 +224,7 @@ private SegmentCommitInfo syncSegment(SegmentCommitInfo segmentCommitInfo, LiveD for (FieldInfo fieldInfo : fieldInfos) { fieldInfoCopy.add(new FieldInfo(fieldInfo.name, fieldInfo.number, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, fieldInfo.attributes(), 0, 0, 0, - fieldInfo.isSoftDeletesField())); + 0, VectorValues.SimilarityFunction.NONE, fieldInfo.isSoftDeletesField())); } FieldInfos newFieldInfos = new FieldInfos(fieldInfoCopy.toArray(new FieldInfo[0])); codec.fieldInfosFormat().write(trackingDir, newSegmentInfo, segmentSuffix, newFieldInfos, IOContext.DEFAULT); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index 952ccd4670944..cc3fd558e81a7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -17,7 +17,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.FilterDirectory; -import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.store.NIOFSDirectory; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterState; @@ -145,7 +145,7 @@ public void snapshotShard(SnapshotShardContext context) { final List toClose = new ArrayList<>(3); try { SourceOnlySnapshot.LinkedFilesDirectory overlayDir = new SourceOnlySnapshot.LinkedFilesDirectory( - new SimpleFSDirectory(snapPath)); + new NIOFSDirectory(snapPath)); toClose.add(overlayDir); Store tempStore = new Store(store.shardId(), store.indexSettings(), overlayDir, new ShardLock(store.shardId()) { @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java index 667295a3eeadd..48752a6d8bab7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java @@ -321,11 +321,6 @@ public void checkIntegrity() throws IOException { public void close() throws IOException { reader.close(); } - - @Override - public long ramBytesUsed() { - return reader.ramBytesUsed(); - } } /** @@ -353,7 +348,7 @@ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { } @Override - public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { + public void stringField(FieldInfo fieldInfo, String value) throws IOException { visitor.stringField(fieldInfo, value); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java index 842020d1208ec..2b71a89213dcd 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java @@ -473,7 +473,7 @@ public void testSortedDocValues() throws Exception { SortedDocValues values = segmentReader.getSortedDocValues("fieldA"); assertNotNull(values); assertTrue(values.advanceExact(0)); - assertEquals(new BytesRef("testA"), values.binaryValue()); + assertEquals(new BytesRef("testA"), values.lookupOrd(values.ordValue())); assertNull(segmentReader.getSortedDocValues("fieldB")); TestUtil.checkReader(ir); diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 27dd960069c62..fb43fd86b65bf 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -13,8 +13,8 @@ import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; +import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; -import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java index d0966e20110f5..10a0cbce1134b 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java @@ -24,7 +24,6 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; -import org.apache.lucene.util.FutureArrays; import org.elasticsearch.action.bulk.BackoffPolicy; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkProcessor; @@ -376,7 +375,7 @@ private static Comparator keyComparator() { return (o1, o2) -> { int keySize1 = readInt(o1.bytes, o1.offset); int keySize2 = readInt(o2.bytes, o2.offset); - return FutureArrays.compareUnsigned(o1.bytes, o1.offset + Integer.BYTES, keySize1 + o1.offset + Integer.BYTES, + return Arrays.compareUnsigned(o1.bytes, o1.offset + Integer.BYTES, keySize1 + o1.offset + Integer.BYTES, o2.bytes, o2.offset + Integer.BYTES, keySize2 + o2.offset + Integer.BYTES); }; } diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScoreWeight.java b/x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScoreWeight.java index 42790a4f68b8e..6128920858ec7 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScoreWeight.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScoreWeight.java @@ -8,10 +8,8 @@ package org.apache.lucene.search; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; import java.io.IOException; -import java.util.Set; /** * A Weight that caps scores of the wrapped query to a maximum value @@ -27,11 +25,6 @@ protected CappedScoreWeight(Query query, Weight innerWeight, float maxScore) { this.innerWeight = innerWeight; } - @Override - public void extractTerms(Set terms) { - innerWeight.extractTerms(terms); - } - @Override public boolean isCacheable(LeafReaderContext ctx) { return innerWeight.isCacheable(ctx); diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectory.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectory.java index 5125291f846bc..b4e0af5096bab 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectory.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectory.java @@ -22,9 +22,6 @@ import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.StepListener; import org.elasticsearch.action.support.GroupedActionListener; -import org.elasticsearch.index.store.Store; -import org.elasticsearch.xpack.searchablesnapshots.cache.blob.BlobStoreCacheService; -import org.elasticsearch.xpack.searchablesnapshots.cache.blob.CachedBlob; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.common.CheckedRunnable; @@ -44,12 +41,8 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot; -import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheFile; -import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey; -import org.elasticsearch.xpack.searchablesnapshots.store.input.CachedBlobContainerIndexInput; -import org.elasticsearch.xpack.searchablesnapshots.store.input.FrozenIndexInput; +import org.elasticsearch.index.store.Store; import org.elasticsearch.indices.recovery.RecoveryState; -import org.elasticsearch.xpack.searchablesnapshots.recovery.SearchableSnapshotRecoveryState; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; @@ -59,12 +52,19 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots; import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants; +import org.elasticsearch.xpack.searchablesnapshots.cache.blob.BlobStoreCacheService; +import org.elasticsearch.xpack.searchablesnapshots.cache.blob.CachedBlob; import org.elasticsearch.xpack.searchablesnapshots.cache.common.ByteRange; +import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheFile; +import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey; import org.elasticsearch.xpack.searchablesnapshots.cache.full.CacheService; import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService; import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService.FrozenCacheFile; +import org.elasticsearch.xpack.searchablesnapshots.recovery.SearchableSnapshotRecoveryState; +import org.elasticsearch.xpack.searchablesnapshots.store.input.CachedBlobContainerIndexInput; import org.elasticsearch.xpack.searchablesnapshots.store.input.ChecksumBlobContainerIndexInput; import org.elasticsearch.xpack.searchablesnapshots.store.input.DirectBlobContainerIndexInput; +import org.elasticsearch.xpack.searchablesnapshots.store.input.FrozenIndexInput; import java.io.FileNotFoundException; import java.io.IOException; @@ -87,7 +87,6 @@ import java.util.function.LongSupplier; import java.util.function.Supplier; -import static org.apache.lucene.store.BufferedIndexInput.bufferSize; import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_BLOB_CACHE_METADATA_FILES_MAX_LENGTH_SETTING; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING; @@ -95,7 +94,6 @@ import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_CACHE_PREWARM_ENABLED_SETTING; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_INDEX_ID_SETTING; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_INDEX_NAME_SETTING; -import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.SNAPSHOT_PARTIAL_SETTING; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_REPOSITORY_NAME_SETTING; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_REPOSITORY_UUID_SETTING; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_SNAPSHOT_ID_SETTING; @@ -103,6 +101,7 @@ import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_UNCACHED_CHUNK_SIZE_SETTING; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.CACHE_PREWARMING_THREAD_POOL_NAME; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.SNAPSHOT_DIRECTORY_FACTORY_KEY; +import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.SNAPSHOT_PARTIAL_SETTING; /** * Implementation of {@link Directory} that exposes files from a snapshot as a Lucene directory. Because snapshot are immutable this @@ -437,8 +436,7 @@ public IndexInput openInput(final String name, final IOContext context) throws I fileInfo, context, inputStats, - getUncachedChunkSize(), - bufferSize(context) + getUncachedChunkSize() ); } } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInput.java index e8eb151bc84a9..d3646da15fde3 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInput.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInput.java @@ -73,10 +73,9 @@ public DirectBlobContainerIndexInput( FileInfo fileInfo, IOContext context, IndexInputStats stats, - long sequentialReadSize, - int bufferSize + long sequentialReadSize ) { - this(name, directory, fileInfo, context, stats, 0L, 0L, fileInfo.length(), sequentialReadSize, bufferSize); + this(name, directory, fileInfo, context, stats, 0L, 0L, fileInfo.length(), sequentialReadSize); stats.incrementOpenCount(); } @@ -89,15 +88,13 @@ private DirectBlobContainerIndexInput( long position, long offset, long length, - long sequentialReadSize, - int bufferSize + long sequentialReadSize ) { super(logger, name, directory, fileInfo, context, stats, offset, length, ByteRange.EMPTY, ByteRange.EMPTY); // TODO should use blob // cache this.position = position; assert sequentialReadSize >= 0; this.sequentialReadSize = sequentialReadSize; - setBufferSize(bufferSize); } @Override @@ -272,8 +269,7 @@ public DirectBlobContainerIndexInput clone() { length, // Clones might not be closed when they are no longer needed, but we must always close streamForSequentialReads. The simple // solution: do not optimize sequential reads on clones. - NO_SEQUENTIAL_READ_OPTIMIZATION, - getBufferSize() + NO_SEQUENTIAL_READ_OPTIMIZATION ); clone.isClone = true; return clone; @@ -293,8 +289,7 @@ public IndexInput slice(String sliceName, long offset, long length) throws IOExc length, // Slices might not be closed when they are no longer needed, but we must always close streamForSequentialReads. The simple // solution: do not optimize sequential reads on slices. - NO_SEQUENTIAL_READ_OPTIMIZATION, - getBufferSize() + NO_SEQUENTIAL_READ_OPTIMIZATION ); slice.isClone = true; slice.seek(0L); diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInputTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInputTests.java index 8a08ed4459beb..6627cb5d7b9da 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInputTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInputTests.java @@ -133,8 +133,7 @@ public int read(byte[] b, int off, int len) throws IOException { fileInfo, randomIOContext(), new IndexInputStats(1L, fileInfo.length(), fileInfo.length(), fileInfo.length(), () -> 0L), - minimumReadSize, - randomBoolean() ? BufferedIndexInput.BUFFER_SIZE : between(BufferedIndexInput.MIN_BUFFER_SIZE, BufferedIndexInput.BUFFER_SIZE) + minimumReadSize ); assertEquals(input.length, indexInput.length()); return indexInput; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractor.java index 820fac1d30c78..d4b05bc09fec3 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractor.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractor.java @@ -7,11 +7,11 @@ package org.elasticsearch.xpack.security.authz.accesscontrol; import org.apache.lucene.index.PrefixCodedTerms.TermIterator; +import org.apache.lucene.sandbox.search.DocValuesNumbersQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.DocValuesFieldExistsQuery; -import org.apache.lucene.search.DocValuesNumbersQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; @@ -23,7 +23,6 @@ import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.Weight; import org.apache.lucene.search.spans.SpanTermQuery; import java.util.HashSet; @@ -32,9 +31,11 @@ /** * Extracts fields from a query, or throws UnsupportedOperationException. *

- * Lucene queries have {@link Weight#extractTerms}, but this is really geared at things + * Lucene queries used to have Weight#extractTerms, but this was really geared at things * such as highlighting, not security. For example terms in a Boolean {@code MUST_NOT} clause * are not included, TermsQuery doesn't implement the method as it could be terribly slow, etc. + * + * TODO reimplement this using QueryVisitors */ class FieldExtractor { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractorTests.java index a39431e56a408..e42f49c3a25d6 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractorTests.java @@ -9,12 +9,12 @@ import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.index.Term; +import org.apache.lucene.sandbox.search.DocValuesNumbersQuery; import org.apache.lucene.search.AssertingQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.DocValuesFieldExistsQuery; -import org.apache.lucene.search.DocValuesNumbersQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; @@ -69,7 +69,10 @@ public void testTerm() { public void testSynonym() { Set fields = new HashSet<>(); - SynonymQuery query = new SynonymQuery(new Term("foo", "bar"), new Term("foo", "baz")); + SynonymQuery query = new SynonymQuery.Builder("foo") + .addTerm(new Term("foo", "bar")) + .addTerm(new Term("foo", "baz")) + .build(); FieldExtractor.extractFields(query, fields); assertEquals(asSet("foo"), fields); } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialUtils.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialUtils.java index 48d1690665419..080ca9f05a693 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialUtils.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialUtils.java @@ -45,8 +45,8 @@ public static Polygon createRegularGeoShapePolygon(Circle circle, int gons) { for(int i=0; i create.get(); Analyzer analyzer = create.get(); - analyzer.setVersion(Version.CURRENT.luceneVersion); this.current = new PreBuiltAnalyzerProvider(name, AnalyzerScope.INDICES, analyzer); } @@ -66,7 +64,6 @@ public AnalyzerProvider get(IndexSettings indexSettings, protected AnalyzerProvider create(Version version) { assert Version.CURRENT.equals(version) == false; Analyzer analyzer = create.apply(version); - analyzer.setVersion(version.luceneVersion); return new PreBuiltAnalyzerProvider(getName(), AnalyzerScope.INDICES, analyzer); } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/SimpleAnalyzerProvider.java b/server/src/main/java/org/elasticsearch/index/analysis/SimpleAnalyzerProvider.java index ff4d81c692227..080daf4e4c1c3 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/SimpleAnalyzerProvider.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/SimpleAnalyzerProvider.java @@ -20,7 +20,6 @@ public class SimpleAnalyzerProvider extends AbstractIndexAnalyzerProvider public static final String NAME = "regexp"; public static final int DEFAULT_FLAGS_VALUE = RegexpFlag.ALL.value(); - public static final int DEFAULT_MAX_DETERMINIZED_STATES = Operations.DEFAULT_MAX_DETERMINIZED_STATES; + public static final int DEFAULT_MAX_DETERMINIZED_STATES = Operations.DEFAULT_DETERMINIZE_WORK_LIMIT; public static final boolean DEFAULT_CASE_INSENSITIVITY = false; private static final ParseField FLAGS_VALUE_FIELD = new ParseField("flags_value"); diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java index 23f4f43003b75..19855fed66777 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java @@ -8,13 +8,13 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanContainingQuery; +import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanContainingQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -24,7 +24,7 @@ import static org.elasticsearch.index.query.SpanQueryBuilder.SpanQueryBuilderUtil.checkNoBoost; /** - * Builder for {@link org.apache.lucene.search.spans.SpanContainingQuery}. + * Builder for {@link org.apache.lucene.queries.spans.SpanContainingQuery}. */ public class SpanContainingQueryBuilder extends AbstractQueryBuilder implements SpanQueryBuilder { public static final String NAME = "span_containing"; diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java index d5d6477241dba..506dd80b639a6 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java @@ -8,13 +8,13 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanFirstQuery; +import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanFirstQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java index 5f92133edd5e7..c2a65cf5e68ca 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java @@ -8,19 +8,19 @@ package org.elasticsearch.index.query; import org.apache.lucene.queries.SpanMatchNoDocsQuery; +import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TopTermsRewrite; -import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; -import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.search.SpanBooleanQueryRewriteWithMaxClause; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.MappedFieldType; diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java index 4b0abbe4b2b2f..2880d563dbc9b 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java @@ -8,14 +8,14 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java index a1af3b254e306..7b31272261f98 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java @@ -8,13 +8,13 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanNotQuery; +import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanNotQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java index 988a2aba2b972..78695bf650df1 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java @@ -8,13 +8,13 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java index 2606f7b29bfae..1eceb5b017e3b 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java @@ -9,13 +9,13 @@ package org.elasticsearch.index.query; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; -import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.lucene.BytesRefs; +import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.MappedFieldType; diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java index 93ca8ac1c52f1..6b8526705657f 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java @@ -8,13 +8,13 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanWithinQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanWithinQuery; -import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -24,7 +24,7 @@ import static org.elasticsearch.index.query.SpanQueryBuilder.SpanQueryBuilderUtil.checkNoBoost; /** - * Builder for {@link org.apache.lucene.search.spans.SpanWithinQuery}. + * Builder for {@link org.apache.lucene.queries.spans.SpanWithinQuery}. */ public class SpanWithinQueryBuilder extends AbstractQueryBuilder implements SpanQueryBuilder { public static final String NAME = "span_within"; diff --git a/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java b/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java index b5f637b15a4c1..04322eedeb679 100644 --- a/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java +++ b/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java @@ -17,17 +17,17 @@ import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute; import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostAttribute; import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.util.QueryBuilder; import org.apache.lucene.util.graph.GraphTokenStreamFiniteStrings; import org.elasticsearch.ElasticsearchException; diff --git a/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java b/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java index 69fb5130dae64..e4d1d52bd9939 100644 --- a/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java +++ b/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java @@ -13,6 +13,9 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.queryparser.classic.Token; import org.apache.lucene.queryparser.classic.XQueryParser; @@ -28,9 +31,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.WildcardQuery; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.common.lucene.search.Queries; @@ -732,7 +732,7 @@ private Query getRegexpQuerySingle(String field, String termStr) throws ParseExc setAnalyzer(forceAnalyzer); return super.getRegexpQuery(field, termStr); } - return currentFieldType.regexpQuery(termStr, RegExp.ALL, 0, getMaxDeterminizedStates(), + return currentFieldType.regexpQuery(termStr, RegExp.ALL, 0, getDeterminizeWorkLimit(), getMultiTermRewriteMethod(), context); } catch (RuntimeException e) { if (lenient) { diff --git a/server/src/main/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzers.java b/server/src/main/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzers.java index 6277de09958c4..a55d8de400466 100644 --- a/server/src/main/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzers.java +++ b/server/src/main/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzers.java @@ -26,9 +26,7 @@ public enum PreBuiltAnalyzers { STANDARD(CachingStrategy.ELASTICSEARCH) { @Override protected Analyzer create(Version version) { - final Analyzer a = new StandardAnalyzer(CharArraySet.EMPTY_SET); - a.setVersion(version.luceneVersion); - return a; + return new StandardAnalyzer(CharArraySet.EMPTY_SET); } }, @@ -51,36 +49,28 @@ protected Analyzer create(Version version) { STOP { @Override protected Analyzer create(Version version) { - Analyzer a = new StopAnalyzer(EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); - a.setVersion(version.luceneVersion); - return a; + return new StopAnalyzer(EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); } }, WHITESPACE { @Override protected Analyzer create(Version version) { - Analyzer a = new WhitespaceAnalyzer(); - a.setVersion(version.luceneVersion); - return a; + return new WhitespaceAnalyzer(); } }, SIMPLE { @Override protected Analyzer create(Version version) { - Analyzer a = new SimpleAnalyzer(); - a.setVersion(version.luceneVersion); - return a; + return new SimpleAnalyzer(); } }, CLASSIC { @Override protected Analyzer create(Version version) { - Analyzer a = new ClassicAnalyzer(); - a.setVersion(version.luceneVersion); - return a; + return new ClassicAnalyzer(); } }; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java index 9d8aaa1a4506e..3af68f191c5cf 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java @@ -10,7 +10,6 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.PointValues; import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/IncludeExclude.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/IncludeExclude.java index 49d4e1b18b058..ce9b9df2c1023 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/IncludeExclude.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/IncludeExclude.java @@ -577,7 +577,7 @@ private Automaton toAutomaton() { a = Automata.makeAnyString(); } if (exclude != null) { - a = Operations.minus(a, exclude.toAutomaton(), Operations.DEFAULT_MAX_DETERMINIZED_STATES); + a = Operations.minus(a, exclude.toAutomaton(), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); } return a; } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java index dd8bd9732d118..cf454d7b1b73a 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java @@ -11,7 +11,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.CollectionTerminatedException; import org.apache.lucene.search.Collector; -import org.apache.lucene.search.ConjunctionDISI; +import org.apache.lucene.search.ConjunctionUtils; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.ScoreMode; @@ -160,7 +160,7 @@ public static void intersect(Weight weight, Weight innerHitQueryWeight, Collecto try { Bits acceptDocs = ctx.reader().getLiveDocs(); - DocIdSetIterator iterator = ConjunctionDISI.intersectIterators(Arrays.asList(innerHitQueryScorer.iterator(), + DocIdSetIterator iterator = ConjunctionUtils.intersectIterators(Arrays.asList(innerHitQueryScorer.iterator(), scorer.iterator())); for (int docId = iterator.nextDoc(); docId < DocIdSetIterator.NO_MORE_DOCS; docId = iterator.nextDoc()) { if (acceptDocs == null || acceptDocs.get(docId)) { diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index d03cafd7be991..be995786500b4 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -16,7 +16,7 @@ import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.CollectionTerminatedException; import org.apache.lucene.search.Collector; -import org.apache.lucene.search.ConjunctionDISI; +import org.apache.lucene.search.ConjunctionUtils; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; @@ -257,7 +257,7 @@ static void intersectScorerAndBitSet(Scorer scorer, BitSet acceptDocs, collector.setScorer(scorer); // ConjunctionDISI uses the DocIdSetIterator#cost() to order the iterators, so if roleBits has the lowest cardinality it should // be used first: - DocIdSetIterator iterator = ConjunctionDISI.intersectIterators(Arrays.asList(new BitSetIterator(acceptDocs, + DocIdSetIterator iterator = ConjunctionUtils.intersectIterators(Arrays.asList(new BitSetIterator(acceptDocs, acceptDocs.approximateCardinality()), scorer.iterator())); int seen = 0; checkCancelled.run(); diff --git a/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java b/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java index d0a70ea11ebb9..e30b2bbfc34e7 100644 --- a/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java +++ b/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java @@ -16,6 +16,7 @@ import org.apache.lucene.index.PointValues; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; +import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.Collector; @@ -39,14 +40,13 @@ import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.grouping.CollapseTopFieldDocs; import org.apache.lucene.search.grouping.CollapsingTopDocsCollector; -import org.apache.lucene.search.spans.SpanQuery; import org.elasticsearch.action.search.MaxScoreCollector; -import org.elasticsearch.core.Nullable; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore; import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery; import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery; import org.elasticsearch.common.util.CachedSupplier; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.search.ESToParentBlockJoinQuery; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.collapse.CollapseContext; diff --git a/server/src/main/java/org/elasticsearch/search/suggest/completion/FuzzyOptions.java b/server/src/main/java/org/elasticsearch/search/suggest/completion/FuzzyOptions.java index ee962deb262dc..9db5efaa449fb 100644 --- a/server/src/main/java/org/elasticsearch/search/suggest/completion/FuzzyOptions.java +++ b/server/src/main/java/org/elasticsearch/search/suggest/completion/FuzzyOptions.java @@ -197,7 +197,7 @@ public static class Builder { private int fuzzyMinLength = FuzzyCompletionQuery.DEFAULT_MIN_FUZZY_LENGTH; private int fuzzyPrefixLength = FuzzyCompletionQuery.DEFAULT_NON_FUZZY_PREFIX; private boolean unicodeAware = FuzzyCompletionQuery.DEFAULT_UNICODE_AWARE; - private int maxDeterminizedStates = Operations.DEFAULT_MAX_DETERMINIZED_STATES; + private int maxDeterminizedStates = Operations.DEFAULT_DETERMINIZE_WORK_LIMIT; public Builder() { } diff --git a/server/src/main/java/org/elasticsearch/search/suggest/completion/RegexOptions.java b/server/src/main/java/org/elasticsearch/search/suggest/completion/RegexOptions.java index 0e0574477ab5d..ba3ae909200c2 100644 --- a/server/src/main/java/org/elasticsearch/search/suggest/completion/RegexOptions.java +++ b/server/src/main/java/org/elasticsearch/search/suggest/completion/RegexOptions.java @@ -131,7 +131,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws */ public static class Builder { private int flagsValue = RegExp.ALL; - private int maxDeterminizedStates = Operations.DEFAULT_MAX_DETERMINIZED_STATES; + private int maxDeterminizedStates = Operations.DEFAULT_DETERMINIZE_WORK_LIMIT; public Builder() { } diff --git a/server/src/test/java/org/apache/lucene/queries/SpanMatchNoDocsQueryTests.java b/server/src/test/java/org/apache/lucene/queries/SpanMatchNoDocsQueryTests.java index fed0f4408ea0d..08157dcc93fac 100644 --- a/server/src/test/java/org/apache/lucene/queries/SpanMatchNoDocsQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/SpanMatchNoDocsQueryTests.java @@ -16,14 +16,14 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryUtils; import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.store.Directory; import org.elasticsearch.test.ESTestCase; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AbstractNonTextScriptFieldTypeTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/AbstractNonTextScriptFieldTypeTestCase.java index 18d4b91135ac4..c5fc0d3d44a8c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/AbstractNonTextScriptFieldTypeTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/AbstractNonTextScriptFieldTypeTestCase.java @@ -28,7 +28,7 @@ public void testPrefixQueryIsError() { public void testRegexpQueryIsError() { assertQueryOnlyOnTextAndKeyword( "regexp", - () -> simpleMappedFieldType().regexpQuery("cat", 0, 0, Operations.DEFAULT_MAX_DETERMINIZED_STATES, null, mockContext()) + () -> simpleMappedFieldType().regexpQuery("cat", 0, 0, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, null, mockContext()) ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index 51f0450ea2edc..d44a506d06362 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -712,7 +712,7 @@ public void testFuzzyQueryType() throws Exception { CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper; Query prefixQuery = completionFieldMapper.fieldType().fuzzyQuery("co", Fuzziness.fromEdits(FuzzyCompletionQuery.DEFAULT_MAX_EDITS), FuzzyCompletionQuery.DEFAULT_NON_FUZZY_PREFIX, - FuzzyCompletionQuery.DEFAULT_MIN_FUZZY_LENGTH, Operations.DEFAULT_MAX_DETERMINIZED_STATES, + FuzzyCompletionQuery.DEFAULT_MIN_FUZZY_LENGTH, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, FuzzyCompletionQuery.DEFAULT_TRANSPOSITIONS, FuzzyCompletionQuery.DEFAULT_UNICODE_AWARE); assertThat(prefixQuery, instanceOf(FuzzyCompletionQuery.class)); } @@ -722,7 +722,7 @@ public void testRegexQueryType() throws Exception { Mapper fieldMapper = defaultMapper.mappers().getMapper("field"); CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper; Query prefixQuery = completionFieldMapper.fieldType() - .regexpQuery(new BytesRef("co"), RegExp.ALL, Operations.DEFAULT_MAX_DETERMINIZED_STATES); + .regexpQuery(new BytesRef("co"), RegExp.ALL, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); assertThat(prefixQuery, instanceOf(RegexCompletionQuery.class)); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java index 128f1ce1e87d9..03f5ad2cab1fe 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java @@ -262,7 +262,7 @@ public void testRegexpQuery() throws IOException { IndexSearcher searcher = newSearcher(reader); assertThat( searcher.count( - simpleMappedFieldType().regexpQuery("ca.+", 0, 0, Operations.DEFAULT_MAX_DETERMINIZED_STATES, null, mockContext()) + simpleMappedFieldType().regexpQuery("ca.+", 0, 0, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, null, mockContext()) ), equalTo(2) ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 1b41a5bea136c..4d220a0b09ef4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -26,6 +26,10 @@ import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.queries.spans.FieldMaskingSpanQuery; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; @@ -38,10 +42,6 @@ import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.spans.FieldMaskingSpanQuery; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Strings; import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery; diff --git a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java index 95853ff8271d9..7947ad35ef4b8 100644 --- a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java @@ -10,7 +10,6 @@ import org.apache.lucene.sandbox.search.CombinedFieldQuery; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.CombinedFieldQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; diff --git a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java index e0eaa4aa7b69f..995d87efc4711 100644 --- a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java @@ -17,7 +17,6 @@ import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; -import org.apache.lucene.search.CombinedFieldQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; diff --git a/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java index 22f09b171fb98..c4cd125a96653 100644 --- a/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java @@ -8,8 +8,8 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.FieldMaskingSpanQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.FieldMaskingSpanQuery; import org.elasticsearch.test.AbstractQueryTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java index 6fbfa88a61b35..5512341eca213 100644 --- a/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java @@ -12,6 +12,10 @@ import org.apache.lucene.analysis.CannedBinaryTokenStream; import org.apache.lucene.analysis.MockSynonymAnalyzer; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.FuzzyQuery; @@ -22,10 +26,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.common.ParsingException; diff --git a/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java index adfc12c801c38..7c79f0d6e39ba 100644 --- a/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java @@ -41,7 +41,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBooleanSubQuery; import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.either; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.hasItems; import static org.hamcrest.CoreMatchers.instanceOf; diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java index 7d8ba36436665..7b93e309cda14 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java @@ -34,9 +34,9 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.search.WildcardQuery; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanTermQuery; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.Automata; import org.apache.lucene.util.automaton.Automaton; diff --git a/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java index 2f0127de0933c..b9fc4becb6159 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java @@ -11,6 +11,10 @@ import org.apache.lucene.analysis.MockSynonymAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; @@ -22,10 +26,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.util.TestUtil; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanContainingQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanContainingQueryBuilderTests.java index 8a7286691a8ad..0981699a2b4a7 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanContainingQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanContainingQueryBuilderTests.java @@ -8,8 +8,8 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanContainingQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanContainingQuery; import org.elasticsearch.common.ParsingException; import org.elasticsearch.test.AbstractQueryTestCase; diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanFirstQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanFirstQueryBuilderTests.java index a7c9f71120385..4d01b52e109fd 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanFirstQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanFirstQueryBuilderTests.java @@ -8,8 +8,8 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanFirstQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanFirstQuery; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanGapQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanGapQueryBuilderTests.java index 6f345de6e96b0..14415785d36c7 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanGapQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanGapQueryBuilderTests.java @@ -8,11 +8,11 @@ package org.elasticsearch.index.query; +import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanBoostQuery; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.elasticsearch.test.AbstractQueryTestCase; import java.io.IOException; @@ -50,7 +50,7 @@ protected SpanNearQueryBuilder doCreateTestQueryBuilder() { protected void doAssertLuceneQuery(SpanNearQueryBuilder queryBuilder, Query query, SearchExecutionContext context) throws IOException { assertThat(query, either(instanceOf(SpanNearQuery.class)) .or(instanceOf(SpanTermQuery.class)) - .or(instanceOf(SpanBoostQuery.class)) + .or(instanceOf(BoostQuery.class)) .or(instanceOf(MatchAllQueryBuilder.class))); if (query instanceof SpanNearQuery) { SpanNearQuery spanNearQuery = (SpanNearQuery) query; @@ -63,7 +63,7 @@ protected void doAssertLuceneQuery(SpanNearQueryBuilder queryBuilder, Query quer if (spanQB instanceof SpanGapQueryBuilder) continue; assertThat(spanQuery, equalTo(spanQB.toQuery(context))); } - } else if (query instanceof SpanTermQuery || query instanceof SpanBoostQuery) { + } else if (query instanceof SpanTermQuery) { assertThat(queryBuilder.clauses().size(), equalTo(1)); assertThat(query, equalTo(queryBuilder.clauses().get(0).toQuery(context))); } diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java index ed987ec1cf816..b1b7b03aec428 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java @@ -15,6 +15,10 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.queries.SpanMatchNoDocsQuery; +import org.apache.lucene.queries.spans.FieldMaskingSpanQuery; +import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.IndexSearcher; @@ -23,10 +27,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopTermsRewrite; -import org.apache.lucene.search.spans.FieldMaskingSpanQuery; -import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.store.Directory; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; @@ -41,9 +41,9 @@ import static java.util.Collections.singleton; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.either; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.CoreMatchers.either; import static org.hamcrest.CoreMatchers.startsWith; public class SpanMultiTermQueryBuilderTests extends AbstractQueryTestCase { diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java index 8651cca2088a2..bafa38cbd1b18 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java @@ -9,11 +9,11 @@ package org.elasticsearch.index.query; import org.apache.lucene.queries.SpanMatchNoDocsQuery; +import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanBoostQuery; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.elasticsearch.common.ParsingException; import org.elasticsearch.test.AbstractQueryTestCase; @@ -41,7 +41,7 @@ protected SpanNearQueryBuilder doCreateTestQueryBuilder() { protected void doAssertLuceneQuery(SpanNearQueryBuilder queryBuilder, Query query, SearchExecutionContext context) throws IOException { assertThat(query, either(instanceOf(SpanNearQuery.class)) .or(instanceOf(SpanTermQuery.class)) - .or(instanceOf(SpanBoostQuery.class)) + .or(instanceOf(BoostQuery.class)) .or(instanceOf(SpanMatchNoDocsQuery.class)) .or(instanceOf(MatchAllQueryBuilder.class))); if (query instanceof SpanNearQuery) { @@ -53,7 +53,7 @@ protected void doAssertLuceneQuery(SpanNearQueryBuilder queryBuilder, Query quer for (SpanQuery spanQuery : spanNearQuery.getClauses()) { assertThat(spanQuery, equalTo(spanQueryBuilderIterator.next().toQuery(context))); } - } else if (query instanceof SpanTermQuery || query instanceof SpanBoostQuery) { + } else if (query instanceof SpanTermQuery) { assertThat(queryBuilder.clauses().size(), equalTo(1)); assertThat(query, equalTo(queryBuilder.clauses().get(0).toQuery(context))); } diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanNotQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanNotQueryBuilderTests.java index 508cd191918c5..398af982d36bd 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanNotQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanNotQueryBuilderTests.java @@ -8,8 +8,8 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanNotQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanNotQuery; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java index 8931656be7ba0..a7a486d5c833e 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java @@ -8,9 +8,9 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanQuery; import org.elasticsearch.common.ParsingException; import org.elasticsearch.test.AbstractQueryTestCase; diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java index 87074e5c74dc4..dfcf124378aa6 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java @@ -10,9 +10,9 @@ import com.fasterxml.jackson.core.io.JsonStringEncoder; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.index.mapper.MappedFieldType; diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanWithinQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanWithinQueryBuilderTests.java index ccc9fea91da87..7ae668fe7f19b 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanWithinQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanWithinQueryBuilderTests.java @@ -8,8 +8,8 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanWithinQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanWithinQuery; import org.elasticsearch.common.ParsingException; import org.elasticsearch.test.AbstractQueryTestCase; diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java index 264246c1d0336..6151ed3b910be 100644 --- a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java @@ -15,7 +15,6 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.hunspell.Dictionary; -import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.store.Directory; import org.apache.lucene.store.NIOFSDirectory; @@ -33,7 +32,6 @@ import org.elasticsearch.index.analysis.CustomAnalyzer; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.MyFilterTokenFilterFactory; -import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.analysis.PreConfiguredCharFilter; import org.elasticsearch.index.analysis.PreConfiguredTokenFilter; import org.elasticsearch.index.analysis.PreConfiguredTokenizer; @@ -68,7 +66,6 @@ import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; public class AnalysisModuleTests extends ESTestCase { private final Settings emptyNodeSettings = Settings.builder() @@ -120,35 +117,6 @@ public void testSimpleConfigurationYaml() throws IOException { testSimpleConfiguration(settings); } - public void testVersionedAnalyzers() throws Exception { - String yaml = "/org/elasticsearch/index/analysis/test1.yml"; - Version version = VersionUtils.randomVersion(random()); - Settings settings2 = Settings.builder() - .loadFromStream(yaml, getClass().getResourceAsStream(yaml), false) - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) - .put(IndexMetadata.SETTING_VERSION_CREATED, version) - .build(); - AnalysisRegistry newRegistry = getNewRegistry(settings2); - IndexAnalyzers indexAnalyzers = getIndexAnalyzers(newRegistry, settings2); - - // registry always has the current version - assertThat(newRegistry.getAnalyzer("default"), is(instanceOf(NamedAnalyzer.class))); - NamedAnalyzer defaultNamedAnalyzer = (NamedAnalyzer) newRegistry.getAnalyzer("default"); - assertThat(defaultNamedAnalyzer.analyzer(), is(instanceOf(StandardAnalyzer.class))); - assertEquals(Version.CURRENT.luceneVersion, defaultNamedAnalyzer.analyzer().getVersion()); - - // analysis service has the expected version - assertThat(indexAnalyzers.get("standard").analyzer(), is(instanceOf(StandardAnalyzer.class))); - assertEquals(version.luceneVersion, - indexAnalyzers.get("standard").analyzer().getVersion()); - assertEquals(version.luceneVersion, - indexAnalyzers.get("stop").analyzer().getVersion()); - - assertThat(indexAnalyzers.get("custom7").analyzer(), is(instanceOf(StandardAnalyzer.class))); - assertEquals(org.apache.lucene.util.Version.fromBits(3,6,0), - indexAnalyzers.get("custom7").analyzer().getVersion()); - } - private void testSimpleConfiguration(Settings settings) throws IOException { IndexAnalyzers indexAnalyzers = getIndexAnalyzers(settings); Analyzer analyzer = indexAnalyzers.get("custom1").analyzer(); diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java index c5aebb918137c..a7c5391562612 100644 --- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java @@ -27,6 +27,8 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.queries.MinDocQuery; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; @@ -54,8 +56,6 @@ import org.apache.lucene.search.Weight; import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.search.join.ScoreMode; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; diff --git a/server/src/test/java/org/elasticsearch/search/runtime/StringScriptFieldRegexpQueryTests.java b/server/src/test/java/org/elasticsearch/search/runtime/StringScriptFieldRegexpQueryTests.java index ba1367a769b8a..09d52cdcbac65 100644 --- a/server/src/test/java/org/elasticsearch/search/runtime/StringScriptFieldRegexpQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/runtime/StringScriptFieldRegexpQueryTests.java @@ -30,7 +30,7 @@ protected StringScriptFieldRegexpQuery createTestInstance() { randomAlphaOfLength(6), randomInt(RegExp.ALL), matchFlags, - Operations.DEFAULT_MAX_DETERMINIZED_STATES + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT ); } @@ -43,7 +43,7 @@ protected StringScriptFieldRegexpQuery copy(StringScriptFieldRegexpQuery orig) { orig.pattern(), orig.syntaxFlags(), orig.matchFlags(), - Operations.DEFAULT_MAX_DETERMINIZED_STATES + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT ); } @@ -80,7 +80,7 @@ protected StringScriptFieldRegexpQuery mutate(StringScriptFieldRegexpQuery orig) pattern, syntaxFlags, matchFlags, - Operations.DEFAULT_MAX_DETERMINIZED_STATES + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT ); } @@ -93,7 +93,7 @@ public void testMatches() { "a.+b", 0, 0, - Operations.DEFAULT_MAX_DETERMINIZED_STATES + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT ); assertTrue(query.matches(List.of("astuffb"))); assertFalse(query.matches(List.of("astuffB"))); @@ -110,7 +110,7 @@ public void testMatches() { "a.+b", 0, RegExp.ASCII_CASE_INSENSITIVE, - Operations.DEFAULT_MAX_DETERMINIZED_STATES + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT ); assertTrue(ciQuery.matches(List.of("astuffB"))); assertTrue(ciQuery.matches(List.of("Astuffb", "fffff"))); @@ -131,7 +131,7 @@ public void testVisit() { "a.+b", 0, 0, - Operations.DEFAULT_MAX_DETERMINIZED_STATES + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT ); ByteRunAutomaton automaton = visitForSingleAutomata(query); BytesRef term = new BytesRef("astuffb"); diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java index 99d9316767d32..e0695b71b163c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java @@ -9,19 +9,16 @@ package org.elasticsearch.test; import com.fasterxml.jackson.core.io.JsonStringEncoder; - import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.spans.SpanBoostQuery; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.core.Tuple; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.StreamInput; @@ -38,11 +35,12 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; -import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.Rewriteable; +import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.support.QueryParsers; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -493,13 +491,8 @@ private void assertLuceneQuery(QB queryBuilder, Query query, SearchExecutionCont } if (query != null) { if (queryBuilder.boost() != AbstractQueryBuilder.DEFAULT_BOOST) { - assertThat(query, either(instanceOf(BoostQuery.class)).or(instanceOf(SpanBoostQuery.class)) - .or(instanceOf(MatchNoDocsQuery.class))); - if (query instanceof SpanBoostQuery) { - SpanBoostQuery spanBoostQuery = (SpanBoostQuery) query; - assertThat(spanBoostQuery.getBoost(), equalTo(queryBuilder.boost())); - query = spanBoostQuery.getQuery(); - } else if (query instanceof BoostQuery) { + assertThat(query, either(instanceOf(BoostQuery.class)).or(instanceOf(MatchNoDocsQuery.class))); + if (query instanceof BoostQuery) { BoostQuery boostQuery = (BoostQuery) query; if (boostQuery.getQuery() instanceof MatchNoDocsQuery == false) { assertThat(boostQuery.getBoost(), equalTo(queryBuilder.boost())); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java index df94c1ac4cadf..7b9a5b8fd5f10 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java @@ -147,8 +147,8 @@ public static Automaton buildPermittedFieldsAutomaton(final String[] grantedFiel deniedFieldsAutomaton = Automatons.patterns(deniedFields); } - grantedFieldsAutomaton = MinimizationOperations.minimize(grantedFieldsAutomaton, Operations.DEFAULT_MAX_DETERMINIZED_STATES); - deniedFieldsAutomaton = MinimizationOperations.minimize(deniedFieldsAutomaton, Operations.DEFAULT_MAX_DETERMINIZED_STATES); + grantedFieldsAutomaton = MinimizationOperations.minimize(grantedFieldsAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); + deniedFieldsAutomaton = MinimizationOperations.minimize(deniedFieldsAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); if (subsetOf(deniedFieldsAutomaton, grantedFieldsAutomaton) == false) { throw new ElasticsearchSecurityException("Exceptions for field permissions must be a subset of the " + diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java index 80848fb567452..821453639b386 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java @@ -16,8 +16,8 @@ import org.elasticsearch.common.cache.CacheBuilder; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.core.TimeValue; import java.util.ArrayList; import java.util.Arrays; @@ -29,7 +29,7 @@ import java.util.function.Function; import java.util.function.Predicate; -import static org.apache.lucene.util.automaton.Operations.DEFAULT_MAX_DETERMINIZED_STATES; +import static org.apache.lucene.util.automaton.Operations.DEFAULT_DETERMINIZE_WORK_LIMIT; import static org.apache.lucene.util.automaton.Operations.concatenate; import static org.apache.lucene.util.automaton.Operations.intersection; import static org.apache.lucene.util.automaton.Operations.minus; @@ -39,7 +39,7 @@ public final class Automatons { static final Setting MAX_DETERMINIZED_STATES_SETTING = - Setting.intSetting("xpack.security.automata.max_determinized_states", 100000, DEFAULT_MAX_DETERMINIZED_STATES, + Setting.intSetting("xpack.security.automata.max_determinized_states", 100000, DEFAULT_DETERMINIZE_WORK_LIMIT, Setting.Property.NodeScope); static final Setting CACHE_ENABLED = diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java index 2b71a89213dcd..0ac4a98542539 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java @@ -645,7 +645,7 @@ public void testSourceFiltering() { // exclude on exact value include = new CharacterRunAutomaton(Operations.minus( Automata.makeAnyString(), Automatons.patterns("foo.bar"), - Operations.DEFAULT_MAX_DETERMINIZED_STATES)); + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT)); filtered = FieldSubsetReader.filter(map, include, 0); expected = new HashMap<>(); expected.put("bar", "baz"); @@ -656,7 +656,7 @@ public void testSourceFiltering() { // exclude on wildcard include = new CharacterRunAutomaton(Operations.minus( Automata.makeAnyString(), Automatons.patterns("foo.*"), - Operations.DEFAULT_MAX_DETERMINIZED_STATES)); + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT)); filtered = FieldSubsetReader.filter(map, include, 0); expected = Collections.singletonMap("bar", "baz"); @@ -696,7 +696,7 @@ public void testSourceFiltering() { // exclude on inner array include = new CharacterRunAutomaton(Operations.minus( Automata.makeAnyString(), Automatons.patterns("foo.baz"), - Operations.DEFAULT_MAX_DETERMINIZED_STATES)); + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT)); filtered = FieldSubsetReader.filter(map, include, 0); expected = new HashMap<>(); subArray = new ArrayList<>(); @@ -711,7 +711,7 @@ public void testSourceFiltering() { // exclude on inner array 2 include = new CharacterRunAutomaton(Operations.minus( Automata.makeAnyString(), Automatons.patterns("foo"), - Operations.DEFAULT_MAX_DETERMINIZED_STATES)); + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT)); filtered = FieldSubsetReader.filter(map, include, 0); expected = new HashMap<>(); subArray = new ArrayList<>(); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonsTests.java index 571cefc9c41f5..f642d36fe9613 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonsTests.java @@ -20,7 +20,7 @@ import java.util.List; import java.util.Locale; -import static org.apache.lucene.util.automaton.Operations.DEFAULT_MAX_DETERMINIZED_STATES; +import static org.apache.lucene.util.automaton.Operations.DEFAULT_DETERMINIZE_WORK_LIMIT; import static org.elasticsearch.xpack.core.security.support.Automatons.pattern; import static org.elasticsearch.xpack.core.security.support.Automatons.patterns; import static org.elasticsearch.xpack.core.security.support.Automatons.predicate; @@ -112,12 +112,12 @@ public void testPatternComplexity() { } private void assertMatch(Automaton automaton, String text) { - CharacterRunAutomaton runAutomaton = new CharacterRunAutomaton(automaton, DEFAULT_MAX_DETERMINIZED_STATES); + CharacterRunAutomaton runAutomaton = new CharacterRunAutomaton(automaton, DEFAULT_DETERMINIZE_WORK_LIMIT); assertTrue(runAutomaton.run(text)); } private void assertMismatch(Automaton automaton, String text) { - CharacterRunAutomaton runAutomaton = new CharacterRunAutomaton(automaton, DEFAULT_MAX_DETERMINIZED_STATES); + CharacterRunAutomaton runAutomaton = new CharacterRunAutomaton(automaton, DEFAULT_DETERMINIZE_WORK_LIMIT); assertFalse(runAutomaton.run(text)); } @@ -158,7 +158,7 @@ public void testSettingMaxDeterminizedStates() { names.add(randomAlphaOfLength(48)); } TooComplexToDeterminizeException e = expectThrows(TooComplexToDeterminizeException.class, () -> Automatons.patterns(names)); - assertThat(e.getMaxDeterminizedStates(), equalTo(10000)); + assertThat(e.getDeterminizeWorkLimit(), equalTo(10000)); } finally { Automatons.updateConfiguration(Settings.EMPTY); assertEquals(100000, Automatons.getMaxDeterminizedStates()); diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java index 7804fe416a80f..28fe8fa43602e 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java @@ -52,7 +52,7 @@ public char escape() { @Override Automaton createAutomaton() { Automaton automaton = WildcardQuery.toAutomaton(new Term(null, wildcard)); - return MinimizationOperations.minimize(automaton, Operations.DEFAULT_MAX_DETERMINIZED_STATES); + return MinimizationOperations.minimize(automaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); } @Override diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectory.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectory.java index ba84671d4f96c..17da03089c8f5 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectory.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectory.java @@ -430,14 +430,7 @@ public IndexInput openInput(final String name, final IOContext context) throws I ); } } else { - return new DirectBlobContainerIndexInput( - name, - this, - fileInfo, - context, - inputStats, - getUncachedChunkSize() - ); + return new DirectBlobContainerIndexInput(name, this, fileInfo, context, inputStats, getUncachedChunkSize()); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractor.java index d4b05bc09fec3..309ca6da30da0 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractor.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractor.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.security.authz.accesscontrol; import org.apache.lucene.index.PrefixCodedTerms.TermIterator; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.sandbox.search.DocValuesNumbersQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; @@ -23,7 +24,6 @@ import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import java.util.HashSet; import java.util.Set; diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.9.0-snapshot-ddc238e5df8.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.9.0-snapshot-ddc238e5df8.jar.sha1 deleted file mode 100644 index aa05b1ccdaf3d..0000000000000 --- a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.9.0-snapshot-ddc238e5df8.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8f9509ff821ab7eeaf90fb77cf4437b81afcdd4b \ No newline at end of file diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-40f66a450ab.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-40f66a450ab.jar.sha1 new file mode 100644 index 0000000000000..a1f1b4f2d200c --- /dev/null +++ b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-40f66a450ab.jar.sha1 @@ -0,0 +1 @@ +4dafc4b2928634f5b0c5d783c91070ddc91154d0 \ No newline at end of file diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java index fd7f2d2726748..84c2f842ff5a0 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java @@ -411,7 +411,7 @@ static CharacterRunAutomaton createAutomaton(List whiteListedHosts) { } Automaton whiteListAutomaton = Regex.simpleMatchToAutomaton(whiteListedHosts.toArray(Strings.EMPTY_ARRAY)); - whiteListAutomaton = MinimizationOperations.minimize(whiteListAutomaton, Operations.DEFAULT_MAX_DETERMINIZED_STATES); + whiteListAutomaton = MinimizationOperations.minimize(whiteListAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); return new CharacterRunAutomaton(whiteListAutomaton); } } From eef74fc8b6dba2751c0a04c5c6d743853a5bb0b1 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 13 Jul 2021 16:38:51 +0200 Subject: [PATCH 03/67] More compile fixes. --- .../lucene/search/XCombinedFieldQuery.java | 5 --- .../diskusage/IndexDiskUsageAnalyzer.java | 20 ++++++---- .../index/engine/TranslogDirectoryReader.java | 21 +++++++--- .../IndexDiskUsageAnalyzerTests.java | 38 +++++++------------ .../FieldMaskingSpanQueryBuilderTests.java | 2 +- 5 files changed, 43 insertions(+), 43 deletions(-) diff --git a/server/src/main/java/org/apache/lucene/search/XCombinedFieldQuery.java b/server/src/main/java/org/apache/lucene/search/XCombinedFieldQuery.java index dee442afbe963..8d133f980f2ed 100644 --- a/server/src/main/java/org/apache/lucene/search/XCombinedFieldQuery.java +++ b/server/src/main/java/org/apache/lucene/search/XCombinedFieldQuery.java @@ -307,11 +307,6 @@ private CollectionStatistics mergeCollectionStatistics(IndexSearcher searcher) "pseudo_field", maxDoc, docCount, sumTotalTermFreq, sumDocFreq); } - @Override - public void extractTerms(Set termSet) { - termSet.addAll(Arrays.asList(fieldTerms)); - } - @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { Weight weight = diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java index 972459d97a831..2f61aa20eba2a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java @@ -9,14 +9,15 @@ package org.elasticsearch.action.admin.indices.diskusage; import org.apache.logging.log4j.Logger; +import org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat; +import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.codecs.FieldsProducer; import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PointsReader; import org.apache.lucene.codecs.StoredFieldsReader; import org.apache.lucene.codecs.TermVectorsReader; -import org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat; -import org.apache.lucene.codecs.lucene84.Lucene84PostingsFormat; +import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValuesType; @@ -42,7 +43,6 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.FutureArrays; import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.lucene.FilterIndexCommit; @@ -53,6 +53,8 @@ import org.elasticsearch.index.store.LuceneFilesExtensions; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -169,8 +171,8 @@ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { } @Override - public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { - trackField(fieldInfo, Integer.BYTES + value.length); + public void stringField(FieldInfo fieldInfo, String value) throws IOException { + trackField(fieldInfo, Integer.BYTES + value.getBytes(StandardCharsets.UTF_8).length); } @Override @@ -298,6 +300,10 @@ private BlockTermState getBlockTermState(TermsEnum termsEnum, BytesRef term) thr final Lucene50PostingsFormat.IntBlockTermState blockTermState = (Lucene50PostingsFormat.IntBlockTermState) termState; return new BlockTermState(blockTermState.docStartFP, blockTermState.posStartFP, blockTermState.payStartFP); } + if (termState instanceof Lucene90PostingsFormat.IntBlockTermState) { + final Lucene90PostingsFormat.IntBlockTermState blockTermState = (Lucene90PostingsFormat.IntBlockTermState) termState; + return new BlockTermState(blockTermState.docStartFP, blockTermState.posStartFP, blockTermState.payStartFP); + } } return null; } @@ -425,8 +431,8 @@ public void visit(int docID, byte[] packedValue) throws IOException { public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { for (int dim = 0; dim < numDims; dim++) { int offset = dim * bytesPerDim; - if (FutureArrays.compareUnsigned(minPackedValue, offset, offset + bytesPerDim, point, offset, offset + bytesPerDim) > 0 || - FutureArrays.compareUnsigned(maxPackedValue, offset, offset + bytesPerDim, point, offset, offset + bytesPerDim) < 0) { + if (Arrays.compareUnsigned(minPackedValue, offset, offset + bytesPerDim, point, offset, offset + bytesPerDim) > 0 || + Arrays.compareUnsigned(maxPackedValue, offset, offset + bytesPerDim, point, offset, offset + bytesPerDim) < 0) { return PointValues.Relation.CELL_OUTSIDE_QUERY; } } diff --git a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java index 48407a02807e4..d0a51712981ac 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java +++ b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java @@ -32,7 +32,9 @@ import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.VectorValues; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; @@ -56,7 +58,6 @@ import org.elasticsearch.index.translog.Translog; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.Map; import java.util.Set; @@ -136,13 +137,13 @@ private static class TranslogLeafReader extends LeafReader { private static final FieldInfo FAKE_SOURCE_FIELD = new FieldInfo(SourceFieldMapper.NAME, 1, false, false, false, IndexOptions.NONE, - DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, false); + DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, 0, VectorValues.SimilarityFunction.NONE, false); private static final FieldInfo FAKE_ROUTING_FIELD = new FieldInfo(RoutingFieldMapper.NAME, 2, false, false, false, IndexOptions.NONE, - DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, false); + DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, 0, VectorValues.SimilarityFunction.NONE, false); private static final FieldInfo FAKE_ID_FIELD = new FieldInfo(IdFieldMapper.NAME, 3, false, false, false, IndexOptions.DOCS, - DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, false); + DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, 0, VectorValues.SimilarityFunction.NONE, false); private static Set TRANSLOG_FIELD_NAMES = Sets.newHashSet(SourceFieldMapper.NAME, RoutingFieldMapper.NAME, IdFieldMapper.NAME); @@ -348,7 +349,7 @@ private void readStoredFieldsDirectly(StoredFieldVisitor visitor) throws IOExcep } } if (operation.routing() != null && visitor.needsField(FAKE_ROUTING_FIELD) == StoredFieldVisitor.Status.YES) { - visitor.stringField(FAKE_ROUTING_FIELD, operation.routing().getBytes(StandardCharsets.UTF_8)); + visitor.stringField(FAKE_ROUTING_FIELD, operation.routing()); } if (visitor.needsField(FAKE_ID_FIELD) == StoredFieldVisitor.Status.YES) { final byte[] id = new byte[uid.length]; @@ -361,6 +362,16 @@ private void readStoredFieldsDirectly(StoredFieldVisitor visitor) throws IOExcep protected synchronized void doClose() throws IOException { IOUtils.close(delegate.get(), directory); } + + @Override + public VectorValues getVectorValues(String field) throws IOException { + return getDelegate().getVectorValues(field); + } + + @Override + public TopDocs searchNearestVectors(String field, float[] target, int k, int fanout) throws IOException { + return getDelegate().searchNearestVectors(field, target, k, fanout); + } } private static class FakeTerms extends Terms { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java index 8a9f2534f7002..35fcef691b33c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java @@ -8,11 +8,11 @@ package org.elasticsearch.action.admin.indices.diskusage; +import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; +import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat; +import org.apache.lucene.codecs.lucene90.Lucene90Codec; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat; -import org.apache.lucene.codecs.lucene84.Lucene84PostingsFormat; -import org.apache.lucene.codecs.lucene87.Lucene87Codec; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.apache.lucene.document.BinaryDocValuesField; @@ -153,7 +153,7 @@ public void testCompletionField() throws Exception { IndexWriterConfig config = new IndexWriterConfig() .setCommitOnClose(true) .setUseCompoundFile(false) - .setCodec(new Lucene87Codec(Lucene87Codec.Mode.BEST_SPEED) { + .setCodec(new Lucene90Codec(Lucene90Codec.Mode.BEST_SPEED) { @Override public PostingsFormat getPostingsFormatForField(String field) { if (field.startsWith("suggest_")) { @@ -214,38 +214,26 @@ public void testMixedFields() throws Exception { enum CodecMode { BEST_SPEED { @Override - Lucene87Codec.Mode mode() { - return Lucene87Codec.Mode.BEST_SPEED; - } - - @Override - DocValuesFormat dvFormat() { - return new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_SPEED); + Lucene90Codec.Mode mode() { + return Lucene90Codec.Mode.BEST_SPEED; } }, BEST_COMPRESSION { @Override - Lucene87Codec.Mode mode() { - return Lucene87Codec.Mode.BEST_COMPRESSION; - } - - @Override - DocValuesFormat dvFormat() { - return new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION); + Lucene90Codec.Mode mode() { + return Lucene90Codec.Mode.BEST_COMPRESSION; } }; - abstract Lucene87Codec.Mode mode(); - - abstract DocValuesFormat dvFormat(); + abstract Lucene90Codec.Mode mode(); } static void indexRandomly(Directory directory, CodecMode codecMode, int numDocs, Consumer addFields) throws IOException { IndexWriterConfig config = new IndexWriterConfig() .setCommitOnClose(true) .setUseCompoundFile(randomBoolean()) - .setCodec(new Lucene87Codec(codecMode.mode())); + .setCodec(new Lucene90Codec(codecMode.mode())); try (IndexWriter writer = new IndexWriter(directory, config)) { for (int i = 0; i < numDocs; i++) { final Document doc = new Document(); @@ -406,15 +394,15 @@ static void rewriteIndexWithPerFieldCodec(Directory source, CodecMode mode, Dire IndexWriterConfig config = new IndexWriterConfig() .setSoftDeletesField(Lucene.SOFT_DELETES_FIELD) .setUseCompoundFile(randomBoolean()) - .setCodec(new Lucene87Codec(mode.mode()) { + .setCodec(new Lucene90Codec(mode.mode()) { @Override public PostingsFormat getPostingsFormatForField(String field) { - return new Lucene84PostingsFormat(); + return new Lucene90PostingsFormat(); } @Override public DocValuesFormat getDocValuesFormatForField(String field) { - return mode.dvFormat(); + return new Lucene90DocValuesFormat(); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java index 36ef19ccc7baf..64621efe28636 100644 --- a/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java @@ -10,9 +10,9 @@ import org.apache.lucene.index.Term; import org.apache.lucene.queries.spans.FieldMaskingSpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanTermQuery; import org.elasticsearch.common.ParsingException; import org.elasticsearch.test.AbstractQueryTestCase; From 3e8ab5c65d5d3a7709fdb5974a68442428c6beb2 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 13 Jul 2021 16:42:35 +0200 Subject: [PATCH 04/67] Undo unnecessary change. --- .../elasticsearch/common/settings/KeyStoreWrapperTests.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java index 28627b0e4c686..02533be7fdbcc 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java @@ -291,8 +291,10 @@ public void testFailWhenSecretStreamNotConsumed() throws Exception { public void testFailWhenEncryptedBytesStreamIsNotConsumed() throws Exception { assumeFalse("Cannot open unprotected keystore on FIPS JVM", inFipsJvm()); Path configDir = env.configFile(); - NIOFSDirectory directory = new NIOFSDirectory(configDir); - try (IndexOutput indexOutput = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)) { + try ( + Directory directory = newFSDirectory(configDir); + IndexOutput indexOutput = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT) + ) { CodecUtil.writeHeader(indexOutput, "elasticsearch.keystore", 3); indexOutput.writeByte((byte) 0); // No password SecureRandom random = Randomness.createSecure(); From 44c6f490036a4f1f725a7c2d2fe268b67f9bcd76 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Fri, 16 Jul 2021 10:19:33 -0400 Subject: [PATCH 05/67] fix compile --- server/src/main/java/org/elasticsearch/Version.java | 5 +++-- .../search/internal/FieldUsageTrackingDirectoryReader.java | 5 ----- .../FilterFieldNamesProvidingStoredFieldsVisitor.java | 2 +- .../search/internal/FilterStoredFieldVisitor.java | 2 +- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 03bdce3c8e1fd..53379d26176bd 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -85,8 +85,9 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_7_13_2 = new Version(7130299, org.apache.lucene.util.Version.LUCENE_8_8_2); public static final Version V_7_13_3 = new Version(7130399, org.apache.lucene.util.Version.LUCENE_8_8_2); public static final Version V_7_13_4 = new Version(7130499, org.apache.lucene.util.Version.LUCENE_8_8_2); - public static final Version V_7_14_0 = new Version(7140099, org.apache.lucene.util.Version.LUCENE_8_9_0); - public static final Version V_7_15_0 = new Version(7150099, org.apache.lucene.util.Version.LUCENE_8_9_0); + // NOCOMMIT: we need to build a new Lucene 9 snapshot and use Lucene 8.9.0 for 7.14 and 7.15 + public static final Version V_7_14_0 = new Version(7140099, org.apache.lucene.util.Version.LUCENE_8_8_2); + public static final Version V_7_15_0 = new Version(7150099, org.apache.lucene.util.Version.LUCENE_8_8_2); public static final Version V_8_0_0 = new Version(8000099, org.apache.lucene.util.Version.LUCENE_9_0_0); public static final Version CURRENT = V_8_0_0; diff --git a/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java b/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java index 34004f7a3ebc3..a51cf8aa77141 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java +++ b/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java @@ -213,11 +213,6 @@ public void checkIntegrity() throws IOException { public void close() throws IOException { reader.close(); } - - @Override - public long ramBytesUsed() { - return reader.ramBytesUsed(); - } } private class FieldUsageTrackingTerms extends FilterTerms { diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilterFieldNamesProvidingStoredFieldsVisitor.java b/server/src/main/java/org/elasticsearch/search/internal/FilterFieldNamesProvidingStoredFieldsVisitor.java index 329a746f075a8..6d3c7395a62a0 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/FilterFieldNamesProvidingStoredFieldsVisitor.java +++ b/server/src/main/java/org/elasticsearch/search/internal/FilterFieldNamesProvidingStoredFieldsVisitor.java @@ -28,7 +28,7 @@ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { } @Override - public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { + public void stringField(FieldInfo fieldInfo, String value) throws IOException { visitor.stringField(fieldInfo, value); } diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilterStoredFieldVisitor.java b/server/src/main/java/org/elasticsearch/search/internal/FilterStoredFieldVisitor.java index a695fae27346d..1051eee5d54f1 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/FilterStoredFieldVisitor.java +++ b/server/src/main/java/org/elasticsearch/search/internal/FilterStoredFieldVisitor.java @@ -26,7 +26,7 @@ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { } @Override - public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { + public void stringField(FieldInfo fieldInfo, String value) throws IOException { visitor.stringField(fieldInfo, value); } From 3e8da46046457a5434260d9a2edfb97bccf59c0f Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Fri, 30 Jul 2021 11:53:17 +0100 Subject: [PATCH 06/67] compilation + precommit --- .../common/settings/KeyStoreWrapperTests.java | 1 + .../lucene-expressions-8.9.0.jar.sha1 | 1 - ...ssions-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...ssions-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + ...is-icu-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...is-icu-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + .../lucene-analyzers-icu-8.9.0.jar.sha1 | 1 - ...romoji-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...romoji-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + .../lucene-analyzers-kuromoji-8.9.0.jar.sha1 | 1 - ...s-nori-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...s-nori-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + .../lucene-analyzers-nori-8.9.0.jar.sha1 | 1 - ...onetic-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...onetic-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + .../lucene-analyzers-phonetic-8.9.0.jar.sha1 | 1 - ...martcn-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...martcn-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + .../lucene-analyzers-smartcn-8.9.0.jar.sha1 | 1 - ...tempel-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...tempel-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + .../lucene-analyzers-stempel-8.9.0.jar.sha1 | 1 - ...ologik-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...ologik-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + ...lucene-analyzers-morfologik-8.9.0.jar.sha1 | 1 - ...common-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...common-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + .../lucene-analyzers-common-8.9.0.jar.sha1 | 1 - .../lucene-backward-codecs-8.9.0.jar.sha1 | 1 - ...codecs-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...codecs-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + server/licenses/lucene-core-8.9.0.jar.sha1 | 1 - ...e-core-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...e-core-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + .../licenses/lucene-grouping-8.9.0.jar.sha1 | 1 - ...ouping-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...ouping-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + .../lucene-highlighter-8.9.0.jar.sha1 | 1 - ...ighter-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...ighter-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + server/licenses/lucene-join-8.9.0.jar.sha1 | 1 - ...e-join-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...e-join-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + server/licenses/lucene-memory-8.9.0.jar.sha1 | 1 - ...memory-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...memory-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + server/licenses/lucene-misc-8.9.0.jar.sha1 | 1 - ...e-misc-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...e-misc-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + server/licenses/lucene-queries-8.9.0.jar.sha1 | 1 - ...ueries-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...ueries-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + .../lucene-queryparser-8.9.0.jar.sha1 | 1 - ...parser-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...parser-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + server/licenses/lucene-sandbox-8.9.0.jar.sha1 | 1 - ...andbox-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...andbox-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + .../lucene-spatial-extras-8.9.0.jar.sha1 | 1 - ...extras-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...extras-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + .../licenses/lucene-spatial3d-8.9.0.jar.sha1 | 1 - ...tial3d-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...tial3d-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + server/licenses/lucene-suggest-8.9.0.jar.sha1 | 1 - ...uggest-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...uggest-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + .../lucene/search/XCombinedFieldQuery.java | 6 +--- .../search/XMultiNormsLeafSimScorer.java | 2 +- .../vectorhighlight/CustomFieldQuery.java | 2 +- .../diskusage/IndexDiskUsageAnalyzer.java | 20 ++++++++----- .../elasticsearch/common/lucene/Lucene.java | 11 +++---- .../common/lucene/MinimumScoreCollector.java | 2 +- .../index/engine/TranslogDirectoryReader.java | 28 ++++++++++++----- .../index/mapper/DocumentLeafReader.java | 9 +++--- .../query/CombinedFieldsQueryBuilder.java | 1 - .../query/FieldMaskingSpanQueryBuilder.java | 4 +-- .../aggregations/MultiBucketCollector.java | 2 +- .../FieldUsageTrackingDirectoryReader.java | 30 ++++++++++++------- ...ieldNamesProvidingStoredFieldsVisitor.java | 2 +- .../internal/FilterStoredFieldVisitor.java | 2 +- .../IndexDiskUsageAnalyzerTests.java | 6 ++-- .../CombinedFieldsQueryBuilderTests.java | 1 - .../CombinedFieldsQueryParsingTests.java | 1 - .../FieldMaskingSpanQueryBuilderTests.java | 4 +-- .../lookup/LeafStoredFieldsLookupTests.java | 4 +-- .../engine/ThrowingLeafReaderWrapper.java | 17 ++++++++--- .../frozen/RewriteCachingDirectoryReader.java | 6 ++-- .../sourceonly/SourceOnlySnapshot.java | 4 +-- .../accesscontrol/FieldSubsetReader.java | 22 ++++++++++---- .../accesscontrol/FieldExtractorTests.java | 2 +- .../authz/store/FileRolesStoreTests.java | 2 +- .../licenses/lucene-core-8.9.0.jar.sha1 | 1 - ...e-core-9.0.0-snapshot-40f66a450ab.jar.sha1 | 1 - ...e-core-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 + 95 files changed, 140 insertions(+), 120 deletions(-) delete mode 100644 modules/lang-expression/licenses/lucene-expressions-8.9.0.jar.sha1 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-56eb76dbaf7.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-56eb76dbaf7.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-8.9.0.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-56eb76dbaf7.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.9.0.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-56eb76dbaf7.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analyzers-nori-8.9.0.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-56eb76dbaf7.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.9.0.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-56eb76dbaf7.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.9.0.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-56eb76dbaf7.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.9.0.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-56eb76dbaf7.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-analysis-common-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 server/licenses/lucene-analysis-common-9.0.0-snapshot-56eb76dbaf7.jar.sha1 delete mode 100644 server/licenses/lucene-analyzers-common-8.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-8.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 server/licenses/lucene-backward-codecs-9.0.0-snapshot-56eb76dbaf7.jar.sha1 delete mode 100644 server/licenses/lucene-core-8.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-core-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 server/licenses/lucene-core-9.0.0-snapshot-56eb76dbaf7.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-8.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 server/licenses/lucene-grouping-9.0.0-snapshot-56eb76dbaf7.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-8.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 server/licenses/lucene-highlighter-9.0.0-snapshot-56eb76dbaf7.jar.sha1 delete mode 100644 server/licenses/lucene-join-8.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-join-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 server/licenses/lucene-join-9.0.0-snapshot-56eb76dbaf7.jar.sha1 delete mode 100644 server/licenses/lucene-memory-8.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-memory-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 server/licenses/lucene-memory-9.0.0-snapshot-56eb76dbaf7.jar.sha1 delete mode 100644 server/licenses/lucene-misc-8.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-misc-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 server/licenses/lucene-misc-9.0.0-snapshot-56eb76dbaf7.jar.sha1 delete mode 100644 server/licenses/lucene-queries-8.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-queries-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 server/licenses/lucene-queries-9.0.0-snapshot-56eb76dbaf7.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-8.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 server/licenses/lucene-queryparser-9.0.0-snapshot-56eb76dbaf7.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-8.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 server/licenses/lucene-sandbox-9.0.0-snapshot-56eb76dbaf7.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-8.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 server/licenses/lucene-spatial-extras-9.0.0-snapshot-56eb76dbaf7.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-8.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 server/licenses/lucene-spatial3d-9.0.0-snapshot-56eb76dbaf7.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-8.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 server/licenses/lucene-suggest-9.0.0-snapshot-56eb76dbaf7.jar.sha1 delete mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-8.9.0.jar.sha1 delete mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-40f66a450ab.jar.sha1 create mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-56eb76dbaf7.jar.sha1 diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java index 3d7c61f73e753..02533be7fdbcc 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java @@ -40,6 +40,7 @@ import java.util.List; import java.util.Locale; import java.util.Set; + import javax.crypto.Cipher; import javax.crypto.CipherOutputStream; import javax.crypto.SecretKey; diff --git a/modules/lang-expression/licenses/lucene-expressions-8.9.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.9.0.jar.sha1 deleted file mode 100644 index ae69d7ff51221..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c52e0f197d8f37fec8e679660a4814b8928e4db2 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-40f66a450ab.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index 60c606033b81e..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d5facf4bcd0f52faf68c6eb0f8d92ebe6879711f \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..b6dfa64f8d20f --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +31b1f2a168e8ce93504682d67f37b6ef23f727a0 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-40f66a450ab.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index 5980cfcfa6bb8..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7066057f1502ea59e9031ac18fb24f820b7f121d \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..7bb3589ba53dd --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +68ea6ee2a48ada4de743cfbdd2a2bdf0c3807946 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.9.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.9.0.jar.sha1 deleted file mode 100644 index 8862b76301215..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -390ebbb1cdb3ab1e8e5286ae4d77d336fd3895ba \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-40f66a450ab.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index 826e70d1fdc18..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6fcd77fdb4bfa4d27b2fe2d5e4c4bd34a88a772f \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..0f20e9851a208 --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +82a7ed2c9845da0c575e538c75d63404c61efb14 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.9.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.9.0.jar.sha1 deleted file mode 100644 index 1b60c193938a9..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d7553997ad55671bb8c4c023f607f71463da4d0f \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-40f66a450ab.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index 3d16d9c07cb93..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -507cae6ed1027e1dc5cedf96c2505e4d01583b89 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..a9f1750f2029c --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +d23fea6880e75d8a8f44126cce618d2ceb49c75b \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.9.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.9.0.jar.sha1 deleted file mode 100644 index b9ea18c6de216..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -99ae12d053d27e2b8d78d084e1bf1c893b896554 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-40f66a450ab.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index 5c53eb4eacac8..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -31ab2fe56dc20bbad557a70686c4136db73625fb \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..ed0d47eaa12e1 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +29587ba58012b1796ea46fff081a9d548cafef76 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.9.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.9.0.jar.sha1 deleted file mode 100644 index 19d4411ac82f5..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -144cbfb8c5a66ae9ac7abddb8855974599d1ea67 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-40f66a450ab.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index 8928e2bf976e8..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -421e6f71a34f670dedb243fdfe55788e5508362f \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..427f6663e34fb --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +2675530dfcf22dd067960cba65b8a474364a7566 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.9.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.9.0.jar.sha1 deleted file mode 100644 index 81834aaa9b9e6..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -35f24955ae3d079a262a3aa7ac2ca2449c6d10ee \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-40f66a450ab.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index c694667f9aeaf..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -11fed62405b7e02508c9a54100e22a75a60a6f75 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..8a5f8ee423906 --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +99f98cd4198f58079f21e3a6ee54105b2bae524c \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.9.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.9.0.jar.sha1 deleted file mode 100644 index ff6375c1634ab..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d2a6fb6327e54c4eccfaa0f8e74953cd69a31e07 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-40f66a450ab.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index d100f2121b7a6..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -227733063d2c26b13e6bab906fce9a59c350223a \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..1a7d6af9439ca --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +e0113034825d158be0e52aa143ab5183c82a1259 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.9.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.9.0.jar.sha1 deleted file mode 100644 index f901aa40fa5a2..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -37fd6e23bbb1c146c729bb32b6c043ed0642a73e \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.0.0-snapshot-40f66a450ab.jar.sha1 b/server/licenses/lucene-analysis-common-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index f1d3136d69922..0000000000000 --- a/server/licenses/lucene-analysis-common-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -79880a5214c64820bde9c006e962de19e92b78de \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-analysis-common-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..c2d1b8fff7d96 --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +e72a4ea400998b4e62d1b7529fec190c111c1312 \ No newline at end of file diff --git a/server/licenses/lucene-analyzers-common-8.9.0.jar.sha1 b/server/licenses/lucene-analyzers-common-8.9.0.jar.sha1 deleted file mode 100644 index 8cd5ba872a31d..0000000000000 --- a/server/licenses/lucene-analyzers-common-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -18f3bbff2b7672ea0b9cc18c8110ef69c763ae6b \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-8.9.0.jar.sha1 b/server/licenses/lucene-backward-codecs-8.9.0.jar.sha1 deleted file mode 100644 index 932c95dc8cfcb..0000000000000 --- a/server/licenses/lucene-backward-codecs-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fec88b5e71c699ceddc3ae0369481697ac9a5c96 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.0.0-snapshot-40f66a450ab.jar.sha1 b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index e32c872333dac..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e852f2c488a3a07fa8a36d437393655f1037aa9f \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..35e8a6d4fa34c --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +eac640612d190818886dd2950aafd579c3a7a867 \ No newline at end of file diff --git a/server/licenses/lucene-core-8.9.0.jar.sha1 b/server/licenses/lucene-core-8.9.0.jar.sha1 deleted file mode 100644 index 859cdb2013408..0000000000000 --- a/server/licenses/lucene-core-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5c3f72357089f7f0c1ef44bbe7b4c67b6149a5af \ No newline at end of file diff --git a/server/licenses/lucene-core-9.0.0-snapshot-40f66a450ab.jar.sha1 b/server/licenses/lucene-core-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index a1f1b4f2d200c..0000000000000 --- a/server/licenses/lucene-core-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4dafc4b2928634f5b0c5d783c91070ddc91154d0 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-core-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..b3d858bef510c --- /dev/null +++ b/server/licenses/lucene-core-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +4ee46802b59e1d0ff15882ef735c500d5bc1cb6b \ No newline at end of file diff --git a/server/licenses/lucene-grouping-8.9.0.jar.sha1 b/server/licenses/lucene-grouping-8.9.0.jar.sha1 deleted file mode 100644 index e28e73fb699bf..0000000000000 --- a/server/licenses/lucene-grouping-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9440fdd430b1c2dadbf3bc72656848d61e6f747f \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.0.0-snapshot-40f66a450ab.jar.sha1 b/server/licenses/lucene-grouping-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index 91a177bd685f8..0000000000000 --- a/server/licenses/lucene-grouping-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ad2d6a215eabae1fdabaf52bd29ed3c9e2a6ec31 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-grouping-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..b8edb14a8a2b8 --- /dev/null +++ b/server/licenses/lucene-grouping-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +0c324076457e9adb989833979979d460ce08ac28 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-8.9.0.jar.sha1 b/server/licenses/lucene-highlighter-8.9.0.jar.sha1 deleted file mode 100644 index 111b023f9502c..0000000000000 --- a/server/licenses/lucene-highlighter-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -94e80bdeab170b0ce1b36a32b6a790d23d7f6d7b \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.0.0-snapshot-40f66a450ab.jar.sha1 b/server/licenses/lucene-highlighter-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index 664f4baf84d92..0000000000000 --- a/server/licenses/lucene-highlighter-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -563d0f6bdf02f165dbd214353c30ec919aada83f \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-highlighter-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..25ac22e4cd3a0 --- /dev/null +++ b/server/licenses/lucene-highlighter-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +4cf11129fa6d4bd756a5bad7a937e6bef291a375 \ No newline at end of file diff --git a/server/licenses/lucene-join-8.9.0.jar.sha1 b/server/licenses/lucene-join-8.9.0.jar.sha1 deleted file mode 100644 index 9b8322851dfec..0000000000000 --- a/server/licenses/lucene-join-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5ae97803efd3344597f6b6bdf823b18d130e8851 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.0.0-snapshot-40f66a450ab.jar.sha1 b/server/licenses/lucene-join-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index 3fc3f8edeb2ec..0000000000000 --- a/server/licenses/lucene-join-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -daa3d8f104e70bf9343827ec6807f1f60e0b3a23 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-join-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..9221f8cddfb06 --- /dev/null +++ b/server/licenses/lucene-join-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +f34253c481cd872418fe6fcd8bc234fc883c1b9c \ No newline at end of file diff --git a/server/licenses/lucene-memory-8.9.0.jar.sha1 b/server/licenses/lucene-memory-8.9.0.jar.sha1 deleted file mode 100644 index bd7fbba668fc4..0000000000000 --- a/server/licenses/lucene-memory-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -09423a6dca2a9ba665e644d86a713d9a6b2b0d3f \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.0.0-snapshot-40f66a450ab.jar.sha1 b/server/licenses/lucene-memory-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index 94bd45d0cdd63..0000000000000 --- a/server/licenses/lucene-memory-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -51dc9c15150bf381e788319306b889cf7fe69667 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-memory-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..379a688bd59c2 --- /dev/null +++ b/server/licenses/lucene-memory-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +c4f46b811e52a208229ef235bccc9039336cd3c3 \ No newline at end of file diff --git a/server/licenses/lucene-misc-8.9.0.jar.sha1 b/server/licenses/lucene-misc-8.9.0.jar.sha1 deleted file mode 100644 index dd4e7baeb3c53..0000000000000 --- a/server/licenses/lucene-misc-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -067494d621ba2ef1f2e4da3ef167106f00b52051 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.0.0-snapshot-40f66a450ab.jar.sha1 b/server/licenses/lucene-misc-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index bcd1f63c66381..0000000000000 --- a/server/licenses/lucene-misc-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c8ecb4533241b40064045ccfb72cb2bd8db57a13 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-misc-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..54a2247d5cf28 --- /dev/null +++ b/server/licenses/lucene-misc-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +df07000dca66571d692bc663cf646daea2f6aa3e \ No newline at end of file diff --git a/server/licenses/lucene-queries-8.9.0.jar.sha1 b/server/licenses/lucene-queries-8.9.0.jar.sha1 deleted file mode 100644 index 682553f877600..0000000000000 --- a/server/licenses/lucene-queries-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c6bda4622abf240da6567a128242f46708fa6c00 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.0.0-snapshot-40f66a450ab.jar.sha1 b/server/licenses/lucene-queries-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index e53baa054896d..0000000000000 --- a/server/licenses/lucene-queries-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f59d7efcba690149743eec8ef4a52a9f70226bd1 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-queries-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..7496218e43788 --- /dev/null +++ b/server/licenses/lucene-queries-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +27673fea59a4cab5067b361d597e6c526f437abf \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-8.9.0.jar.sha1 b/server/licenses/lucene-queryparser-8.9.0.jar.sha1 deleted file mode 100644 index d1978b318fd67..0000000000000 --- a/server/licenses/lucene-queryparser-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -95a9d8cf8ca8eaf9f241fd323697d26d211721b2 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.0.0-snapshot-40f66a450ab.jar.sha1 b/server/licenses/lucene-queryparser-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index 8ebe1681b3aea..0000000000000 --- a/server/licenses/lucene-queryparser-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6b980979ae46e8ed4c734883c4363cdd5b6cc74a \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-queryparser-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..e58db60076867 --- /dev/null +++ b/server/licenses/lucene-queryparser-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +8d8bdccb8ac1a4014d48572d26b2ef56a5aa4b21 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-8.9.0.jar.sha1 b/server/licenses/lucene-sandbox-8.9.0.jar.sha1 deleted file mode 100644 index 9e0539ea7ae5d..0000000000000 --- a/server/licenses/lucene-sandbox-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -af9f6c0287465e17a520b93b684474712433b293 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.0.0-snapshot-40f66a450ab.jar.sha1 b/server/licenses/lucene-sandbox-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index 2e202c273d3bc..0000000000000 --- a/server/licenses/lucene-sandbox-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c92db252f2bfb4ccfdc6b0aa34608eb81256e3ad \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-sandbox-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..6fa870fcc1bd8 --- /dev/null +++ b/server/licenses/lucene-sandbox-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +ca9c5e1655d3fbe3e63cfdde38a0f9c9285d611c \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-8.9.0.jar.sha1 b/server/licenses/lucene-spatial-extras-8.9.0.jar.sha1 deleted file mode 100644 index cedc910adb51b..0000000000000 --- a/server/licenses/lucene-spatial-extras-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c89f4e78712806e8d5bb4adfb21cf0722ad3f175 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.0.0-snapshot-40f66a450ab.jar.sha1 b/server/licenses/lucene-spatial-extras-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index 795893280eb20..0000000000000 --- a/server/licenses/lucene-spatial-extras-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fe0ea6395fea540bea37308b98b1a0b2ba04f0dc \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-spatial-extras-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..88785ab5b712f --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +d04b440abe6e7c25b3aa4d502c4c4cd565184764 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-8.9.0.jar.sha1 b/server/licenses/lucene-spatial3d-8.9.0.jar.sha1 deleted file mode 100644 index f0c08e6e6bd29..0000000000000 --- a/server/licenses/lucene-spatial3d-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0d1238c4e8bf4409b3bb3fbddf2e977b0f19b24b \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.0.0-snapshot-40f66a450ab.jar.sha1 b/server/licenses/lucene-spatial3d-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index ad2bebdf38c5d..0000000000000 --- a/server/licenses/lucene-spatial3d-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -711163fcd9711c5776751e242b589ad8e36c591d \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-spatial3d-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..4d90a60f2fcfa --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +ae3f4375db1b3e461130f3baa8c4817a3e2198c9 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-8.9.0.jar.sha1 b/server/licenses/lucene-suggest-8.9.0.jar.sha1 deleted file mode 100644 index f6f8bb83d66ae..0000000000000 --- a/server/licenses/lucene-suggest-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -98cb69950b48b829b6605a003c99aa7eb86fa9eb \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.0.0-snapshot-40f66a450ab.jar.sha1 b/server/licenses/lucene-suggest-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index c02136cac21b4..0000000000000 --- a/server/licenses/lucene-suggest-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9ad59a62a9741e7747535827de648e00bf960a39 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-suggest-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..878f726798215 --- /dev/null +++ b/server/licenses/lucene-suggest-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +059f2aa35c80619786b50ca4462a5810896b21fb \ No newline at end of file diff --git a/server/src/main/java/org/apache/lucene/search/XCombinedFieldQuery.java b/server/src/main/java/org/apache/lucene/search/XCombinedFieldQuery.java index 76c1ffbeb1024..85904764cf9d4 100644 --- a/server/src/main/java/org/apache/lucene/search/XCombinedFieldQuery.java +++ b/server/src/main/java/org/apache/lucene/search/XCombinedFieldQuery.java @@ -27,6 +27,7 @@ import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermStates; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.sandbox.search.CombinedFieldQuery; import org.apache.lucene.search.similarities.BM25Similarity; import org.apache.lucene.search.similarities.DFRSimilarity; import org.apache.lucene.search.similarities.Similarity; @@ -356,11 +357,6 @@ private CollectionStatistics mergeCollectionStatistics(IndexSearcher searcher) "pseudo_field", maxDoc, docCount, sumTotalTermFreq, sumDocFreq); } - @Override - public void extractTerms(Set termSet) { - termSet.addAll(Arrays.asList(fieldTerms)); - } - @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { Weight weight = diff --git a/server/src/main/java/org/apache/lucene/search/XMultiNormsLeafSimScorer.java b/server/src/main/java/org/apache/lucene/search/XMultiNormsLeafSimScorer.java index 0a4fdf0ff4a14..74c920f94528f 100644 --- a/server/src/main/java/org/apache/lucene/search/XMultiNormsLeafSimScorer.java +++ b/server/src/main/java/org/apache/lucene/search/XMultiNormsLeafSimScorer.java @@ -31,7 +31,7 @@ import java.util.Objects; /** - * Copy of {@link MultiNormsLeafSimScorer} that contains a fix for LUCENE-9999. + * Copy of {@code org.apache.lucene.sandbox.search.MultiNormsLeafSimScorer} that contains a fix for LUCENE-9999. * TODO: remove once LUCENE-9999 is fixed and integrated * *

For all fields, norms must be encoded using {@link SmallFloat#intToByte4}. This scorer also diff --git a/server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java b/server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java index dd12837fce8c0..41e60faa57f46 100644 --- a/server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java +++ b/server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java @@ -11,6 +11,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.queries.BlendedTermQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.MultiPhraseQuery; @@ -19,7 +20,6 @@ import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.XCombinedFieldQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery; import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery; import org.elasticsearch.index.search.ESToParentBlockJoinQuery; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java index 972459d97a831..1bdf84a7c90b8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java @@ -15,8 +15,9 @@ import org.apache.lucene.codecs.PointsReader; import org.apache.lucene.codecs.StoredFieldsReader; import org.apache.lucene.codecs.TermVectorsReader; -import org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat; -import org.apache.lucene.codecs.lucene84.Lucene84PostingsFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat; +import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat; +import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValuesType; @@ -42,7 +43,6 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.FutureArrays; import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.lucene.FilterIndexCommit; @@ -53,6 +53,8 @@ import org.elasticsearch.index.store.LuceneFilesExtensions; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -169,8 +171,8 @@ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { } @Override - public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { - trackField(fieldInfo, Integer.BYTES + value.length); + public void stringField(FieldInfo fieldInfo, String value) throws IOException { + trackField(fieldInfo, Integer.BYTES + value.getBytes(StandardCharsets.UTF_8).length); } @Override @@ -290,6 +292,10 @@ private void readProximity(Terms terms, PostingsEnum postings) throws IOExceptio private BlockTermState getBlockTermState(TermsEnum termsEnum, BytesRef term) throws IOException { if (term != null && termsEnum.seekExact(term)) { final TermState termState = termsEnum.termState(); + if (termState instanceof Lucene90PostingsFormat.IntBlockTermState) { + final Lucene90PostingsFormat.IntBlockTermState blockTermState = (Lucene90PostingsFormat.IntBlockTermState) termState; + return new BlockTermState(blockTermState.docStartFP, blockTermState.posStartFP, blockTermState.payStartFP); + } if (termState instanceof Lucene84PostingsFormat.IntBlockTermState) { final Lucene84PostingsFormat.IntBlockTermState blockTermState = (Lucene84PostingsFormat.IntBlockTermState) termState; return new BlockTermState(blockTermState.docStartFP, blockTermState.posStartFP, blockTermState.payStartFP); @@ -425,8 +431,8 @@ public void visit(int docID, byte[] packedValue) throws IOException { public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { for (int dim = 0; dim < numDims; dim++) { int offset = dim * bytesPerDim; - if (FutureArrays.compareUnsigned(minPackedValue, offset, offset + bytesPerDim, point, offset, offset + bytesPerDim) > 0 || - FutureArrays.compareUnsigned(maxPackedValue, offset, offset + bytesPerDim, point, offset, offset + bytesPerDim) < 0) { + if (Arrays.compareUnsigned(minPackedValue, offset, offset + bytesPerDim, point, offset, offset + bytesPerDim) > 0 || + Arrays.compareUnsigned(maxPackedValue, offset, offset + bytesPerDim, point, offset, offset + bytesPerDim) < 0) { return PointValues.Relation.CELL_OUTSIDE_QUERY; } } diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java index 6018c282699c8..caaeaac49a5e6 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -24,7 +24,6 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.Fields; import org.apache.lucene.index.FilterCodecReader; import org.apache.lucene.index.FilterDirectoryReader; import org.apache.lucene.index.FilterLeafReader; @@ -48,6 +47,7 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.VectorValues; import org.apache.lucene.search.DocIdSetIterator; @@ -77,14 +77,14 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Version; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.core.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader; import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore; import org.elasticsearch.common.util.iterable.Iterables; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -1030,7 +1030,7 @@ public VectorValues getVectorValues(String field) throws IOException { } @Override - public TopDocs searchNearestVectors(String field, float[] target, int k, int fanout) throws IOException { + public TopDocs searchNearestVectors(String field, float[] target, int k) throws IOException { return null; } @@ -1049,7 +1049,8 @@ public PointValues getPointValues(String fieldName) { public void checkIntegrity() { } - public Fields getTermVectors(int docID) { + @Override + public TermVectors getTermVectorsReader() { return null; } diff --git a/server/src/main/java/org/elasticsearch/common/lucene/MinimumScoreCollector.java b/server/src/main/java/org/elasticsearch/common/lucene/MinimumScoreCollector.java index 3dcc869d53d39..eeff832c36d70 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/MinimumScoreCollector.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/MinimumScoreCollector.java @@ -34,7 +34,7 @@ public MinimumScoreCollector(Collector collector, float minimumScore) { @Override public void setScorer(Scorable scorer) throws IOException { if ((scorer instanceof ScoreCachingWrappingScorer) == false) { - scorer = new ScoreCachingWrappingScorer(scorer); + scorer = ScoreCachingWrappingScorer.wrap(scorer); } this.scorer = scorer; leafCollector.setScorer(scorer); diff --git a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java index d14af5a264d37..c304d8f85bf3d 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java +++ b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java @@ -15,7 +15,6 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.Fields; import org.apache.lucene.index.ImpactsEnum; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexOptions; @@ -30,9 +29,13 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.index.VectorValues; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; @@ -56,7 +59,6 @@ import org.elasticsearch.index.translog.Translog; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.Map; import java.util.Set; @@ -136,13 +138,13 @@ private static class TranslogLeafReader extends LeafReader { private static final FieldInfo FAKE_SOURCE_FIELD = new FieldInfo(SourceFieldMapper.NAME, 1, false, false, false, IndexOptions.NONE, - DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, false); + DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, 0, VectorSimilarityFunction.EUCLIDEAN, false); private static final FieldInfo FAKE_ROUTING_FIELD = new FieldInfo(RoutingFieldMapper.NAME, 2, false, false, false, IndexOptions.NONE, - DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, false); + DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, 0, VectorSimilarityFunction.EUCLIDEAN, false); private static final FieldInfo FAKE_ID_FIELD = new FieldInfo(IdFieldMapper.NAME, 3, false, false, false, IndexOptions.DOCS, - DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, false); + DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, 0, VectorSimilarityFunction.EUCLIDEAN, false); private static Set TRANSLOG_FIELD_NAMES = Sets.newHashSet(SourceFieldMapper.NAME, RoutingFieldMapper.NAME, IdFieldMapper.NAME); @@ -273,6 +275,16 @@ public NumericDocValues getNormValues(String field) throws IOException { return getDelegate().getNormValues(field); } + @Override + public VectorValues getVectorValues(String field) throws IOException { + return getDelegate().getVectorValues(field); + } + + @Override + public TopDocs searchNearestVectors(String field, float[] target, int k) throws IOException { + return getDelegate().searchNearestVectors(field, target, k); + } + @Override public FieldInfos getFieldInfos() { return getDelegate().getFieldInfos(); @@ -298,8 +310,8 @@ public LeafMetaData getMetaData() { } @Override - public Fields getTermVectors(int docID) throws IOException { - return getDelegate().getTermVectors(docID); + public TermVectors getTermVectorsReader() { + return getDelegate().getTermVectorsReader(); } @Override @@ -348,7 +360,7 @@ private void readStoredFieldsDirectly(StoredFieldVisitor visitor) throws IOExcep } } if (operation.routing() != null && visitor.needsField(FAKE_ROUTING_FIELD) == StoredFieldVisitor.Status.YES) { - visitor.stringField(FAKE_ROUTING_FIELD, operation.routing().getBytes(StandardCharsets.UTF_8)); + visitor.stringField(FAKE_ROUTING_FIELD, operation.routing()); } if (visitor.needsField(FAKE_ID_FIELD) == StoredFieldVisitor.Status.YES) { final byte[] id = new byte[uid.length]; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java index 408258d8a0678..485fa1481dd07 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java @@ -12,7 +12,6 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafMetaData; @@ -24,7 +23,9 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; +import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.index.VectorValues; import org.apache.lucene.index.memory.MemoryIndex; import org.apache.lucene.search.DocIdSetIterator; @@ -194,7 +195,7 @@ public VectorValues getVectorValues(String field) throws IOException { } @Override - public TopDocs searchNearestVectors(String field, float[] target, int k, int fanout) throws IOException { + public TopDocs searchNearestVectors(String field, float[] target, int k) throws IOException { return null; } @@ -219,7 +220,7 @@ public LeafMetaData getMetaData() { } @Override - public Fields getTermVectors(int docID) throws IOException { + public TermVectors getTermVectorsReader() { throw new UnsupportedOperationException(); } @@ -260,7 +261,7 @@ private static FieldInfo fieldInfo(String name) { 0, 0, 0, - VectorValues.SimilarityFunction.NONE, + VectorSimilarityFunction.EUCLIDEAN, false ); } diff --git a/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java index 94dc2181c4ea1..6c090f7a2bd56 100644 --- a/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java @@ -13,7 +13,6 @@ import org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute; import org.apache.lucene.index.Term; import org.apache.lucene.misc.search.similarity.LegacyBM25Similarity; -import org.apache.lucene.sandbox.search.CombinedFieldQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostAttribute; diff --git a/server/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java index 23b9b66f39b28..27b060f94744a 100644 --- a/server/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java @@ -8,9 +8,9 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.FieldMaskingSpanQuery; +import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.FieldMaskingSpanQuery; -import org.apache.lucene.search.spans.SpanQuery; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/MultiBucketCollector.java b/server/src/main/java/org/elasticsearch/search/aggregations/MultiBucketCollector.java index e5a357938046d..a8cc26ecc2e44 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/MultiBucketCollector.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/MultiBucketCollector.java @@ -212,7 +212,7 @@ private MultiLeafBucketCollector(List collectors, boolean c @Override public void setScorer(Scorable scorer) throws IOException { if (cacheScores) { - scorer = new ScoreCachingWrappingScorer(scorer); + scorer = ScoreCachingWrappingScorer.wrap(scorer); } for (int i = 0; i < numCollectors; ++i) { final LeafCollector c = collectors[i]; diff --git a/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java b/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java index 34004f7a3ebc3..8749e49de8240 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java +++ b/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.suggest.document.CompletionTerms; @@ -85,12 +86,12 @@ public FieldUsageTrackingLeafReader(LeafReader in, FieldUsageNotifier notifier) } @Override - public Fields getTermVectors(int docID) throws IOException { - Fields f = super.getTermVectors(docID); - if (f != null) { - f = new FieldUsageTrackingTermVectorFields(f); + public TermVectors getTermVectorsReader() { + TermVectors in = super.getTermVectorsReader(); + if (in != null) { + return new FieldUsageTrackingTermVectors(in); } - return f; + return in; } @Override @@ -213,11 +214,6 @@ public void checkIntegrity() throws IOException { public void close() throws IOException { reader.close(); } - - @Override - public long ramBytesUsed() { - return reader.ramBytesUsed(); - } } private class FieldUsageTrackingTerms extends FilterTerms { @@ -330,6 +326,20 @@ public Terms terms(String field) throws IOException { } + private class FieldUsageTrackingTermVectors extends TermVectors { + + final TermVectors in; + + private FieldUsageTrackingTermVectors(TermVectors in) { + this.in = in; + } + + @Override + public Fields get(int doc) throws IOException { + return new FieldUsageTrackingTermVectorFields(in.get(doc)); + } + } + private class FieldUsageStoredFieldVisitor extends FilterStoredFieldVisitor { FieldUsageStoredFieldVisitor(StoredFieldVisitor visitor) { super(visitor); diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilterFieldNamesProvidingStoredFieldsVisitor.java b/server/src/main/java/org/elasticsearch/search/internal/FilterFieldNamesProvidingStoredFieldsVisitor.java index 329a746f075a8..6d3c7395a62a0 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/FilterFieldNamesProvidingStoredFieldsVisitor.java +++ b/server/src/main/java/org/elasticsearch/search/internal/FilterFieldNamesProvidingStoredFieldsVisitor.java @@ -28,7 +28,7 @@ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { } @Override - public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { + public void stringField(FieldInfo fieldInfo, String value) throws IOException { visitor.stringField(fieldInfo, value); } diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilterStoredFieldVisitor.java b/server/src/main/java/org/elasticsearch/search/internal/FilterStoredFieldVisitor.java index a695fae27346d..1051eee5d54f1 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/FilterStoredFieldVisitor.java +++ b/server/src/main/java/org/elasticsearch/search/internal/FilterStoredFieldVisitor.java @@ -26,7 +26,7 @@ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { } @Override - public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { + public void stringField(FieldInfo fieldInfo, String value) throws IOException { visitor.stringField(fieldInfo, value); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java index 8a9f2534f7002..b90c35107b0a4 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java @@ -10,9 +10,9 @@ import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat; -import org.apache.lucene.codecs.lucene84.Lucene84PostingsFormat; -import org.apache.lucene.codecs.lucene87.Lucene87Codec; +import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat; +import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat; +import org.apache.lucene.backward_codecs.lucene87.Lucene87Codec; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.apache.lucene.document.BinaryDocValuesField; diff --git a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java index 9578e306583ac..789b214ba71e2 100644 --- a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.index.query; -import org.apache.lucene.sandbox.search.CombinedFieldQuery; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; diff --git a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java index 7b570bcb40646..8161b9917368d 100644 --- a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java @@ -13,7 +13,6 @@ import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.index.Term; -import org.apache.lucene.sandbox.search.CombinedFieldQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; diff --git a/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java index 7874867773f83..64621efe28636 100644 --- a/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java @@ -9,10 +9,10 @@ package org.elasticsearch.index.query; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.spans.FieldMaskingSpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.FieldMaskingSpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.elasticsearch.common.ParsingException; import org.elasticsearch.test.AbstractQueryTestCase; diff --git a/server/src/test/java/org/elasticsearch/search/lookup/LeafStoredFieldsLookupTests.java b/server/src/test/java/org/elasticsearch/search/lookup/LeafStoredFieldsLookupTests.java index f6acc9e5947de..3cd0c8dc4304b 100644 --- a/server/src/test/java/org/elasticsearch/search/lookup/LeafStoredFieldsLookupTests.java +++ b/server/src/test/java/org/elasticsearch/search/lookup/LeafStoredFieldsLookupTests.java @@ -10,7 +10,7 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.index.VectorValues; +import org.apache.lucene.index.VectorSimilarityFunction; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.test.ESTestCase; import org.junit.Before; @@ -36,7 +36,7 @@ public void setUp() throws Exception { (Double) invocation.getArguments()[0] + 10); FieldInfo mockFieldInfo = new FieldInfo("field", 1, false, false, true, - IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, 0, VectorValues.SimilarityFunction.NONE, false); + IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, 0, VectorSimilarityFunction.EUCLIDEAN, false); fieldsLookup = new LeafStoredFieldsLookup(field -> field.equals("field") || field.equals("alias") ? fieldType : null, (doc, visitor) -> visitor.doubleField(mockFieldInfo, 2.718)); diff --git a/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java b/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java index 619b7484c7f78..db2913b0bb7c8 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java +++ b/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java @@ -16,6 +16,7 @@ import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; @@ -82,10 +83,18 @@ public Terms terms(String field) throws IOException { } @Override - public Fields getTermVectors(int docID) throws IOException { - Fields fields = super.getTermVectors(docID); - thrower.maybeThrow(Flags.TermVectors); - return fields == null ? null : new ThrowingFields(fields, thrower); + public TermVectors getTermVectorsReader() { + TermVectors in = super.getTermVectorsReader(); + if (in == null) { + return null; + } + return new TermVectors() { + @Override + public Fields get(int doc) throws IOException { + thrower.maybeThrow(Flags.TermVectors); + return new ThrowingFields(in.get(doc), thrower); + } + }; } /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java index 210a4a5cced39..7439326675765 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java @@ -10,7 +10,6 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.LeafMetaData; @@ -22,6 +21,7 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.VectorValues; import org.apache.lucene.search.TopDocs; @@ -221,7 +221,7 @@ public VectorValues getVectorValues(String field) throws IOException { } @Override - public TopDocs searchNearestVectors(String field, float[] target, int k, int fanout) throws IOException { + public TopDocs searchNearestVectors(String field, float[] target, int k) throws IOException { throw new UnsupportedOperationException(); } @@ -250,7 +250,7 @@ public LeafMetaData getMetaData() { } @Override - public Fields getTermVectors(int docID) { + public TermVectors getTermVectorsReader() { throw new UnsupportedOperationException(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java index 19f5e0b0cdb46..30f047a0e9a94 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java @@ -23,7 +23,7 @@ import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.index.SoftDeletesDirectoryReaderWrapper; import org.apache.lucene.index.StandardDirectoryReader; -import org.apache.lucene.index.VectorValues; +import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -224,7 +224,7 @@ private SegmentCommitInfo syncSegment(SegmentCommitInfo segmentCommitInfo, LiveD for (FieldInfo fieldInfo : fieldInfos) { fieldInfoCopy.add(new FieldInfo(fieldInfo.name, fieldInfo.number, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, fieldInfo.attributes(), 0, 0, 0, - 0, VectorValues.SimilarityFunction.NONE, fieldInfo.isSoftDeletesField())); + 0, VectorSimilarityFunction.EUCLIDEAN, fieldInfo.isSoftDeletesField())); } FieldInfos newFieldInfos = new FieldInfos(fieldInfoCopy.toArray(new FieldInfo[0])); codec.fieldInfosFormat().write(trackingDir, newSegmentInfo, segmentSuffix, newFieldInfos, IOContext.DEFAULT); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java index 7d1515a05d7ec..ca7d72a264bf9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java @@ -22,6 +22,7 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.TermState; +import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; @@ -149,14 +150,23 @@ public FieldInfos getFieldInfos() { } @Override - public Fields getTermVectors(int docID) throws IOException { - Fields f = super.getTermVectors(docID); - if (f == null) { + public TermVectors getTermVectorsReader() { + TermVectors in = super.getTermVectorsReader(); + if (in == null) { return null; } - f = new FieldFilterFields(f); - // we need to check for emptyness, so we can return null: - return f.iterator().hasNext() ? f : null; + return new TermVectors() { + @Override + public Fields get(int doc) throws IOException { + Fields f = in.get(doc); + if (f == null) { + return null; + } + f = new FieldFilterFields(f); + // we need to check for emptyness, so we can return null: + return f.iterator().hasNext() ? f : null; + } + }; } /** Filter a map by a {@link CharacterRunAutomaton} that defines the fields to retain. */ diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractorTests.java index e42f49c3a25d6..fc30c61cd0aee 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractorTests.java @@ -9,6 +9,7 @@ import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.sandbox.search.DocValuesNumbersQuery; import org.apache.lucene.search.AssertingQuery; import org.apache.lucene.search.BooleanClause; @@ -24,7 +25,6 @@ import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.test.ESTestCase; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/FileRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/FileRolesStoreTests.java index 4b33aa6f9551c..e2625a2f11ff0 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/FileRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/FileRolesStoreTests.java @@ -153,7 +153,7 @@ public void testParseFile() throws Exception { assertThat(group.privilege(), notNullValue()); assertTrue(Operations.sameLanguage(group.privilege().getAutomaton(), MinimizationOperations.minimize(Operations.union(IndexPrivilege.READ.getAutomaton(), IndexPrivilege.WRITE.getAutomaton()), - Operations.DEFAULT_MAX_DETERMINIZED_STATES))); + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT))); descriptor = roles.get("role4"); assertNull(descriptor); diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.9.0.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.9.0.jar.sha1 deleted file mode 100644 index 859cdb2013408..0000000000000 --- a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5c3f72357089f7f0c1ef44bbe7b4c67b6149a5af \ No newline at end of file diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-40f66a450ab.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-40f66a450ab.jar.sha1 deleted file mode 100644 index a1f1b4f2d200c..0000000000000 --- a/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-40f66a450ab.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4dafc4b2928634f5b0c5d783c91070ddc91154d0 \ No newline at end of file diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-56eb76dbaf7.jar.sha1 new file mode 100644 index 0000000000000..b3d858bef510c --- /dev/null +++ b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-56eb76dbaf7.jar.sha1 @@ -0,0 +1 @@ +4ee46802b59e1d0ff15882ef735c500d5bc1cb6b \ No newline at end of file From 2dc6a3def6b01c43b35a60b94bfe86199224c80a Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Fri, 30 Jul 2021 13:46:32 +0100 Subject: [PATCH 07/67] small updates --- .../elasticsearch/common/lucene/Lucene.java | 2 +- .../org/elasticsearch/index/store/Store.java | 28 +++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java index caaeaac49a5e6..ebd2c5d6a20cf 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -101,7 +101,7 @@ import java.util.Map; public class Lucene { - public static final String LATEST_CODEC = "Lucene87"; + public static final String LATEST_CODEC = "Lucene90"; public static final String SOFT_DELETES_FIELD = "__soft_deletes"; diff --git a/server/src/main/java/org/elasticsearch/index/store/Store.java b/server/src/main/java/org/elasticsearch/index/store/Store.java index b303811829bf4..fecf8c932d851 100644 --- a/server/src/main/java/org/elasticsearch/index/store/Store.java +++ b/server/src/main/java/org/elasticsearch/index/store/Store.java @@ -52,10 +52,10 @@ import org.elasticsearch.common.lucene.store.InputStreamIndexInput; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; -import org.elasticsearch.core.TimeValue; +import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.RefCounted; -import org.elasticsearch.common.util.iterable.Iterables; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.ShardLock; @@ -1213,6 +1213,17 @@ static class VerifyingIndexInput extends ChecksumIndexInput { private final byte[] checksum = new byte[8]; private long verifiedPosition = 0; + private static final int SKIP_BUFFER_SIZE = 1024; + + /* This buffer is used when skipping bytes in skipBytes(). Skipping bytes + * still requires reading in the bytes we skip in order to update the checksum. + * The reason we need to use an instance member instead of sharing a single + * static instance across threads is that multiple instances invoking skipBytes() + * concurrently on different threads can clobber the contents of a shared buffer, + * corrupting the checksum. See LUCENE-5583 for additional context. + */ + private byte[] skipBuffer; + VerifyingIndexInput(IndexInput input) { this(input, new BufferedChecksum(new CRC32())); } @@ -1291,6 +1302,19 @@ public void seek(long pos) throws IOException { } } + @Override + public void skipBytes(long numBytes) throws IOException { + if (skipBuffer == null) { + skipBuffer = new byte[SKIP_BUFFER_SIZE]; + } + assert skipBuffer.length == SKIP_BUFFER_SIZE; + for (long skipped = 0; skipped < numBytes; ) { + final int step = (int) Math.min(SKIP_BUFFER_SIZE, numBytes - skipped); + readBytes(skipBuffer, 0, step); + skipped += step; + } + } + @Override public void close() throws IOException { input.close(); From a4e93591f0d0be1722d7788ec5a00b6fa5e3ac94 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 2 Aug 2021 10:22:51 +0100 Subject: [PATCH 08/67] iter --- build-tools-internal/version.properties | 2 +- docs/Versions.asciidoc | 4 ++-- .../main/java/org/elasticsearch/index/store/Store.java | 10 ++++++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 972b0c588d99b..3e972fbef2898 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,5 +1,5 @@ elasticsearch = 8.0.0 -lucene = 9.0.0-snapshot-56eb76dbaf7 +lucene = 9.0.0-snapshot-7450a7e64be bundled_jdk_vendor = adoptopenjdk bundled_jdk = 16.0.1+9 diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc index 21d9604113d07..b95585ea13e43 100644 --- a/docs/Versions.asciidoc +++ b/docs/Versions.asciidoc @@ -1,8 +1,8 @@ include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[] -:lucene_version: 8.9.0 -:lucene_version_path: 8_9_0 +:lucene_version: 9.0.0 +:lucene_version_path: 9_0_0 :jdk: 11.0.2 :jdk_major: 11 :build_flavor: default diff --git a/server/src/main/java/org/elasticsearch/index/store/Store.java b/server/src/main/java/org/elasticsearch/index/store/Store.java index fecf8c932d851..46de9a1cdb68c 100644 --- a/server/src/main/java/org/elasticsearch/index/store/Store.java +++ b/server/src/main/java/org/elasticsearch/index/store/Store.java @@ -75,6 +75,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; +import java.io.UncheckedIOException; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.util.ArrayList; @@ -1145,7 +1146,7 @@ public void verify() throws IOException { String footerDigest = null; if (metadata.checksum().equals(actualChecksum) && writtenBytes == metadata.length()) { ByteArrayIndexInput indexInput = new ByteArrayIndexInput("checksum", this.footerChecksum); - footerDigest = digestToString(indexInput.readLong()); + footerDigest = digestToString(CodecUtil.readBELong(indexInput)); if (metadata.checksum().equals(footerDigest)) { return; } @@ -1341,7 +1342,12 @@ public IndexInput slice(String sliceDescription, long offset, long length) throw } public long getStoredChecksum() { - return new ByteArrayDataInput(checksum).readLong(); + try { + return CodecUtil.readBELong(new ByteArrayDataInput(checksum)); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } } public long verify() throws CorruptIndexException { From 896b1e0294a4047906b430045156e8461bb7e87a Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 2 Aug 2021 10:42:42 +0100 Subject: [PATCH 09/67] another upgrade... --- .../lucene-expressions-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - .../lucene-expressions-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + .../lucene-analysis-icu-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - .../lucene-analysis-icu-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + .../lucene-analysis-kuromoji-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - .../lucene-analysis-kuromoji-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + .../lucene-analysis-nori-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - .../lucene-analysis-nori-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + .../lucene-analysis-phonetic-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - .../lucene-analysis-phonetic-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + .../lucene-analysis-smartcn-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - .../lucene-analysis-smartcn-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + .../lucene-analysis-stempel-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - .../lucene-analysis-stempel-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + ...ucene-analysis-morfologik-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - ...ucene-analysis-morfologik-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + .../lucene-analysis-common-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - .../lucene-analysis-common-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + .../lucene-backward-codecs-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - .../lucene-backward-codecs-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + server/licenses/lucene-core-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - server/licenses/lucene-core-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + .../licenses/lucene-grouping-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - .../licenses/lucene-grouping-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + .../lucene-highlighter-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - .../lucene-highlighter-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + server/licenses/lucene-join-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - server/licenses/lucene-join-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + .../licenses/lucene-memory-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - .../licenses/lucene-memory-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + server/licenses/lucene-misc-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - server/licenses/lucene-misc-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + .../licenses/lucene-queries-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - .../licenses/lucene-queries-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + .../lucene-queryparser-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - .../lucene-queryparser-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + .../licenses/lucene-sandbox-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - .../licenses/lucene-sandbox-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + .../lucene-spatial-extras-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - .../lucene-spatial-extras-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + .../lucene-spatial3d-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - .../lucene-spatial3d-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + .../licenses/lucene-suggest-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - .../licenses/lucene-suggest-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + .../licenses/lucene-core-9.0.0-snapshot-56eb76dbaf7.jar.sha1 | 1 - .../licenses/lucene-core-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 + 46 files changed, 23 insertions(+), 23 deletions(-) delete mode 100644 modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-7450a7e64be.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-7450a7e64be.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-7450a7e64be.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-7450a7e64be.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-7450a7e64be.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-7450a7e64be.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-7450a7e64be.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-7450a7e64be.jar.sha1 delete mode 100644 server/licenses/lucene-analysis-common-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 server/licenses/lucene-analysis-common-9.0.0-snapshot-7450a7e64be.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 server/licenses/lucene-backward-codecs-9.0.0-snapshot-7450a7e64be.jar.sha1 delete mode 100644 server/licenses/lucene-core-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 server/licenses/lucene-core-9.0.0-snapshot-7450a7e64be.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 server/licenses/lucene-grouping-9.0.0-snapshot-7450a7e64be.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 server/licenses/lucene-highlighter-9.0.0-snapshot-7450a7e64be.jar.sha1 delete mode 100644 server/licenses/lucene-join-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 server/licenses/lucene-join-9.0.0-snapshot-7450a7e64be.jar.sha1 delete mode 100644 server/licenses/lucene-memory-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 server/licenses/lucene-memory-9.0.0-snapshot-7450a7e64be.jar.sha1 delete mode 100644 server/licenses/lucene-misc-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 server/licenses/lucene-misc-9.0.0-snapshot-7450a7e64be.jar.sha1 delete mode 100644 server/licenses/lucene-queries-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 server/licenses/lucene-queries-9.0.0-snapshot-7450a7e64be.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 server/licenses/lucene-queryparser-9.0.0-snapshot-7450a7e64be.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 server/licenses/lucene-sandbox-9.0.0-snapshot-7450a7e64be.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 server/licenses/lucene-spatial-extras-9.0.0-snapshot-7450a7e64be.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 server/licenses/lucene-spatial3d-9.0.0-snapshot-7450a7e64be.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 server/licenses/lucene-suggest-9.0.0-snapshot-7450a7e64be.jar.sha1 delete mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-56eb76dbaf7.jar.sha1 create mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-7450a7e64be.jar.sha1 diff --git a/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index b6dfa64f8d20f..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -31b1f2a168e8ce93504682d67f37b6ef23f727a0 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-7450a7e64be.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..520fdbe7366e1 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +42eaef59486b8d205136510bb4adbcda64b07ff9 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index 7bb3589ba53dd..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -68ea6ee2a48ada4de743cfbdd2a2bdf0c3807946 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-7450a7e64be.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..e7a01f6dbb35e --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +0dd17e1de76b20e3d1babf7b05d93a79f08c06f6 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index 0f20e9851a208..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -82a7ed2c9845da0c575e538c75d63404c61efb14 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-7450a7e64be.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..14495774b62b0 --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +a96be6ae01b8a7d11d2aefd77471ba196e0e8300 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index a9f1750f2029c..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d23fea6880e75d8a8f44126cce618d2ceb49c75b \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-7450a7e64be.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..2329895df70aa --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +75cbbfab82dc45ba4e042f7a72f63e564c560a4d \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index ed0d47eaa12e1..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -29587ba58012b1796ea46fff081a9d548cafef76 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-7450a7e64be.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..8013689bc03b4 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +232334a042b1c40bf6f9e84f39ecbdba8a6fc653 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index 427f6663e34fb..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2675530dfcf22dd067960cba65b8a474364a7566 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-7450a7e64be.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..6a42857d6a2a4 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +60bffa02083a2f5d02e86ad2e492d9fd2c1f5524 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index 8a5f8ee423906..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -99f98cd4198f58079f21e3a6ee54105b2bae524c \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-7450a7e64be.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..9b954c1f7264d --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +84d9fcd0dbd37cd1865275956e113a9b1ab99f59 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index 1a7d6af9439ca..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e0113034825d158be0e52aa143ab5183c82a1259 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-7450a7e64be.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..8907c9d3689d9 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +625c56a51dd167183161df7ab18e8bb9822e93f1 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-analysis-common-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index c2d1b8fff7d96..0000000000000 --- a/server/licenses/lucene-analysis-common-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e72a4ea400998b4e62d1b7529fec190c111c1312 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-analysis-common-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..d505d5a84e70a --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +8cf0f514ff85880fdf334ac7d7c06c6189b265b9 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index 35e8a6d4fa34c..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -eac640612d190818886dd2950aafd579c3a7a867 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..df06d7e6cb66b --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +2b72141c8d38ad290f1d2b10ee45b3fe6b243976 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-core-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index b3d858bef510c..0000000000000 --- a/server/licenses/lucene-core-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4ee46802b59e1d0ff15882ef735c500d5bc1cb6b \ No newline at end of file diff --git a/server/licenses/lucene-core-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-core-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..a6ac4928e168a --- /dev/null +++ b/server/licenses/lucene-core-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +e329f855502ece4db3155378a90463e9068ca7c8 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-grouping-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index b8edb14a8a2b8..0000000000000 --- a/server/licenses/lucene-grouping-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0c324076457e9adb989833979979d460ce08ac28 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-grouping-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..b1c5042ea1cf1 --- /dev/null +++ b/server/licenses/lucene-grouping-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +681c676d7a1ee4620b434f6a5c888f482407873f \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-highlighter-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index 25ac22e4cd3a0..0000000000000 --- a/server/licenses/lucene-highlighter-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4cf11129fa6d4bd756a5bad7a937e6bef291a375 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-highlighter-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..e215f0e64f356 --- /dev/null +++ b/server/licenses/lucene-highlighter-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +8b9902a8d9972ca93e08ff7117431aa1edf31adb \ No newline at end of file diff --git a/server/licenses/lucene-join-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-join-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index 9221f8cddfb06..0000000000000 --- a/server/licenses/lucene-join-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f34253c481cd872418fe6fcd8bc234fc883c1b9c \ No newline at end of file diff --git a/server/licenses/lucene-join-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-join-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..3ee6334fc43de --- /dev/null +++ b/server/licenses/lucene-join-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +7f8556d5c71fd81e198ea86020d7a7e535e6ca66 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-memory-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index 379a688bd59c2..0000000000000 --- a/server/licenses/lucene-memory-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c4f46b811e52a208229ef235bccc9039336cd3c3 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-memory-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..2baf04ae15e8e --- /dev/null +++ b/server/licenses/lucene-memory-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +e052e67aee4cce3efdfa891f3e56ab9357c27979 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-misc-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index 54a2247d5cf28..0000000000000 --- a/server/licenses/lucene-misc-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -df07000dca66571d692bc663cf646daea2f6aa3e \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-misc-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..2419f8b68f9ce --- /dev/null +++ b/server/licenses/lucene-misc-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +3e3f073bb1c3886433995f3c8af151d5b5f2eea2 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-queries-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index 7496218e43788..0000000000000 --- a/server/licenses/lucene-queries-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -27673fea59a4cab5067b361d597e6c526f437abf \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-queries-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..9acaba5b9787d --- /dev/null +++ b/server/licenses/lucene-queries-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +890ac3ade381800495e7cb1d7681b16253e2b954 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-queryparser-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index e58db60076867..0000000000000 --- a/server/licenses/lucene-queryparser-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8d8bdccb8ac1a4014d48572d26b2ef56a5aa4b21 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-queryparser-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..b565678a409b2 --- /dev/null +++ b/server/licenses/lucene-queryparser-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +1d07f35c6898613f8e90e1c291ec310e261b1fb9 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-sandbox-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index 6fa870fcc1bd8..0000000000000 --- a/server/licenses/lucene-sandbox-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ca9c5e1655d3fbe3e63cfdde38a0f9c9285d611c \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-sandbox-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..3a81fb1b1cb59 --- /dev/null +++ b/server/licenses/lucene-sandbox-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +08ea5e7590e31366573353cf7bb58d3ef8139af0 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-spatial-extras-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index 88785ab5b712f..0000000000000 --- a/server/licenses/lucene-spatial-extras-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d04b440abe6e7c25b3aa4d502c4c4cd565184764 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-spatial-extras-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..44295b512f3f4 --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +3f93555280b3d536b4ff8c2994b7ffda569fd268 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-spatial3d-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index 4d90a60f2fcfa..0000000000000 --- a/server/licenses/lucene-spatial3d-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ae3f4375db1b3e461130f3baa8c4817a3e2198c9 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-spatial3d-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..268cfc7fe4336 --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +8414a83e906f3a9acb1a6420b765c59539f7da92 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/server/licenses/lucene-suggest-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index 878f726798215..0000000000000 --- a/server/licenses/lucene-suggest-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -059f2aa35c80619786b50ca4462a5810896b21fb \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-suggest-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..113ad21da0033 --- /dev/null +++ b/server/licenses/lucene-suggest-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +2d4183e10ae0ca10515c9db6523b861783363cd0 \ No newline at end of file diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-56eb76dbaf7.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-56eb76dbaf7.jar.sha1 deleted file mode 100644 index b3d858bef510c..0000000000000 --- a/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-56eb76dbaf7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4ee46802b59e1d0ff15882ef735c500d5bc1cb6b \ No newline at end of file diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-7450a7e64be.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-7450a7e64be.jar.sha1 new file mode 100644 index 0000000000000..a6ac4928e168a --- /dev/null +++ b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-7450a7e64be.jar.sha1 @@ -0,0 +1 @@ +e329f855502ece4db3155378a90463e9068ca7c8 \ No newline at end of file From 104787f559c898634df58a1d6dd68a70743a2edc Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 2 Aug 2021 11:06:47 +0100 Subject: [PATCH 10/67] Lucene test naming conventions --- ...okenFilterTests.java => TestDeDuplicatingTokenFilter.java} | 2 +- ...cateTokenFilterTests.java => TestTruncateTokenFilter.java} | 2 +- ...ollectorTests.java => TestCollapsingTopDocsCollector.java} | 2 +- ...ts.java => TestCollapsingTopDocsCollectorSearchAfter.java} | 4 ++-- ...ts.java => TestLazySoftDeletesDirectoryReaderWrapper.java} | 2 +- ...ergePolicyTests.java => TestShuffleForcedMergePolicy.java} | 2 +- ...angeQueryTests.java => TestBinaryDocValuesRangeQuery.java} | 2 +- .../{BlendedTermQueryTests.java => TestBlendedTermQuery.java} | 2 +- ...ts.java => TestDoubleRandomBinaryDocValuesRangeQuery.java} | 2 +- ...sts.java => TestFloatRandomBinaryDocValuesRangeQuery.java} | 2 +- ...va => TestInetAddressRandomBinaryDocValuesRangeQuery.java} | 2 +- ...s.java => TestIntegerRandomBinaryDocValuesRangeQuery.java} | 2 +- ...ests.java => TestLongRandomBinaryDocValuesRangeQuery.java} | 2 +- .../queries/{MinDocQueryTests.java => TestMinDocQuery.java} | 2 +- ...dDocQueryTests.java => TestSearchAfterSortedDocQuery.java} | 2 +- ...tchNoDocsQueryTests.java => TestSpanMatchNoDocsQuery.java} | 2 +- ...binedFieldQueryTests.java => TestXCombinedFieldQuery.java} | 2 +- ...ScannerTests.java => TestBoundedBreakIteratorScanner.java} | 2 +- ...ageFormatterTests.java => TestCustomPassageFormatter.java} | 2 +- ...ighlighterTests.java => TestCustomUnifiedHighlighter.java} | 2 +- .../{CombinedBitSetTests.java => TestCombinedBitSet.java} | 2 +- 21 files changed, 22 insertions(+), 22 deletions(-) rename server/src/test/java/org/apache/lucene/analysis/miscellaneous/{DeDuplicatingTokenFilterTests.java => TestDeDuplicatingTokenFilter.java} (98%) rename server/src/test/java/org/apache/lucene/analysis/miscellaneous/{TruncateTokenFilterTests.java => TestTruncateTokenFilter.java} (97%) rename server/src/test/java/org/apache/lucene/grouping/{CollapsingTopDocsCollectorTests.java => TestCollapsingTopDocsCollector.java} (99%) rename server/src/test/java/org/apache/lucene/grouping/{CollapsingTopDocsCollectorSearchAfterTests.java => TestCollapsingTopDocsCollectorSearchAfter.java} (98%) rename server/src/test/java/org/apache/lucene/index/{LazySoftDeletesDirectoryReaderWrapperTests.java => TestLazySoftDeletesDirectoryReaderWrapper.java} (98%) rename server/src/test/java/org/apache/lucene/index/{ShuffleForcedMergePolicyTests.java => TestShuffleForcedMergePolicy.java} (97%) rename server/src/test/java/org/apache/lucene/queries/{BinaryDocValuesRangeQueryTests.java => TestBinaryDocValuesRangeQuery.java} (99%) rename server/src/test/java/org/apache/lucene/queries/{BlendedTermQueryTests.java => TestBlendedTermQuery.java} (99%) rename server/src/test/java/org/apache/lucene/queries/{DoubleRandomBinaryDocValuesRangeQueryTests.java => TestDoubleRandomBinaryDocValuesRangeQuery.java} (96%) rename server/src/test/java/org/apache/lucene/queries/{FloatRandomBinaryDocValuesRangeQueryTests.java => TestFloatRandomBinaryDocValuesRangeQuery.java} (96%) rename server/src/test/java/org/apache/lucene/queries/{InetAddressRandomBinaryDocValuesRangeQueryTests.java => TestInetAddressRandomBinaryDocValuesRangeQuery.java} (97%) rename server/src/test/java/org/apache/lucene/queries/{IntegerRandomBinaryDocValuesRangeQueryTests.java => TestIntegerRandomBinaryDocValuesRangeQuery.java} (96%) rename server/src/test/java/org/apache/lucene/queries/{LongRandomBinaryDocValuesRangeQueryTests.java => TestLongRandomBinaryDocValuesRangeQuery.java} (96%) rename server/src/test/java/org/apache/lucene/queries/{MinDocQueryTests.java => TestMinDocQuery.java} (97%) rename server/src/test/java/org/apache/lucene/queries/{SearchAfterSortedDocQueryTests.java => TestSearchAfterSortedDocQuery.java} (98%) rename server/src/test/java/org/apache/lucene/queries/{SpanMatchNoDocsQueryTests.java => TestSpanMatchNoDocsQuery.java} (98%) rename server/src/test/java/org/apache/lucene/search/{XCombinedFieldQueryTests.java => TestXCombinedFieldQuery.java} (99%) rename server/src/test/java/org/apache/lucene/search/uhighlight/{BoundedBreakIteratorScannerTests.java => TestBoundedBreakIteratorScanner.java} (98%) rename server/src/test/java/org/apache/lucene/search/uhighlight/{CustomPassageFormatterTests.java => TestCustomPassageFormatter.java} (98%) rename server/src/test/java/org/apache/lucene/search/uhighlight/{CustomUnifiedHighlighterTests.java => TestCustomUnifiedHighlighter.java} (99%) rename server/src/test/java/org/apache/lucene/util/{CombinedBitSetTests.java => TestCombinedBitSet.java} (98%) diff --git a/server/src/test/java/org/apache/lucene/analysis/miscellaneous/DeDuplicatingTokenFilterTests.java b/server/src/test/java/org/apache/lucene/analysis/miscellaneous/TestDeDuplicatingTokenFilter.java similarity index 98% rename from server/src/test/java/org/apache/lucene/analysis/miscellaneous/DeDuplicatingTokenFilterTests.java rename to server/src/test/java/org/apache/lucene/analysis/miscellaneous/TestDeDuplicatingTokenFilter.java index 5b44c260cab97..71ba08df3d3fb 100644 --- a/server/src/test/java/org/apache/lucene/analysis/miscellaneous/DeDuplicatingTokenFilterTests.java +++ b/server/src/test/java/org/apache/lucene/analysis/miscellaneous/TestDeDuplicatingTokenFilter.java @@ -19,7 +19,7 @@ import static org.hamcrest.Matchers.equalTo; -public class DeDuplicatingTokenFilterTests extends ESTestCase { +public class TestDeDuplicatingTokenFilter extends ESTestCase { public void testSimple() throws IOException { DuplicateByteSequenceSpotter bytesDeDuper = new DuplicateByteSequenceSpotter(); Analyzer analyzer = new Analyzer() { diff --git a/server/src/test/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilterTests.java b/server/src/test/java/org/apache/lucene/analysis/miscellaneous/TestTruncateTokenFilter.java similarity index 97% rename from server/src/test/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilterTests.java rename to server/src/test/java/org/apache/lucene/analysis/miscellaneous/TestTruncateTokenFilter.java index 91732345fd311..f0b4f9c475531 100644 --- a/server/src/test/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilterTests.java +++ b/server/src/test/java/org/apache/lucene/analysis/miscellaneous/TestTruncateTokenFilter.java @@ -19,7 +19,7 @@ import static org.hamcrest.Matchers.equalTo; -public class TruncateTokenFilterTests extends ESTestCase { +public class TestTruncateTokenFilter extends ESTestCase { public void testSimple() throws IOException { Analyzer analyzer = new Analyzer() { @Override diff --git a/server/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorTests.java b/server/src/test/java/org/apache/lucene/grouping/TestCollapsingTopDocsCollector.java similarity index 99% rename from server/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorTests.java rename to server/src/test/java/org/apache/lucene/grouping/TestCollapsingTopDocsCollector.java index 13a9da4727820..e1e86fab2170b 100644 --- a/server/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorTests.java +++ b/server/src/test/java/org/apache/lucene/grouping/TestCollapsingTopDocsCollector.java @@ -49,7 +49,7 @@ import java.util.Set; -public class CollapsingTopDocsCollectorTests extends ESTestCase { +public class TestCollapsingTopDocsCollector extends ESTestCase { private static class SegmentSearcher extends IndexSearcher { private final List ctx; diff --git a/server/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorSearchAfterTests.java b/server/src/test/java/org/apache/lucene/grouping/TestCollapsingTopDocsCollectorSearchAfter.java similarity index 98% rename from server/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorSearchAfterTests.java rename to server/src/test/java/org/apache/lucene/grouping/TestCollapsingTopDocsCollectorSearchAfter.java index 347b99c332171..f4e36c39314dd 100644 --- a/server/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorSearchAfterTests.java +++ b/server/src/test/java/org/apache/lucene/grouping/TestCollapsingTopDocsCollectorSearchAfter.java @@ -40,10 +40,10 @@ import java.util.Set; /** - * This test is adapted from {@link CollapsingTopDocsCollectorTests} with + * This test is adapted from {@link TestCollapsingTopDocsCollector} with * modifications to test {@link FieldDoc after} parameter. */ -public class CollapsingTopDocsCollectorSearchAfterTests extends ESTestCase { +public class TestCollapsingTopDocsCollectorSearchAfter extends ESTestCase { interface CollapsingDocValuesProducer> { T randomGroup(int maxGroup); void add(Document doc, T value); diff --git a/server/src/test/java/org/apache/lucene/index/LazySoftDeletesDirectoryReaderWrapperTests.java b/server/src/test/java/org/apache/lucene/index/TestLazySoftDeletesDirectoryReaderWrapper.java similarity index 98% rename from server/src/test/java/org/apache/lucene/index/LazySoftDeletesDirectoryReaderWrapperTests.java rename to server/src/test/java/org/apache/lucene/index/TestLazySoftDeletesDirectoryReaderWrapper.java index 59dbc08f72084..14b9665425cf9 100644 --- a/server/src/test/java/org/apache/lucene/index/LazySoftDeletesDirectoryReaderWrapperTests.java +++ b/server/src/test/java/org/apache/lucene/index/TestLazySoftDeletesDirectoryReaderWrapper.java @@ -24,7 +24,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; -public class LazySoftDeletesDirectoryReaderWrapperTests extends LuceneTestCase { +public class TestLazySoftDeletesDirectoryReaderWrapper extends LuceneTestCase { public void testDropFullyDeletedSegments() throws IOException { IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); diff --git a/server/src/test/java/org/apache/lucene/index/ShuffleForcedMergePolicyTests.java b/server/src/test/java/org/apache/lucene/index/TestShuffleForcedMergePolicy.java similarity index 97% rename from server/src/test/java/org/apache/lucene/index/ShuffleForcedMergePolicyTests.java rename to server/src/test/java/org/apache/lucene/index/TestShuffleForcedMergePolicy.java index ec1d03394fe25..c6df138debb6e 100644 --- a/server/src/test/java/org/apache/lucene/index/ShuffleForcedMergePolicyTests.java +++ b/server/src/test/java/org/apache/lucene/index/TestShuffleForcedMergePolicy.java @@ -22,7 +22,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -public class ShuffleForcedMergePolicyTests extends BaseMergePolicyTestCase { +public class TestShuffleForcedMergePolicy extends BaseMergePolicyTestCase { public void testDiagnostics() throws IOException { try (Directory dir = newDirectory()) { IndexWriterConfig iwc = newIndexWriterConfig(); diff --git a/server/src/test/java/org/apache/lucene/queries/BinaryDocValuesRangeQueryTests.java b/server/src/test/java/org/apache/lucene/queries/TestBinaryDocValuesRangeQuery.java similarity index 99% rename from server/src/test/java/org/apache/lucene/queries/BinaryDocValuesRangeQueryTests.java rename to server/src/test/java/org/apache/lucene/queries/TestBinaryDocValuesRangeQuery.java index e799782e6cd8e..6ac22aa1bcc58 100644 --- a/server/src/test/java/org/apache/lucene/queries/BinaryDocValuesRangeQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/TestBinaryDocValuesRangeQuery.java @@ -27,7 +27,7 @@ import static org.apache.lucene.queries.BinaryDocValuesRangeQuery.QueryType.INTERSECTS; import static org.apache.lucene.queries.BinaryDocValuesRangeQuery.QueryType.WITHIN; -public class BinaryDocValuesRangeQueryTests extends ESTestCase { +public class TestBinaryDocValuesRangeQuery extends ESTestCase { public void testBasics() throws Exception { String fieldName = "long_field"; diff --git a/server/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java b/server/src/test/java/org/apache/lucene/queries/TestBlendedTermQuery.java similarity index 99% rename from server/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java rename to server/src/test/java/org/apache/lucene/queries/TestBlendedTermQuery.java index 815c86d8ab476..76f7cd4e5f805 100644 --- a/server/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/TestBlendedTermQuery.java @@ -46,7 +46,7 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.instanceOf; -public class BlendedTermQueryTests extends ESTestCase { +public class TestBlendedTermQuery extends ESTestCase { public void testDismaxQuery() throws IOException { Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); diff --git a/server/src/test/java/org/apache/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java b/server/src/test/java/org/apache/lucene/queries/TestDoubleRandomBinaryDocValuesRangeQuery.java similarity index 96% rename from server/src/test/java/org/apache/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java rename to server/src/test/java/org/apache/lucene/queries/TestDoubleRandomBinaryDocValuesRangeQuery.java index ebecf6d36feb7..b54c55904f7f3 100644 --- a/server/src/test/java/org/apache/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/TestDoubleRandomBinaryDocValuesRangeQuery.java @@ -9,7 +9,7 @@ import org.elasticsearch.index.mapper.RangeType; -public class DoubleRandomBinaryDocValuesRangeQueryTests extends BaseRandomBinaryDocValuesRangeQueryTestCase { +public class TestDoubleRandomBinaryDocValuesRangeQuery extends BaseRandomBinaryDocValuesRangeQueryTestCase { @Override protected String fieldName() { diff --git a/server/src/test/java/org/apache/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java b/server/src/test/java/org/apache/lucene/queries/TestFloatRandomBinaryDocValuesRangeQuery.java similarity index 96% rename from server/src/test/java/org/apache/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java rename to server/src/test/java/org/apache/lucene/queries/TestFloatRandomBinaryDocValuesRangeQuery.java index 681d5434b5618..671ba8702dc98 100644 --- a/server/src/test/java/org/apache/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/TestFloatRandomBinaryDocValuesRangeQuery.java @@ -9,7 +9,7 @@ import org.elasticsearch.index.mapper.RangeType; -public class FloatRandomBinaryDocValuesRangeQueryTests extends BaseRandomBinaryDocValuesRangeQueryTestCase { +public class TestFloatRandomBinaryDocValuesRangeQuery extends BaseRandomBinaryDocValuesRangeQueryTestCase { @Override protected String fieldName() { diff --git a/server/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java b/server/src/test/java/org/apache/lucene/queries/TestInetAddressRandomBinaryDocValuesRangeQuery.java similarity index 97% rename from server/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java rename to server/src/test/java/org/apache/lucene/queries/TestInetAddressRandomBinaryDocValuesRangeQuery.java index cccdd89398c77..80d523a62991a 100644 --- a/server/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/TestInetAddressRandomBinaryDocValuesRangeQuery.java @@ -14,7 +14,7 @@ import java.net.UnknownHostException; import java.util.Arrays; -public class InetAddressRandomBinaryDocValuesRangeQueryTests extends BaseRandomBinaryDocValuesRangeQueryTestCase { +public class TestInetAddressRandomBinaryDocValuesRangeQuery extends BaseRandomBinaryDocValuesRangeQueryTestCase { @Override protected String fieldName() { diff --git a/server/src/test/java/org/apache/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java b/server/src/test/java/org/apache/lucene/queries/TestIntegerRandomBinaryDocValuesRangeQuery.java similarity index 96% rename from server/src/test/java/org/apache/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java rename to server/src/test/java/org/apache/lucene/queries/TestIntegerRandomBinaryDocValuesRangeQuery.java index 72fd223c52501..5af1c22844498 100644 --- a/server/src/test/java/org/apache/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/TestIntegerRandomBinaryDocValuesRangeQuery.java @@ -10,7 +10,7 @@ import org.apache.lucene.util.TestUtil; import org.elasticsearch.index.mapper.RangeType; -public class IntegerRandomBinaryDocValuesRangeQueryTests extends BaseRandomBinaryDocValuesRangeQueryTestCase { +public class TestIntegerRandomBinaryDocValuesRangeQuery extends BaseRandomBinaryDocValuesRangeQueryTestCase { @Override protected String fieldName() { diff --git a/server/src/test/java/org/apache/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java b/server/src/test/java/org/apache/lucene/queries/TestLongRandomBinaryDocValuesRangeQuery.java similarity index 96% rename from server/src/test/java/org/apache/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java rename to server/src/test/java/org/apache/lucene/queries/TestLongRandomBinaryDocValuesRangeQuery.java index 2f75d49bd82a1..a0225566c1efb 100644 --- a/server/src/test/java/org/apache/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/TestLongRandomBinaryDocValuesRangeQuery.java @@ -10,7 +10,7 @@ import org.apache.lucene.util.TestUtil; import org.elasticsearch.index.mapper.RangeType; -public class LongRandomBinaryDocValuesRangeQueryTests extends BaseRandomBinaryDocValuesRangeQueryTestCase { +public class TestLongRandomBinaryDocValuesRangeQuery extends BaseRandomBinaryDocValuesRangeQueryTestCase { @Override protected String fieldName() { diff --git a/server/src/test/java/org/apache/lucene/queries/MinDocQueryTests.java b/server/src/test/java/org/apache/lucene/queries/TestMinDocQuery.java similarity index 97% rename from server/src/test/java/org/apache/lucene/queries/MinDocQueryTests.java rename to server/src/test/java/org/apache/lucene/queries/TestMinDocQuery.java index c3290a4dc66b7..eafba190fd4d3 100644 --- a/server/src/test/java/org/apache/lucene/queries/MinDocQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/TestMinDocQuery.java @@ -20,7 +20,7 @@ import java.io.IOException; -public class MinDocQueryTests extends ESTestCase { +public class TestMinDocQuery extends ESTestCase { public void testBasics() { MinDocQuery query1 = new MinDocQuery(42); diff --git a/server/src/test/java/org/apache/lucene/queries/SearchAfterSortedDocQueryTests.java b/server/src/test/java/org/apache/lucene/queries/TestSearchAfterSortedDocQuery.java similarity index 98% rename from server/src/test/java/org/apache/lucene/queries/SearchAfterSortedDocQueryTests.java rename to server/src/test/java/org/apache/lucene/queries/TestSearchAfterSortedDocQuery.java index 5859f3fa35d73..eb6b757b13a4b 100644 --- a/server/src/test/java/org/apache/lucene/queries/SearchAfterSortedDocQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/TestSearchAfterSortedDocQuery.java @@ -35,7 +35,7 @@ import static org.hamcrest.Matchers.equalTo; -public class SearchAfterSortedDocQueryTests extends ESTestCase { +public class TestSearchAfterSortedDocQuery extends ESTestCase { public void testBasics() { Sort sort1 = new Sort( diff --git a/server/src/test/java/org/apache/lucene/queries/SpanMatchNoDocsQueryTests.java b/server/src/test/java/org/apache/lucene/queries/TestSpanMatchNoDocsQuery.java similarity index 98% rename from server/src/test/java/org/apache/lucene/queries/SpanMatchNoDocsQueryTests.java rename to server/src/test/java/org/apache/lucene/queries/TestSpanMatchNoDocsQuery.java index 08157dcc93fac..3005874d624a0 100644 --- a/server/src/test/java/org/apache/lucene/queries/SpanMatchNoDocsQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/TestSpanMatchNoDocsQuery.java @@ -29,7 +29,7 @@ import java.io.IOException; -public class SpanMatchNoDocsQueryTests extends ESTestCase { +public class TestSpanMatchNoDocsQuery extends ESTestCase { public void testSimple() throws Exception { SpanMatchNoDocsQuery query = new SpanMatchNoDocsQuery("field", "a good reason"); assertEquals(query.toString(), "SpanMatchNoDocsQuery(\"a good reason\")"); diff --git a/server/src/test/java/org/apache/lucene/search/XCombinedFieldQueryTests.java b/server/src/test/java/org/apache/lucene/search/TestXCombinedFieldQuery.java similarity index 99% rename from server/src/test/java/org/apache/lucene/search/XCombinedFieldQueryTests.java rename to server/src/test/java/org/apache/lucene/search/TestXCombinedFieldQuery.java index 7b98109c9984e..c08b678e375b0 100644 --- a/server/src/test/java/org/apache/lucene/search/XCombinedFieldQueryTests.java +++ b/server/src/test/java/org/apache/lucene/search/TestXCombinedFieldQuery.java @@ -49,7 +49,7 @@ * TODO remove once LUCENE 9999 is fixed and integrated and we remove our copy of the query * */ -public class XCombinedFieldQueryTests extends LuceneTestCase { +public class TestXCombinedFieldQuery extends LuceneTestCase { public void testRewrite() throws IOException { IndexReader reader = new MultiReader(); diff --git a/server/src/test/java/org/apache/lucene/search/uhighlight/BoundedBreakIteratorScannerTests.java b/server/src/test/java/org/apache/lucene/search/uhighlight/TestBoundedBreakIteratorScanner.java similarity index 98% rename from server/src/test/java/org/apache/lucene/search/uhighlight/BoundedBreakIteratorScannerTests.java rename to server/src/test/java/org/apache/lucene/search/uhighlight/TestBoundedBreakIteratorScanner.java index 26cc2c6a5a01f..6e700e5e5e861 100644 --- a/server/src/test/java/org/apache/lucene/search/uhighlight/BoundedBreakIteratorScannerTests.java +++ b/server/src/test/java/org/apache/lucene/search/uhighlight/TestBoundedBreakIteratorScanner.java @@ -20,7 +20,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; -public class BoundedBreakIteratorScannerTests extends ESTestCase { +public class TestBoundedBreakIteratorScanner extends ESTestCase { private static final String[] WORD_BOUNDARIES = new String[] { " ", " ", "\t", "#", "\n" }; private static final String[] SENTENCE_BOUNDARIES = diff --git a/server/src/test/java/org/apache/lucene/search/uhighlight/CustomPassageFormatterTests.java b/server/src/test/java/org/apache/lucene/search/uhighlight/TestCustomPassageFormatter.java similarity index 98% rename from server/src/test/java/org/apache/lucene/search/uhighlight/CustomPassageFormatterTests.java rename to server/src/test/java/org/apache/lucene/search/uhighlight/TestCustomPassageFormatter.java index 6050c71638770..30ec6c11391e6 100644 --- a/server/src/test/java/org/apache/lucene/search/uhighlight/CustomPassageFormatterTests.java +++ b/server/src/test/java/org/apache/lucene/search/uhighlight/TestCustomPassageFormatter.java @@ -17,7 +17,7 @@ import static org.hamcrest.CoreMatchers.notNullValue; -public class CustomPassageFormatterTests extends ESTestCase { +public class TestCustomPassageFormatter extends ESTestCase { public void testSimpleFormat() { String content = "This is a really cool highlighter. Unified highlighter gives nice snippets back. No matches here."; diff --git a/server/src/test/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java b/server/src/test/java/org/apache/lucene/search/uhighlight/TestCustomUnifiedHighlighter.java similarity index 99% rename from server/src/test/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java rename to server/src/test/java/org/apache/lucene/search/uhighlight/TestCustomUnifiedHighlighter.java index 16bc9792ffe48..9ed42fe490dee 100644 --- a/server/src/test/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java +++ b/server/src/test/java/org/apache/lucene/search/uhighlight/TestCustomUnifiedHighlighter.java @@ -43,7 +43,7 @@ import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery; import org.elasticsearch.test.ESTestCase; -public class CustomUnifiedHighlighterTests extends ESTestCase { +public class TestCustomUnifiedHighlighter extends ESTestCase { private void assertHighlightOneDoc(String fieldName, String[] inputs, Analyzer analyzer, Query query, Locale locale, BreakIterator breakIterator, diff --git a/server/src/test/java/org/apache/lucene/util/CombinedBitSetTests.java b/server/src/test/java/org/apache/lucene/util/TestCombinedBitSet.java similarity index 98% rename from server/src/test/java/org/apache/lucene/util/CombinedBitSetTests.java rename to server/src/test/java/org/apache/lucene/util/TestCombinedBitSet.java index 93f8e4b115cd9..b659ab7901807 100644 --- a/server/src/test/java/org/apache/lucene/util/CombinedBitSetTests.java +++ b/server/src/test/java/org/apache/lucene/util/TestCombinedBitSet.java @@ -11,7 +11,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.elasticsearch.test.ESTestCase; -public class CombinedBitSetTests extends ESTestCase { +public class TestCombinedBitSet extends ESTestCase { public void testEmpty() { for (float percent : new float[] {0f, 0.1f, 0.5f, 0.9f, 1f}) { testCase(randomIntBetween(1, 10000), 0f, percent); From 019482f305169bd7f12c744be7188a8dfc254ac8 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 2 Aug 2021 11:07:04 +0100 Subject: [PATCH 11/67] schema consistency --- .../index/fielddata/AbstractStringFieldDataTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java index 2dc9c9258e8d1..bae54a1d82461 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java @@ -203,7 +203,7 @@ protected void fillExtendedMvSet() throws Exception { writer.addDocument(d); d = new Document(); - d.add(new StringField("_id", "6", Field.Store.NO)); + addField(d, "_id", "6"); writer.addDocument(d); d = new Document(); From 4731b140d09fe69ea28963ccda64bd065dea1f75 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 2 Aug 2021 12:03:44 +0100 Subject: [PATCH 12/67] Translog + a few other places need to use CodecUtil.read/writeBEInt for BWC --- .../common/settings/KeyStoreWrapperTests.java | 13 +++++----- .../common/settings/KeyStoreWrapper.java | 26 +++++++++---------- .../gateway/MetadataStateFormat.java | 4 +-- .../index/translog/Checkpoint.java | 4 +-- .../query/QueryStringQueryBuilderTests.java | 12 ++++----- .../elasticsearch/index/store/StoreTests.java | 4 +-- .../translog/TranslogDeletionPolicyTests.java | 7 ++--- .../index/translog/TranslogTests.java | 8 +++--- .../indices/recovery/RecoveryStatusTests.java | 4 +-- .../ChecksumBlobContainerIndexInput.java | 4 +-- .../xpack/spatial/index/fielddata/Extent.java | 3 ++- .../fielddata/GeometryDocValueWriter.java | 5 ++-- 12 files changed, 48 insertions(+), 46 deletions(-) diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java index 02533be7fdbcc..dec41e967292f 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java @@ -40,7 +40,6 @@ import java.util.List; import java.util.Locale; import java.util.Set; - import javax.crypto.Cipher; import javax.crypto.CipherOutputStream; import javax.crypto.SecretKey; @@ -345,12 +344,12 @@ private void possiblyAlterEncryptedBytes( byte[] encryptedBytes, int truncEncryptedDataLength ) throws Exception { - indexOutput.writeInt(4 + salt.length + 4 + iv.length + 4 + encryptedBytes.length); - indexOutput.writeInt(salt.length); + CodecUtil.writeBEInt(indexOutput, 4 + salt.length + 4 + iv.length + 4 + encryptedBytes.length); + CodecUtil.writeBEInt(indexOutput, salt.length); indexOutput.writeBytes(salt, salt.length); - indexOutput.writeInt(iv.length); + CodecUtil.writeBEInt(indexOutput, iv.length); indexOutput.writeBytes(iv, iv.length); - indexOutput.writeInt(encryptedBytes.length - truncEncryptedDataLength); + CodecUtil.writeBEInt(indexOutput, encryptedBytes.length - truncEncryptedDataLength); indexOutput.writeBytes(encryptedBytes, encryptedBytes.length); } @@ -399,7 +398,7 @@ public void testBackcompatV1() throws Exception { ByteArrayOutputStream keystoreBytesStream = new ByteArrayOutputStream(); keystore.store(keystoreBytesStream, new char[0]); byte[] keystoreBytes = keystoreBytesStream.toByteArray(); - output.writeInt(keystoreBytes.length); + CodecUtil.writeBEInt(output, keystoreBytes.length); output.writeBytes(keystoreBytes, keystoreBytes.length); CodecUtil.writeFooter(output); } @@ -450,7 +449,7 @@ public void testBackcompatV2() throws Exception { ByteArrayOutputStream keystoreBytesStream = new ByteArrayOutputStream(); keystore.store(keystoreBytesStream, new char[0]); byte[] keystoreBytes = keystoreBytesStream.toByteArray(); - output.writeInt(keystoreBytes.length); + CodecUtil.writeBEInt(output, keystoreBytes.length); output.writeBytes(keystoreBytes, keystoreBytes.length); CodecUtil.writeFooter(output); } diff --git a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java index 83a959ce97733..c1b9770f034b1 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java +++ b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java @@ -25,15 +25,6 @@ import org.elasticsearch.common.Randomness; import org.elasticsearch.common.hash.MessageDigests; -import javax.crypto.AEADBadTagException; -import javax.crypto.Cipher; -import javax.crypto.CipherInputStream; -import javax.crypto.CipherOutputStream; -import javax.crypto.SecretKey; -import javax.crypto.SecretKeyFactory; -import javax.crypto.spec.GCMParameterSpec; -import javax.crypto.spec.PBEKeySpec; -import javax.crypto.spec.SecretKeySpec; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; @@ -63,6 +54,15 @@ import java.util.Map; import java.util.Set; import java.util.regex.Pattern; +import javax.crypto.AEADBadTagException; +import javax.crypto.Cipher; +import javax.crypto.CipherInputStream; +import javax.crypto.CipherOutputStream; +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.GCMParameterSpec; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.SecretKeySpec; /** * A disk based container for sensitive settings in Elasticsearch. @@ -533,13 +533,13 @@ public synchronized void save(Path configDir, char[] password, boolean preserveP byte[] encryptedBytes = encrypt(password, salt, iv); // size of data block - output.writeInt(4 + salt.length + 4 + iv.length + 4 + encryptedBytes.length); + CodecUtil.writeBEInt(output, 4 + salt.length + 4 + iv.length + 4 + encryptedBytes.length); - output.writeInt(salt.length); + CodecUtil.writeBEInt(output, salt.length); output.writeBytes(salt, salt.length); - output.writeInt(iv.length); + CodecUtil.writeBEInt(output, iv.length); output.writeBytes(iv, iv.length); - output.writeInt(encryptedBytes.length); + CodecUtil.writeBEInt(output, encryptedBytes.length); output.writeBytes(encryptedBytes, encryptedBytes.length); CodecUtil.writeFooter(output); diff --git a/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java b/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java index e4133a15b5b4d..9ee4138c12648 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java @@ -93,7 +93,7 @@ private void writeStateToFirstLocation(final T state, Path stateLocation, Direct deleteFileIfExists(stateLocation, stateDir, tmpFileName); try (IndexOutput out = stateDir.createOutput(tmpFileName, IOContext.DEFAULT)) { CodecUtil.writeHeader(out, STATE_FILE_CODEC, STATE_FILE_VERSION); - out.writeInt(FORMAT.index()); + CodecUtil.writeBEInt(out, FORMAT.index()); try (XContentBuilder builder = newXContentBuilder(FORMAT, new IndexOutputOutputStream(out) { @Override public void close() { @@ -271,7 +271,7 @@ public final T read(NamedXContentRegistry namedXContentRegistry, Path file) thro // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here. CodecUtil.checksumEntireFile(indexInput); CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, MIN_COMPATIBLE_STATE_FILE_VERSION, STATE_FILE_VERSION); - final XContentType xContentType = XContentType.values()[indexInput.readInt()]; + final XContentType xContentType = XContentType.values()[CodecUtil.readBEInt(indexInput)]; if (xContentType != FORMAT) { throw new IllegalStateException("expected state in " + file + " to be " + FORMAT + " format but was " + xContentType); } diff --git a/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java b/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java index 4d6865976492b..3aea5fb355df0 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java +++ b/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java @@ -86,7 +86,7 @@ final class Checkpoint { private void write(DataOutput out) throws IOException { out.writeLong(offset); - out.writeInt(numOps); + CodecUtil.writeBEInt(out, numOps); out.writeLong(generation); out.writeLong(minSeqNo); out.writeLong(maxSeqNo); @@ -116,7 +116,7 @@ static Checkpoint emptyTranslogCheckpoint(final long offset, final long generati static Checkpoint readCheckpointV3(final DataInput in) throws IOException { final long offset = in.readLong(); - final int numOps = in.readInt(); + final int numOps = CodecUtil.readBEInt(in); final long generation = in.readLong(); final long minSeqNo = in.readLong(); final long maxSeqNo = in.readLong(); diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java index 7b93e309cda14..a740f197a1415 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java @@ -492,7 +492,7 @@ public void testToQueryMultipleFieldsBooleanQuery() throws Exception { Query expected = new DisjunctionMaxQuery(List.of( new TermQuery(new Term(TEXT_FIELD_NAME, "test")), new TermQuery(new Term(KEYWORD_FIELD_NAME, "test"))), - 1 + 0 ); assertEquals(expected, query); } @@ -503,7 +503,7 @@ public void testToQueryMultipleFieldsDisMaxQuery() throws Exception { Query expected = new DisjunctionMaxQuery(List.of( new TermQuery(new Term(TEXT_FIELD_NAME, "test")), new TermQuery(new Term(KEYWORD_FIELD_NAME, "test"))), - 1 + 0 ); assertEquals(expected, query); } @@ -513,7 +513,7 @@ public void testToQueryFieldsWildcard() throws Exception { Query expected = new DisjunctionMaxQuery(List.of( new TermQuery(new Term(TEXT_FIELD_NAME, "test")), new TermQuery(new Term(KEYWORD_FIELD_NAME, "test"))), - 1 + 0 ); assertEquals(expected, query); } @@ -539,7 +539,7 @@ public void testToQueryDisMaxQuery() throws Exception { Query expected = new DisjunctionMaxQuery(List.of( new BoostQuery(new TermQuery(new Term(TEXT_FIELD_NAME, "test")), 2.2f), new TermQuery(new Term(KEYWORD_FIELD_NAME, "test"))), - 1 + 0 ); assertEquals(expected, query); } @@ -756,7 +756,7 @@ public void testToQueryRegExpQueryTooComplex() throws Exception { TooComplexToDeterminizeException e = expectThrows(TooComplexToDeterminizeException.class, () -> queryBuilder.toQuery(createSearchExecutionContext())); assertThat(e.getMessage(), containsString("Determinizing [ac]*")); - assertThat(e.getMessage(), containsString("would result in more than 10000 states")); + assertThat(e.getMessage(), containsString("would require more than 10000 effort")); } /** @@ -778,7 +778,7 @@ public void testToQueryRegExpQueryMaxDeterminizedStatesParsing() throws Exceptio TooComplexToDeterminizeException e = expectThrows(TooComplexToDeterminizeException.class, () -> queryBuilder.toQuery(createSearchExecutionContext())); assertThat(e.getMessage(), containsString("Determinizing [ac]*")); - assertThat(e.getMessage(), containsString("would result in more than 10 states")); + assertThat(e.getMessage(), containsString("would require more than 10 effort")); } public void testToQueryFuzzyQueryAutoFuziness() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/index/store/StoreTests.java b/server/src/test/java/org/elasticsearch/index/store/StoreTests.java index 2d2509e208f8f..01e0e4ca187fa 100644 --- a/server/src/test/java/org/elasticsearch/index/store/StoreTests.java +++ b/server/src/test/java/org/elasticsearch/index/store/StoreTests.java @@ -209,8 +209,8 @@ public void testChecksumCorrupted() throws IOException { BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024)); output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length); } - output.writeInt(CodecUtil.FOOTER_MAGIC); - output.writeInt(0); + CodecUtil.writeBEInt(output, CodecUtil.FOOTER_MAGIC); + CodecUtil.writeBEInt(output, 0); String checksum = Store.digestToString(output.getChecksum()); output.writeLong(output.getChecksum() + 1); // write a wrong checksum to the file output.close(); diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java index 27f320300bab4..750fa582e44a3 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java @@ -8,13 +8,14 @@ package org.elasticsearch.index.translog; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.store.ByteArrayDataOutput; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.ReleasableBytesReference; -import org.elasticsearch.core.Tuple; -import org.elasticsearch.core.Releasable; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Tuple; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; @@ -86,7 +87,7 @@ private Tuple, TranslogWriter> createReadersAndWriter() thr for (int ops = randomIntBetween(0, 20); ops > 0; ops--) { out.reset(bytes); - out.writeInt(ops); + CodecUtil.writeBEInt(out, ops); writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), ops); } } diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index 2831799d1fa7c..ec6acb05cd49d 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -1260,7 +1260,7 @@ public void testTranslogWriter() throws IOException { for (int i = 0; i < numOps; i++) { byte[] bytes = new byte[4]; ByteArrayDataOutput out = new ByteArrayDataOutput(bytes); - out.writeInt(i); + CodecUtil.writeBEInt(out, i); long seqNo; do { seqNo = opsHaveValidSequenceNumbers ? randomNonNegativeLong() : SequenceNumbers.UNASSIGNED_SEQ_NO; @@ -1292,7 +1292,7 @@ public void testTranslogWriter() throws IOException { byte[] bytes = new byte[4]; ByteArrayDataOutput out = new ByteArrayDataOutput(bytes); - out.writeInt(2048); + CodecUtil.writeBEInt(out, 2048); writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), randomNonNegativeLong()); if (reader instanceof TranslogReader) { @@ -1470,7 +1470,7 @@ ChannelFactory getChannelFactory() { byte[] bytes = new byte[4]; ByteArrayDataOutput out = new ByteArrayDataOutput(new byte[4]); - out.writeInt(1); + CodecUtil.writeBEInt(out, 1); writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), 1); assertThat(persistedSeqNos, empty()); startBlocking.set(true); @@ -1503,7 +1503,7 @@ public void testCloseIntoReader() throws IOException { final byte[] bytes = new byte[4]; final ByteArrayDataOutput out = new ByteArrayDataOutput(bytes); out.reset(bytes); - out.writeInt(i); + CodecUtil.writeBEInt(out, i); writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), randomNonNegativeLong()); } writer.sync(); diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java index 87dd75998319c..57c70e64afbca 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java @@ -33,10 +33,10 @@ public void testRenameTempFiles() throws IOException { new StoreFileMetadata("foo.bar", 8 + CodecUtil.footerLength(), "9z51nw", MIN_SUPPORTED_LUCENE_VERSION.toString()), indexShard.store())) { - indexOutput.writeInt(1); + CodecUtil.writeBEInt(indexOutput, 1); IndexOutput openIndexOutput = multiFileWriter.getOpenIndexOutput("foo.bar"); assertSame(openIndexOutput, indexOutput); - openIndexOutput.writeInt(1); + CodecUtil.writeBEInt(openIndexOutput, 1); CodecUtil.writeFooter(indexOutput); } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/ChecksumBlobContainerIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/ChecksumBlobContainerIndexInput.java index aac42c81334e6..4aa5da3c701cf 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/ChecksumBlobContainerIndexInput.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/ChecksumBlobContainerIndexInput.java @@ -140,8 +140,8 @@ public static byte[] checksumToBytesArray(String checksum) throws IOException { assert result.length >= Integer.BYTES + Integer.BYTES + Long.BYTES; // ensure that nobody changed the file format under us final ByteArrayDataOutput output = new ByteArrayDataOutput(result); // reverse CodecUtil.writeFooter() - output.writeInt(CodecUtil.FOOTER_MAGIC); - output.writeInt(0); + CodecUtil.writeBEInt(output, CodecUtil.FOOTER_MAGIC); + CodecUtil.writeBEInt(output, 0); output.writeLong(Long.parseLong(checksum, Character.MAX_RADIX)); assert output.getPosition() == result.length; return result; diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/Extent.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/Extent.java index 709aec621ed33..7e64dc37a3a56 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/Extent.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/Extent.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.spatial.index.fielddata; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.ByteBuffersDataOutput; @@ -134,7 +135,7 @@ static void readFromCompressed(ByteArrayDataInput input, Extent extent) { } void writeCompressed(ByteBuffersDataOutput output) throws IOException { - output.writeInt(this.top); + CodecUtil.writeBEInt(output, this.top); output.writeVLong((long) this.top - this.bottom); byte type; if (this.negLeft == Integer.MAX_VALUE && this.negRight == Integer.MIN_VALUE) { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeometryDocValueWriter.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeometryDocValueWriter.java index d44f99c3dfd3b..9b916960bdc91 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeometryDocValueWriter.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeometryDocValueWriter.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.spatial.index.fielddata; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.IndexableField; import org.apache.lucene.store.ByteBuffersDataOutput; import org.apache.lucene.util.BytesRef; @@ -29,8 +30,8 @@ public static BytesRef write(List fields, CentroidCalculator centroidCalculator) throws IOException { final ByteBuffersDataOutput out = new ByteBuffersDataOutput(); // normalization may be required due to floating point precision errors - out.writeInt(coordinateEncoder.encodeX(coordinateEncoder.normalizeX(centroidCalculator.getX()))); - out.writeInt(coordinateEncoder.encodeY(coordinateEncoder.normalizeY(centroidCalculator.getY()))); + CodecUtil.writeBEInt(out, coordinateEncoder.encodeX(coordinateEncoder.normalizeX(centroidCalculator.getX()))); + CodecUtil.writeBEInt(out, coordinateEncoder.encodeY(coordinateEncoder.normalizeY(centroidCalculator.getY()))); centroidCalculator.getDimensionalShapeType().writeTo(out); out.writeVLong(Double.doubleToLongBits(centroidCalculator.sumWeight())); TriangleTreeWriter.writeTo(out, fields); From d139867a0a7f638f0ffffdbb1b2db87ff786486f Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 2 Aug 2021 15:19:01 +0100 Subject: [PATCH 13/67] TopDocs.merge doesn't set shardIndex; new filters; codecs; math is hard; spi --- .../apache/lucene/util/SPIClassIterator.java | 168 ++++++++++++++++++ .../action/search/SearchPhaseController.java | 17 +- .../elasticsearch/common/geo/GeoUtils.java | 2 +- .../elasticsearch/plugins/PluginsService.java | 16 +- .../aggregations/metrics/InternalTopHits.java | 6 + .../IndexDiskUsageAnalyzerTests.java | 4 +- .../segments/IndicesSegmentsRequestTests.java | 2 +- .../engine/CompletionStatsCacheTests.java | 4 +- .../index/engine/InternalEngineTests.java | 8 +- .../index/query/DisMaxQueryBuilderTests.java | 2 +- .../query/MultiMatchQueryBuilderTests.java | 4 +- .../fetch/subphase/FieldFetcherTests.java | 6 +- .../analysis/AnalysisFactoryTestCase.java | 2 + 13 files changed, 217 insertions(+), 24 deletions(-) create mode 100644 server/src/main/java/org/apache/lucene/util/SPIClassIterator.java diff --git a/server/src/main/java/org/apache/lucene/util/SPIClassIterator.java b/server/src/main/java/org/apache/lucene/util/SPIClassIterator.java new file mode 100644 index 0000000000000..5c2842df131da --- /dev/null +++ b/server/src/main/java/org/apache/lucene/util/SPIClassIterator.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.util; + +import java.io.IOException; +import java.io.InputStream; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Enumeration; +import java.util.Iterator; +import java.util.Locale; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.ServiceConfigurationError; + +/** + * Helper class for loading SPI classes from classpath (META-INF files). + * This is a light impl of {@link java.util.ServiceLoader} but is guaranteed to + * be bug-free regarding classpath order and does not instantiate or initialize + * the classes found. + * + * @lucene.internal + */ +public final class SPIClassIterator implements Iterator> { + private static final String META_INF_SERVICES = "META-INF/services/"; + + private final Class clazz; + private final ClassLoader loader; + private final Enumeration profilesEnum; + private Iterator linesIterator; + + /** Creates a new SPI iterator to lookup services of type {@code clazz} using + * the same {@link ClassLoader} as the argument. */ + public static SPIClassIterator get(Class clazz) { + return new SPIClassIterator<>(clazz, + Objects.requireNonNull(clazz.getClassLoader(), () -> clazz + " has no classloader.")); + } + + /** Creates a new SPI iterator to lookup services of type {@code clazz} using the given classloader. */ + public static SPIClassIterator get(Class clazz, ClassLoader loader) { + return new SPIClassIterator<>(clazz, loader); + } + + /** + * Utility method to check if some class loader is a (grand-)parent of or the same as another one. + * This means the child will be able to load all classes from the parent, too. + *

+ * If caller's codesource doesn't have enough permissions to do the check, {@code false} is returned + * (this is fine, because if we get a {@code SecurityException} it is for sure no parent). + */ + public static boolean isParentClassLoader(final ClassLoader parent, final ClassLoader child) { + try { + ClassLoader cl = child; + while (cl != null) { + if (cl == parent) { + return true; + } + cl = cl.getParent(); + } + return false; + } catch (SecurityException se) { + return false; + } + } + + private SPIClassIterator(Class clazz, ClassLoader loader) { + this.clazz = Objects.requireNonNull(clazz, "clazz"); + this.loader = Objects.requireNonNull(loader, "loader"); + try { + final String fullName = META_INF_SERVICES + clazz.getName(); + this.profilesEnum = loader.getResources(fullName); + } catch (IOException ioe) { + throw new ServiceConfigurationError("Error loading SPI profiles for type " + clazz.getName() + " from classpath", ioe); + } + this.linesIterator = Collections.emptySet().iterator(); + } + + private boolean loadNextProfile() { + ArrayList lines = null; + while (profilesEnum.hasMoreElements()) { + if (lines != null) { + lines.clear(); + } else { + lines = new ArrayList<>(); + } + final URL url = profilesEnum.nextElement(); + try { + final InputStream in = url.openStream(); + boolean success = false; + try { + final BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)); + String line; + while ((line = reader.readLine()) != null) { + final int pos = line.indexOf('#'); + if (pos >= 0) { + line = line.substring(0, pos); + } + line = line.trim(); + if (line.length() > 0) { + lines.add(line); + } + } + success = true; + } finally { + if (success) { + IOUtils.close(in); + } else { + IOUtils.closeWhileHandlingException(in); + } + } + } catch (IOException ioe) { + throw new ServiceConfigurationError("Error loading SPI class list from URL: " + url, ioe); + } + if (lines.isEmpty() == false) { + this.linesIterator = lines.iterator(); + return true; + } + } + return false; + } + + @Override + public boolean hasNext() { + return linesIterator.hasNext() || loadNextProfile(); + } + + @Override + public Class next() { + // hasNext() implicitely loads the next profile, so it is essential to call this here! + if (hasNext() == false) { + throw new NoSuchElementException(); + } + assert linesIterator.hasNext(); + final String c = linesIterator.next(); + try { + // don't initialize the class (pass false as 2nd parameter): + return Class.forName(c, false, loader).asSubclass(clazz); + } catch (ClassNotFoundException cnfe) { + throw new ServiceConfigurationError(String.format(Locale.ROOT, "An SPI class of type %s with classname %s does not exist, "+ + "please fix the file '%s%1$s' in your classpath.", clazz.getName(), c, META_INF_SERVICES)); + } + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + +} diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index 159a1eb2c408a..dab29bc0a3aa3 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -196,14 +196,29 @@ static TopDocs mergeTopDocs(Collection results, int topN, int from) { CollapseTopFieldDocs firstTopDocs = (CollapseTopFieldDocs) topDocs; final Sort sort = new Sort(firstTopDocs.fields); final CollapseTopFieldDocs[] shardTopDocs = results.toArray(new CollapseTopFieldDocs[numShards]); - mergedTopDocs = CollapseTopFieldDocs.merge(sort, from, topN, shardTopDocs); + for (int i = 0; i < shardTopDocs.length; i++) { + for (ScoreDoc sd : shardTopDocs[i].scoreDocs) { + sd.shardIndex = i; + } + } + mergedTopDocs = CollapseTopFieldDocs.merge(sort, from, topN, shardTopDocs, false); } else if (topDocs instanceof TopFieldDocs) { TopFieldDocs firstTopDocs = (TopFieldDocs) topDocs; final Sort sort = new Sort(firstTopDocs.fields); final TopFieldDocs[] shardTopDocs = results.toArray(new TopFieldDocs[numShards]); + for (int i = 0; i < shardTopDocs.length; i++) { + for (ScoreDoc sd : shardTopDocs[i].scoreDocs) { + sd.shardIndex = i; + } + } mergedTopDocs = TopDocs.merge(sort, from, topN, shardTopDocs); } else { final TopDocs[] shardTopDocs = results.toArray(new TopDocs[numShards]); + for (int i = 0; i < shardTopDocs.length; i++) { + for (ScoreDoc sd : shardTopDocs[i].scoreDocs) { + sd.shardIndex = i; + } + } mergedTopDocs = TopDocs.merge(from, topN, shardTopDocs); } return mergedTopDocs; diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java b/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java index cda0f41e71962..661e78bf682ff 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java @@ -599,7 +599,7 @@ public static double arcDistance(double lat1, double lon1, double lat2, double l * 4 decimal degrees */ public static double planeDistance(double lat1, double lon1, double lat2, double lon2) { - double x = Math.toRadians(lon2 - lon1) * Math.cos((lat2 + lat1) / Math.toRadians(2.0)); + double x = Math.toRadians(lon2 - lon1) * Math.cos(Math.toRadians((lat2 + lat1) / 2.0)); double y = Math.toRadians(lat2 - lat1); return Math.sqrt(x * x + y * y) * EARTH_MEAN_RADIUS; } diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java index 9a3e1f55f307a..8dd8f13c8fab7 100644 --- a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -16,18 +16,19 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; +import org.apache.lucene.util.SPIClassIterator; import org.elasticsearch.Build; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules; -import org.elasticsearch.jdk.JarHell; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.Tuple; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexModule; +import org.elasticsearch.jdk.JarHell; import org.elasticsearch.node.ReportingService; import org.elasticsearch.threadpool.ExecutorBuilder; @@ -51,7 +52,6 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; -import java.util.ServiceLoader; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -469,12 +469,12 @@ public List loadExtensions(Class extensionPointType) { } private static List createExtensions(Class extensionPointType, Plugin plugin) { + SPIClassIterator classIterator = SPIClassIterator.get(extensionPointType, plugin.getClass().getClassLoader()); List extensions = new ArrayList<>(); - ServiceLoader.load(extensionPointType, plugin.getClass().getClassLoader()).stream() - .map(ServiceLoader.Provider::type) - .forEachOrdered(service -> { - extensions.add(createExtension(service, extensionPointType, plugin)); - }); + while (classIterator.hasNext()) { + Class extensionClass = classIterator.next(); + extensions.add(createExtension(extensionClass, extensionPointType, plugin)); + } return extensions; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java index 2a6a0201ec340..2ffcbfef04c46 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java @@ -112,6 +112,9 @@ public InternalAggregation reduce(List aggregations, Reduce InternalTopHits topHitsAgg = (InternalTopHits) aggregations.get(i); shardDocs[i] = topHitsAgg.topDocs.topDocs; shardHits[i] = topHitsAgg.searchHits; + for (ScoreDoc doc : shardDocs[i].scoreDocs) { + doc.shardIndex = i; + } } reducedTopDocs = TopDocs.merge(sort, from, size, (TopFieldDocs[]) shardDocs); } else { @@ -120,6 +123,9 @@ public InternalAggregation reduce(List aggregations, Reduce InternalTopHits topHitsAgg = (InternalTopHits) aggregations.get(i); shardDocs[i] = topHitsAgg.topDocs.topDocs; shardHits[i] = topHitsAgg.searchHits; + for (ScoreDoc doc : shardDocs[i].scoreDocs) { + doc.shardIndex = i; + } } reducedTopDocs = TopDocs.merge(from, size, shardDocs); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java index 2b36af7e21590..080097b9894c2 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java @@ -38,7 +38,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentReader; -import org.apache.lucene.search.suggest.document.Completion84PostingsFormat; +import org.apache.lucene.search.suggest.document.Completion90PostingsFormat; import org.apache.lucene.search.suggest.document.CompletionPostingsFormat; import org.apache.lucene.search.suggest.document.SuggestField; import org.apache.lucene.store.Directory; @@ -157,7 +157,7 @@ public void testCompletionField() throws Exception { @Override public PostingsFormat getPostingsFormatForField(String field) { if (field.startsWith("suggest_")) { - return new Completion84PostingsFormat(randomFrom(CompletionPostingsFormat.FSTLoadMode.values())); + return new Completion90PostingsFormat(randomFrom(CompletionPostingsFormat.FSTLoadMode.values())); } else { return super.postingsFormat(); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java index c9a32f55c25d4..ecddf51c49cad 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java @@ -56,7 +56,7 @@ public void testBasic() { public void testVerbose() { IndicesSegmentResponse rsp = client().admin().indices().prepareSegments("test").setVerbose(true).get(); List segments = rsp.getIndices().get("test").iterator().next().getShards()[0].getSegments(); - assertNotNull(segments.get(0).toString(), segments.get(0).ramTree); + assertNull(segments.get(0).toString(), segments.get(0).ramTree); } /** diff --git a/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java b/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java index c17661c3f6a94..502fca04063d5 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java @@ -15,7 +15,7 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryCachingPolicy; -import org.apache.lucene.search.suggest.document.Completion84PostingsFormat; +import org.apache.lucene.search.suggest.document.Completion90PostingsFormat; import org.apache.lucene.search.suggest.document.SuggestField; import org.apache.lucene.store.Directory; import org.elasticsearch.ElasticsearchException; @@ -45,7 +45,7 @@ public void testExceptionsAreNotCached() { public void testCompletionStatsCache() throws IOException, InterruptedException { final IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); - final PostingsFormat postingsFormat = new Completion84PostingsFormat(); + final PostingsFormat postingsFormat = new Completion90PostingsFormat(); indexWriterConfig.setCodec(new Lucene90Codec() { @Override public PostingsFormat getPostingsFormatForField(String field) { diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index e93c98112c121..e181fc1ffcf95 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -284,7 +284,7 @@ public void testVerboseSegments() throws Exception { segments = engine.segments(true); assertThat(segments.size(), equalTo(1)); - assertThat(segments.get(0).ramTree, notNullValue()); + assertThat(segments.get(0).ramTree, nullValue()); ParsedDocument doc2 = testParsedDocument("2", null, testDocumentWithTextField(), B_2, null); engine.index(indexForDoc(doc2)); @@ -295,9 +295,9 @@ public void testVerboseSegments() throws Exception { segments = engine.segments(true); assertThat(segments.size(), equalTo(3)); - assertThat(segments.get(0).ramTree, notNullValue()); - assertThat(segments.get(1).ramTree, notNullValue()); - assertThat(segments.get(2).ramTree, notNullValue()); + assertThat(segments.get(0).ramTree, nullValue()); + assertThat(segments.get(1).ramTree, nullValue()); + assertThat(segments.get(2).ramTree, nullValue()); } } diff --git a/server/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java index f547ae25aa4ec..26e75ae63383a 100644 --- a/server/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java @@ -83,7 +83,7 @@ public void testToQueryInnerPrefixQuery() throws Exception { Query query = parseQuery(queryAsString).toQuery(createSearchExecutionContext()); Query expected = new DisjunctionMaxQuery(List.of( new BoostQuery(new PrefixQuery(new Term(TEXT_FIELD_NAME, "sh")), 1.2f) - ), 1); + ), 0); assertEquals(expected, query); } diff --git a/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java index 7c79f0d6e39ba..d528964ec01b5 100644 --- a/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java @@ -247,8 +247,8 @@ public void testToQueryBooleanPrefixMultipleFields() throws IOException { final Query query = queryBuilder.toQuery(createSearchExecutionContext()); Query expected = new DisjunctionMaxQuery(List.of( new BooleanQuery.Builder() - .add(new TermQuery(new Term(TEXT_FIELD_NAME, "foo")), BooleanClause.Occur.MUST) - .add(new PrefixQuery(new Term(TEXT_FIELD_NAME, "bar")), BooleanClause.Occur.MUST) + .add(new TermQuery(new Term(TEXT_FIELD_NAME, "foo")), BooleanClause.Occur.SHOULD) + .add(new PrefixQuery(new Term(TEXT_FIELD_NAME, "bar")), BooleanClause.Occur.SHOULD) .build(), new PrefixQuery(new Term(KEYWORD_FIELD_NAME, "foo bar"))), 1 diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index 3707678c980d4..2e8f84227b146 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.search.fetch.subphase; -import org.apache.lucene.util.automaton.TooComplexToDeterminizeException; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; @@ -907,6 +906,8 @@ public void testLastFormatWins() throws IOException { * Field patterns retrieved with "include_unmapped" use an automaton with a maximal allowed size internally. * This test checks we have a bound in place to avoid misuse of this with exceptionally large field patterns */ + // nocommit LUCENE-9981 seems to make this essentially impossible to hit, so remove the test? + /* public void testTooManyUnmappedFieldWildcardPattern() throws IOException { MapperService mapperService = createMapperService(); @@ -914,11 +915,12 @@ public void testTooManyUnmappedFieldWildcardPattern() throws IOException { List fieldAndFormatList = new ArrayList<>(); boolean includeUnmapped = true; - for (int i = 0; i < 1000; i++) { + for (int i = 0; i < 4000; i++) { fieldAndFormatList.add(new FieldAndFormat(randomAlphaOfLength(150) + "*", null, includeUnmapped)); } expectThrows(TooComplexToDeterminizeException.class, () -> fetchFields(mapperService, source, fieldAndFormatList)); } + */ private List fieldAndFormatList(String name, String format, boolean includeUnmapped) { return Collections.singletonList(new FieldAndFormat(name, format, includeUnmapped)); diff --git a/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java index b91a420a3ebbc..d4ee2ad4b3634 100644 --- a/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java @@ -111,6 +111,7 @@ public abstract class AnalysisFactoryTestCase extends ESTestCase { entry("ngram", MovedToAnalysisCommon.class), entry("norwegianlightstem", MovedToAnalysisCommon.class), entry("norwegianminimalstem", MovedToAnalysisCommon.class), + entry("norwegiannormalization", MovedToAnalysisCommon.class), entry("patterncapturegroup", MovedToAnalysisCommon.class), entry("patternreplace", MovedToAnalysisCommon.class), entry("persiannormalization", MovedToAnalysisCommon.class), @@ -132,6 +133,7 @@ public abstract class AnalysisFactoryTestCase extends ESTestCase { entry("stemmeroverride", MovedToAnalysisCommon.class), entry("stop", StopTokenFilterFactory.class), entry("swedishlightstem", MovedToAnalysisCommon.class), + entry("swedishminimalstem", MovedToAnalysisCommon.class), entry("synonym", MovedToAnalysisCommon.class), entry("synonymgraph", MovedToAnalysisCommon.class), entry("trim", MovedToAnalysisCommon.class), From 873f02d8dc4f74ede4bd3d7cde388a9630a6b32d Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 2 Aug 2021 16:13:37 +0100 Subject: [PATCH 14/67] iter --- .../org/elasticsearch/validate/SimpleValidateQueryIT.java | 7 +++++-- .../org/elasticsearch/common/settings/KeyStoreWrapper.java | 4 ++-- .../elasticsearch/index/mapper/CompletionFieldMapper.java | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java index df9e8216b4086..85e7c9634a2bc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java @@ -111,8 +111,11 @@ public void testExplainValidateQueryTwoNodes() throws IOException { assertThat(response.isValid(), equalTo(true)); assertThat(response.getQueryExplanation().size(), equalTo(1)); assertThat(response.getQueryExplanation().get(0).getExplanation(), - equalTo("(MatchNoDocsQuery(\"failed [bar] query, caused by number_format_exception:[For input string: \"foo\"]\") " + - "| foo:foo | baz:foo)")); + containsString("MatchNoDocsQuery(\"failed [bar] query, caused by number_format_exception:[For input string: \"foo\"]\")")); + assertThat(response.getQueryExplanation().get(0).getExplanation(), + containsString("foo:foo")); + assertThat(response.getQueryExplanation().get(0).getExplanation(), + containsString("baz:foo")); assertThat(response.getQueryExplanation().get(0).getError(), nullValue()); } } diff --git a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java index c1b9770f034b1..c6a0e999a4c71 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java +++ b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java @@ -286,14 +286,14 @@ public static KeyStoreWrapper load(Path configDir) throws IOException { output.writeUTF(entry.getKey()); output.writeUTF(entry.getValue()); } - int keystoreLen = input.readInt(); + int keystoreLen = CodecUtil.readBEInt(input); byte[] keystoreBytes = new byte[keystoreLen]; input.readBytes(keystoreBytes, 0, keystoreLen); output.write(keystoreBytes); } dataBytes = bytes.toByteArray(); } else { - int dataBytesLen = input.readInt(); + int dataBytesLen = CodecUtil.readBEInt(input); dataBytes = new byte[dataBytesLen]; input.readBytes(dataBytes, 0, dataBytesLen); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index 3052d860a3a92..e9c885d3ed825 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -11,7 +11,7 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; -import org.apache.lucene.search.suggest.document.Completion84PostingsFormat; +import org.apache.lucene.search.suggest.document.Completion90PostingsFormat; import org.apache.lucene.search.suggest.document.CompletionAnalyzer; import org.apache.lucene.search.suggest.document.CompletionQuery; import org.apache.lucene.search.suggest.document.FuzzyCompletionQuery; @@ -241,7 +241,7 @@ public ContextMappings getContextMappings() { */ public static synchronized PostingsFormat postingsFormat() { if (postingsFormat == null) { - postingsFormat = new Completion84PostingsFormat(); + postingsFormat = new Completion90PostingsFormat(); } return postingsFormat; } From fd3f4a98089af90a94367c84943ab7abf6b84085 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 3 Aug 2021 08:58:42 +0100 Subject: [PATCH 15/67] more precommit --- .../elasticsearch/common/settings/KeyStoreWrapperTests.java | 1 + .../src/main/java/org/apache/lucene/util/SPIClassIterator.java | 3 ++- .../elasticsearch/search/fetch/subphase/FieldFetcherTests.java | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java index dec41e967292f..53b1db9ec4bac 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java @@ -40,6 +40,7 @@ import java.util.List; import java.util.Locale; import java.util.Set; + import javax.crypto.Cipher; import javax.crypto.CipherOutputStream; import javax.crypto.SecretKey; diff --git a/server/src/main/java/org/apache/lucene/util/SPIClassIterator.java b/server/src/main/java/org/apache/lucene/util/SPIClassIterator.java index 5c2842df131da..cf05094affddd 100644 --- a/server/src/main/java/org/apache/lucene/util/SPIClassIterator.java +++ b/server/src/main/java/org/apache/lucene/util/SPIClassIterator.java @@ -1,4 +1,4 @@ -/* +/* @notice * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -40,6 +40,7 @@ * * @lucene.internal */ +@SuppressForbidden(reason="Copied from Lucene") public final class SPIClassIterator implements Iterator> { private static final String META_INF_SERVICES = "META-INF/services/"; diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index 2e8f84227b146..6b4d20bd7712c 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -906,7 +906,7 @@ public void testLastFormatWins() throws IOException { * Field patterns retrieved with "include_unmapped" use an automaton with a maximal allowed size internally. * This test checks we have a bound in place to avoid misuse of this with exceptionally large field patterns */ - // nocommit LUCENE-9981 seems to make this essentially impossible to hit, so remove the test? + // TODO LUCENE-9981 seems to make this essentially impossible to hit, so remove the test? /* public void testTooManyUnmappedFieldWildcardPattern() throws IOException { MapperService mapperService = createMapperService(); From 34ba4099bf2584d30aeee8996bbf36cbdae1a4d8 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 4 Aug 2021 09:32:33 +0100 Subject: [PATCH 16/67] move and rename org.apache.lucene tests --- .../elasticsearch/plugins/PluginsService.java | 2 +- .../plugins/spi}/SPIClassIterator.java | 13 ++++++++++- .../DeDuplicatingTokenFilterTests.java} | 7 ++++-- .../TruncateTokenFilterTests.java} | 5 +++-- ...singTopDocsCollectorSearchAfterTests.java} | 6 ++--- .../CollapsingTopDocsCollectorTests.java} | 4 ++-- ...ftDeletesDirectoryReaderWrapperTests.java} | 12 ++++++++-- .../index/ShuffleForcedMergePolicyTests.java} | 13 +++++++++-- ...ndomBinaryDocValuesRangeQueryTestCase.java | 2 +- .../BinaryDocValuesRangeQueryTests.java} | 4 ++-- .../queries/BlendedTermQueryTests.java} | 5 +++-- ...RandomBinaryDocValuesRangeQueryTests.java} | 4 ++-- ...RandomBinaryDocValuesRangeQueryTests.java} | 4 ++-- ...RandomBinaryDocValuesRangeQueryTests.java} | 4 ++-- ...RandomBinaryDocValuesRangeQueryTests.java} | 4 ++-- ...RandomBinaryDocValuesRangeQueryTests.java} | 4 ++-- .../lucene/queries/MinDocQueryTests.java} | 5 +++-- .../SearchAfterSortedDocQueryTests.java} | 5 +++-- .../queries/SpanMatchNoDocsQueryTests.java} | 5 +++-- .../search/XCombinedFieldQueryTests.java} | 22 +++++++++++++++++-- .../BoundedBreakIteratorScannerTests.java} | 5 +++-- .../CustomPassageFormatterTests.java} | 7 ++++-- .../CustomUnifiedHighlighterTests.java} | 9 ++++++-- .../lucene/util/CombinedBitSetTests.java} | 8 +++++-- 24 files changed, 113 insertions(+), 46 deletions(-) rename server/src/main/java/org/{apache/lucene/util => elasticsearch/plugins/spi}/SPIClassIterator.java (93%) rename server/src/test/java/org/{apache/lucene/analysis/miscellaneous/TestDeDuplicatingTokenFilter.java => elasticsearch/lucene/analysis/miscellaneous/DeDuplicatingTokenFilterTests.java} (93%) rename server/src/test/java/org/{apache/lucene/analysis/miscellaneous/TestTruncateTokenFilter.java => elasticsearch/lucene/analysis/miscellaneous/TruncateTokenFilterTests.java} (91%) rename server/src/test/java/org/{apache/lucene/grouping/TestCollapsingTopDocsCollectorSearchAfter.java => elasticsearch/lucene/grouping/CollapsingTopDocsCollectorSearchAfterTests.java} (98%) rename server/src/test/java/org/{apache/lucene/grouping/TestCollapsingTopDocsCollector.java => elasticsearch/lucene/grouping/CollapsingTopDocsCollectorTests.java} (99%) rename server/src/test/java/org/{apache/lucene/index/TestLazySoftDeletesDirectoryReaderWrapper.java => elasticsearch/lucene/index/LazySoftDeletesDirectoryReaderWrapperTests.java} (93%) rename server/src/test/java/org/{apache/lucene/index/TestShuffleForcedMergePolicy.java => elasticsearch/lucene/index/ShuffleForcedMergePolicyTests.java} (84%) rename server/src/test/java/org/{apache => elasticsearch}/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java (98%) rename server/src/test/java/org/{apache/lucene/queries/TestBinaryDocValuesRangeQuery.java => elasticsearch/lucene/queries/BinaryDocValuesRangeQueryTests.java} (98%) rename server/src/test/java/org/{apache/lucene/queries/TestBlendedTermQuery.java => elasticsearch/lucene/queries/BlendedTermQueryTests.java} (99%) rename server/src/test/java/org/{apache/lucene/queries/TestDoubleRandomBinaryDocValuesRangeQuery.java => elasticsearch/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java} (95%) rename server/src/test/java/org/{apache/lucene/queries/TestFloatRandomBinaryDocValuesRangeQuery.java => elasticsearch/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java} (95%) rename server/src/test/java/org/{apache/lucene/queries/TestInetAddressRandomBinaryDocValuesRangeQuery.java => elasticsearch/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java} (96%) rename server/src/test/java/org/{apache/lucene/queries/TestIntegerRandomBinaryDocValuesRangeQuery.java => elasticsearch/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java} (95%) rename server/src/test/java/org/{apache/lucene/queries/TestLongRandomBinaryDocValuesRangeQuery.java => elasticsearch/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java} (95%) rename server/src/test/java/org/{apache/lucene/queries/TestMinDocQuery.java => elasticsearch/lucene/queries/MinDocQueryTests.java} (94%) rename server/src/test/java/org/{apache/lucene/queries/TestSearchAfterSortedDocQuery.java => elasticsearch/lucene/queries/SearchAfterSortedDocQueryTests.java} (96%) rename server/src/test/java/org/{apache/lucene/queries/TestSpanMatchNoDocsQuery.java => elasticsearch/lucene/queries/SpanMatchNoDocsQueryTests.java} (96%) rename server/src/test/java/org/{apache/lucene/search/TestXCombinedFieldQuery.java => elasticsearch/lucene/search/XCombinedFieldQueryTests.java} (90%) rename server/src/test/java/org/{apache/lucene/search/uhighlight/TestBoundedBreakIteratorScanner.java => elasticsearch/lucene/search/uhighlight/BoundedBreakIteratorScannerTests.java} (96%) rename server/src/test/java/org/{apache/lucene/search/uhighlight/TestCustomPassageFormatter.java => elasticsearch/lucene/search/uhighlight/CustomPassageFormatterTests.java} (93%) rename server/src/test/java/org/{apache/lucene/search/uhighlight/TestCustomUnifiedHighlighter.java => elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java} (97%) rename server/src/test/java/org/{apache/lucene/util/TestCombinedBitSet.java => elasticsearch/lucene/util/CombinedBitSetTests.java} (92%) diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java index 8dd8f13c8fab7..d1c84111cbec0 100644 --- a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -16,7 +16,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.util.SPIClassIterator; +import org.elasticsearch.plugins.spi.SPIClassIterator; import org.elasticsearch.Build; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; diff --git a/server/src/main/java/org/apache/lucene/util/SPIClassIterator.java b/server/src/main/java/org/elasticsearch/plugins/spi/SPIClassIterator.java similarity index 93% rename from server/src/main/java/org/apache/lucene/util/SPIClassIterator.java rename to server/src/main/java/org/elasticsearch/plugins/spi/SPIClassIterator.java index cf05094affddd..ae18d9e9c845c 100644 --- a/server/src/main/java/org/apache/lucene/util/SPIClassIterator.java +++ b/server/src/main/java/org/elasticsearch/plugins/spi/SPIClassIterator.java @@ -1,3 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + /* @notice * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -15,7 +23,10 @@ * limitations under the License. */ -package org.apache.lucene.util; +package org.elasticsearch.plugins.spi; + +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.SuppressForbidden; import java.io.IOException; import java.io.InputStream; diff --git a/server/src/test/java/org/apache/lucene/analysis/miscellaneous/TestDeDuplicatingTokenFilter.java b/server/src/test/java/org/elasticsearch/lucene/analysis/miscellaneous/DeDuplicatingTokenFilterTests.java similarity index 93% rename from server/src/test/java/org/apache/lucene/analysis/miscellaneous/TestDeDuplicatingTokenFilter.java rename to server/src/test/java/org/elasticsearch/lucene/analysis/miscellaneous/DeDuplicatingTokenFilterTests.java index 71ba08df3d3fb..013e6ee2ab8f6 100644 --- a/server/src/test/java/org/apache/lucene/analysis/miscellaneous/TestDeDuplicatingTokenFilter.java +++ b/server/src/test/java/org/elasticsearch/lucene/analysis/miscellaneous/DeDuplicatingTokenFilterTests.java @@ -6,12 +6,15 @@ * Side Public License, v 1. */ -package org.apache.lucene.analysis.miscellaneous; +package org.elasticsearch.lucene.analysis.miscellaneous; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.miscellaneous.DeDuplicatingTokenFilter; +import org.apache.lucene.analysis.miscellaneous.DuplicateByteSequenceSpotter; +import org.apache.lucene.analysis.miscellaneous.DuplicateSequenceAttribute; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.elasticsearch.test.ESTestCase; @@ -19,7 +22,7 @@ import static org.hamcrest.Matchers.equalTo; -public class TestDeDuplicatingTokenFilter extends ESTestCase { +public class DeDuplicatingTokenFilterTests extends ESTestCase { public void testSimple() throws IOException { DuplicateByteSequenceSpotter bytesDeDuper = new DuplicateByteSequenceSpotter(); Analyzer analyzer = new Analyzer() { diff --git a/server/src/test/java/org/apache/lucene/analysis/miscellaneous/TestTruncateTokenFilter.java b/server/src/test/java/org/elasticsearch/lucene/analysis/miscellaneous/TruncateTokenFilterTests.java similarity index 91% rename from server/src/test/java/org/apache/lucene/analysis/miscellaneous/TestTruncateTokenFilter.java rename to server/src/test/java/org/elasticsearch/lucene/analysis/miscellaneous/TruncateTokenFilterTests.java index f0b4f9c475531..c5ad79670c401 100644 --- a/server/src/test/java/org/apache/lucene/analysis/miscellaneous/TestTruncateTokenFilter.java +++ b/server/src/test/java/org/elasticsearch/lucene/analysis/miscellaneous/TruncateTokenFilterTests.java @@ -6,12 +6,13 @@ * Side Public License, v 1. */ -package org.apache.lucene.analysis.miscellaneous; +package org.elasticsearch.lucene.analysis.miscellaneous; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.miscellaneous.TruncateTokenFilter; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.elasticsearch.test.ESTestCase; @@ -19,7 +20,7 @@ import static org.hamcrest.Matchers.equalTo; -public class TestTruncateTokenFilter extends ESTestCase { +public class TruncateTokenFilterTests extends ESTestCase { public void testSimple() throws IOException { Analyzer analyzer = new Analyzer() { @Override diff --git a/server/src/test/java/org/apache/lucene/grouping/TestCollapsingTopDocsCollectorSearchAfter.java b/server/src/test/java/org/elasticsearch/lucene/grouping/CollapsingTopDocsCollectorSearchAfterTests.java similarity index 98% rename from server/src/test/java/org/apache/lucene/grouping/TestCollapsingTopDocsCollectorSearchAfter.java rename to server/src/test/java/org/elasticsearch/lucene/grouping/CollapsingTopDocsCollectorSearchAfterTests.java index f4e36c39314dd..0f035350bdf73 100644 --- a/server/src/test/java/org/apache/lucene/grouping/TestCollapsingTopDocsCollectorSearchAfter.java +++ b/server/src/test/java/org/elasticsearch/lucene/grouping/CollapsingTopDocsCollectorSearchAfterTests.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.apache.lucene.grouping; +package org.elasticsearch.lucene.grouping; import org.apache.lucene.document.Document; import org.apache.lucene.document.NumericDocValuesField; @@ -40,10 +40,10 @@ import java.util.Set; /** - * This test is adapted from {@link TestCollapsingTopDocsCollector} with + * This test is adapted from {@link CollapsingTopDocsCollectorTests} with * modifications to test {@link FieldDoc after} parameter. */ -public class TestCollapsingTopDocsCollectorSearchAfter extends ESTestCase { +public class CollapsingTopDocsCollectorSearchAfterTests extends ESTestCase { interface CollapsingDocValuesProducer> { T randomGroup(int maxGroup); void add(Document doc, T value); diff --git a/server/src/test/java/org/apache/lucene/grouping/TestCollapsingTopDocsCollector.java b/server/src/test/java/org/elasticsearch/lucene/grouping/CollapsingTopDocsCollectorTests.java similarity index 99% rename from server/src/test/java/org/apache/lucene/grouping/TestCollapsingTopDocsCollector.java rename to server/src/test/java/org/elasticsearch/lucene/grouping/CollapsingTopDocsCollectorTests.java index e1e86fab2170b..61d8e25abe237 100644 --- a/server/src/test/java/org/apache/lucene/grouping/TestCollapsingTopDocsCollector.java +++ b/server/src/test/java/org/elasticsearch/lucene/grouping/CollapsingTopDocsCollectorTests.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.apache.lucene.grouping; +package org.elasticsearch.lucene.grouping; import org.apache.lucene.document.Document; import org.apache.lucene.document.NumericDocValuesField; @@ -49,7 +49,7 @@ import java.util.Set; -public class TestCollapsingTopDocsCollector extends ESTestCase { +public class CollapsingTopDocsCollectorTests extends ESTestCase { private static class SegmentSearcher extends IndexSearcher { private final List ctx; diff --git a/server/src/test/java/org/apache/lucene/index/TestLazySoftDeletesDirectoryReaderWrapper.java b/server/src/test/java/org/elasticsearch/lucene/index/LazySoftDeletesDirectoryReaderWrapperTests.java similarity index 93% rename from server/src/test/java/org/apache/lucene/index/TestLazySoftDeletesDirectoryReaderWrapper.java rename to server/src/test/java/org/elasticsearch/lucene/index/LazySoftDeletesDirectoryReaderWrapperTests.java index 14b9665425cf9..ece103a1b8e36 100644 --- a/server/src/test/java/org/apache/lucene/index/TestLazySoftDeletesDirectoryReaderWrapper.java +++ b/server/src/test/java/org/elasticsearch/lucene/index/LazySoftDeletesDirectoryReaderWrapperTests.java @@ -6,12 +6,20 @@ * Side Public License, v 1. */ -package org.apache.lucene.index; +package org.elasticsearch.lucene.index; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LazySoftDeletesDirectoryReaderWrapper; +import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.index.SoftDeletesRetentionMergePolicy; +import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.TermQuery; @@ -24,7 +32,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; -public class TestLazySoftDeletesDirectoryReaderWrapper extends LuceneTestCase { +public class LazySoftDeletesDirectoryReaderWrapperTests extends LuceneTestCase { public void testDropFullyDeletedSegments() throws IOException { IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); diff --git a/server/src/test/java/org/apache/lucene/index/TestShuffleForcedMergePolicy.java b/server/src/test/java/org/elasticsearch/lucene/index/ShuffleForcedMergePolicyTests.java similarity index 84% rename from server/src/test/java/org/apache/lucene/index/TestShuffleForcedMergePolicy.java rename to server/src/test/java/org/elasticsearch/lucene/index/ShuffleForcedMergePolicyTests.java index c6df138debb6e..debae061a0f34 100644 --- a/server/src/test/java/org/apache/lucene/index/TestShuffleForcedMergePolicy.java +++ b/server/src/test/java/org/elasticsearch/lucene/index/ShuffleForcedMergePolicyTests.java @@ -6,12 +6,21 @@ * Side Public License, v 1. */ -package org.apache.lucene.index; +package org.elasticsearch.lucene.index; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.StringField; +import org.apache.lucene.index.BaseMergePolicyTestCase; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.MergePolicy; +import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.index.ShuffleForcedMergePolicy; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.store.Directory; @@ -22,7 +31,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -public class TestShuffleForcedMergePolicy extends BaseMergePolicyTestCase { +public class ShuffleForcedMergePolicyTests extends BaseMergePolicyTestCase { public void testDiagnostics() throws IOException { try (Directory dir = newDirectory()) { IndexWriterConfig iwc = newIndexWriterConfig(); diff --git a/server/src/test/java/org/apache/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java b/server/src/test/java/org/elasticsearch/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java similarity index 98% rename from server/src/test/java/org/apache/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java rename to server/src/test/java/org/elasticsearch/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java index 0a3c858445e6c..ed78d8af92fbe 100644 --- a/server/src/test/java/org/apache/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Field; diff --git a/server/src/test/java/org/apache/lucene/queries/TestBinaryDocValuesRangeQuery.java b/server/src/test/java/org/elasticsearch/lucene/queries/BinaryDocValuesRangeQueryTests.java similarity index 98% rename from server/src/test/java/org/apache/lucene/queries/TestBinaryDocValuesRangeQuery.java rename to server/src/test/java/org/elasticsearch/lucene/queries/BinaryDocValuesRangeQueryTests.java index 6ac22aa1bcc58..26a54e0b5175a 100644 --- a/server/src/test/java/org/apache/lucene/queries/TestBinaryDocValuesRangeQuery.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/BinaryDocValuesRangeQueryTests.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Document; @@ -27,7 +27,7 @@ import static org.apache.lucene.queries.BinaryDocValuesRangeQuery.QueryType.INTERSECTS; import static org.apache.lucene.queries.BinaryDocValuesRangeQuery.QueryType.WITHIN; -public class TestBinaryDocValuesRangeQuery extends ESTestCase { +public class BinaryDocValuesRangeQueryTests extends ESTestCase { public void testBasics() throws Exception { String fieldName = "long_field"; diff --git a/server/src/test/java/org/apache/lucene/queries/TestBlendedTermQuery.java b/server/src/test/java/org/elasticsearch/lucene/queries/BlendedTermQueryTests.java similarity index 99% rename from server/src/test/java/org/apache/lucene/queries/TestBlendedTermQuery.java rename to server/src/test/java/org/elasticsearch/lucene/queries/BlendedTermQueryTests.java index 76f7cd4e5f805..3489316823285 100644 --- a/server/src/test/java/org/apache/lucene/queries/TestBlendedTermQuery.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/BlendedTermQueryTests.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; @@ -17,6 +17,7 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermStates; +import org.apache.lucene.queries.BlendedTermQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.DisjunctionMaxQuery; @@ -46,7 +47,7 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.instanceOf; -public class TestBlendedTermQuery extends ESTestCase { +public class BlendedTermQueryTests extends ESTestCase { public void testDismaxQuery() throws IOException { Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); diff --git a/server/src/test/java/org/apache/lucene/queries/TestDoubleRandomBinaryDocValuesRangeQuery.java b/server/src/test/java/org/elasticsearch/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java similarity index 95% rename from server/src/test/java/org/apache/lucene/queries/TestDoubleRandomBinaryDocValuesRangeQuery.java rename to server/src/test/java/org/elasticsearch/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java index b54c55904f7f3..0b20c42d052f6 100644 --- a/server/src/test/java/org/apache/lucene/queries/TestDoubleRandomBinaryDocValuesRangeQuery.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java @@ -5,11 +5,11 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.elasticsearch.index.mapper.RangeType; -public class TestDoubleRandomBinaryDocValuesRangeQuery extends BaseRandomBinaryDocValuesRangeQueryTestCase { +public class DoubleRandomBinaryDocValuesRangeQueryTests extends BaseRandomBinaryDocValuesRangeQueryTestCase { @Override protected String fieldName() { diff --git a/server/src/test/java/org/apache/lucene/queries/TestFloatRandomBinaryDocValuesRangeQuery.java b/server/src/test/java/org/elasticsearch/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java similarity index 95% rename from server/src/test/java/org/apache/lucene/queries/TestFloatRandomBinaryDocValuesRangeQuery.java rename to server/src/test/java/org/elasticsearch/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java index 671ba8702dc98..907cde732c25c 100644 --- a/server/src/test/java/org/apache/lucene/queries/TestFloatRandomBinaryDocValuesRangeQuery.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java @@ -5,11 +5,11 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.elasticsearch.index.mapper.RangeType; -public class TestFloatRandomBinaryDocValuesRangeQuery extends BaseRandomBinaryDocValuesRangeQueryTestCase { +public class FloatRandomBinaryDocValuesRangeQueryTests extends BaseRandomBinaryDocValuesRangeQueryTestCase { @Override protected String fieldName() { diff --git a/server/src/test/java/org/apache/lucene/queries/TestInetAddressRandomBinaryDocValuesRangeQuery.java b/server/src/test/java/org/elasticsearch/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java similarity index 96% rename from server/src/test/java/org/apache/lucene/queries/TestInetAddressRandomBinaryDocValuesRangeQuery.java rename to server/src/test/java/org/elasticsearch/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java index 80d523a62991a..995953cb3b5a6 100644 --- a/server/src/test/java/org/apache/lucene/queries/TestInetAddressRandomBinaryDocValuesRangeQuery.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.apache.lucene.document.InetAddressPoint; import org.elasticsearch.index.mapper.RangeType; @@ -14,7 +14,7 @@ import java.net.UnknownHostException; import java.util.Arrays; -public class TestInetAddressRandomBinaryDocValuesRangeQuery extends BaseRandomBinaryDocValuesRangeQueryTestCase { +public class InetAddressRandomBinaryDocValuesRangeQueryTests extends BaseRandomBinaryDocValuesRangeQueryTestCase { @Override protected String fieldName() { diff --git a/server/src/test/java/org/apache/lucene/queries/TestIntegerRandomBinaryDocValuesRangeQuery.java b/server/src/test/java/org/elasticsearch/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java similarity index 95% rename from server/src/test/java/org/apache/lucene/queries/TestIntegerRandomBinaryDocValuesRangeQuery.java rename to server/src/test/java/org/elasticsearch/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java index 5af1c22844498..6861bd7284128 100644 --- a/server/src/test/java/org/apache/lucene/queries/TestIntegerRandomBinaryDocValuesRangeQuery.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java @@ -5,12 +5,12 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.apache.lucene.util.TestUtil; import org.elasticsearch.index.mapper.RangeType; -public class TestIntegerRandomBinaryDocValuesRangeQuery extends BaseRandomBinaryDocValuesRangeQueryTestCase { +public class IntegerRandomBinaryDocValuesRangeQueryTests extends BaseRandomBinaryDocValuesRangeQueryTestCase { @Override protected String fieldName() { diff --git a/server/src/test/java/org/apache/lucene/queries/TestLongRandomBinaryDocValuesRangeQuery.java b/server/src/test/java/org/elasticsearch/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java similarity index 95% rename from server/src/test/java/org/apache/lucene/queries/TestLongRandomBinaryDocValuesRangeQuery.java rename to server/src/test/java/org/elasticsearch/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java index a0225566c1efb..87116153b11bd 100644 --- a/server/src/test/java/org/apache/lucene/queries/TestLongRandomBinaryDocValuesRangeQuery.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java @@ -5,12 +5,12 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.apache.lucene.util.TestUtil; import org.elasticsearch.index.mapper.RangeType; -public class TestLongRandomBinaryDocValuesRangeQuery extends BaseRandomBinaryDocValuesRangeQueryTestCase { +public class LongRandomBinaryDocValuesRangeQueryTests extends BaseRandomBinaryDocValuesRangeQueryTestCase { @Override protected String fieldName() { diff --git a/server/src/test/java/org/apache/lucene/queries/TestMinDocQuery.java b/server/src/test/java/org/elasticsearch/lucene/queries/MinDocQueryTests.java similarity index 94% rename from server/src/test/java/org/apache/lucene/queries/TestMinDocQuery.java rename to server/src/test/java/org/elasticsearch/lucene/queries/MinDocQueryTests.java index eafba190fd4d3..b61f8f9c72a5c 100644 --- a/server/src/test/java/org/apache/lucene/queries/TestMinDocQuery.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/MinDocQueryTests.java @@ -6,12 +6,13 @@ * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.queries.MinDocQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryUtils; @@ -20,7 +21,7 @@ import java.io.IOException; -public class TestMinDocQuery extends ESTestCase { +public class MinDocQueryTests extends ESTestCase { public void testBasics() { MinDocQuery query1 = new MinDocQuery(42); diff --git a/server/src/test/java/org/apache/lucene/queries/TestSearchAfterSortedDocQuery.java b/server/src/test/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQueryTests.java similarity index 96% rename from server/src/test/java/org/apache/lucene/queries/TestSearchAfterSortedDocQuery.java rename to server/src/test/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQueryTests.java index eb6b757b13a4b..957d838a75f61 100644 --- a/server/src/test/java/org/apache/lucene/queries/TestSearchAfterSortedDocQuery.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQueryTests.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.apache.lucene.document.Document; import org.apache.lucene.document.SortedDocValuesField; @@ -16,6 +16,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.ReaderUtil; +import org.apache.lucene.queries.SearchAfterSortedDocQuery; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; @@ -35,7 +36,7 @@ import static org.hamcrest.Matchers.equalTo; -public class TestSearchAfterSortedDocQuery extends ESTestCase { +public class SearchAfterSortedDocQueryTests extends ESTestCase { public void testBasics() { Sort sort1 = new Sort( diff --git a/server/src/test/java/org/apache/lucene/queries/TestSpanMatchNoDocsQuery.java b/server/src/test/java/org/elasticsearch/lucene/queries/SpanMatchNoDocsQueryTests.java similarity index 96% rename from server/src/test/java/org/apache/lucene/queries/TestSpanMatchNoDocsQuery.java rename to server/src/test/java/org/elasticsearch/lucene/queries/SpanMatchNoDocsQueryTests.java index 3005874d624a0..2fef323146043 100644 --- a/server/src/test/java/org/apache/lucene/queries/TestSpanMatchNoDocsQuery.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/SpanMatchNoDocsQueryTests.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; @@ -16,6 +16,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.SpanMatchNoDocsQuery; import org.apache.lucene.queries.spans.SpanNearQuery; import org.apache.lucene.queries.spans.SpanOrQuery; import org.apache.lucene.queries.spans.SpanQuery; @@ -29,7 +30,7 @@ import java.io.IOException; -public class TestSpanMatchNoDocsQuery extends ESTestCase { +public class SpanMatchNoDocsQueryTests extends ESTestCase { public void testSimple() throws Exception { SpanMatchNoDocsQuery query = new SpanMatchNoDocsQuery("field", "a good reason"); assertEquals(query.toString(), "SpanMatchNoDocsQuery(\"a good reason\")"); diff --git a/server/src/test/java/org/apache/lucene/search/TestXCombinedFieldQuery.java b/server/src/test/java/org/elasticsearch/lucene/search/XCombinedFieldQueryTests.java similarity index 90% rename from server/src/test/java/org/apache/lucene/search/TestXCombinedFieldQuery.java rename to server/src/test/java/org/elasticsearch/lucene/search/XCombinedFieldQueryTests.java index c08b678e375b0..46b341ca1d936 100644 --- a/server/src/test/java/org/apache/lucene/search/TestXCombinedFieldQuery.java +++ b/server/src/test/java/org/elasticsearch/lucene/search/XCombinedFieldQueryTests.java @@ -1,3 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + /* @notice * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -17,7 +25,7 @@ * Modifications copyright (C) 2020 Elasticsearch B.V. */ -package org.apache.lucene.search; +package org.elasticsearch.lucene.search; import com.carrotsearch.randomizedtesting.generators.RandomPicks; @@ -30,6 +38,16 @@ import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.CheckHits; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TopScoreDocCollector; +import org.apache.lucene.search.TotalHits; +import org.apache.lucene.search.XCombinedFieldQuery; import org.apache.lucene.search.similarities.BM25Similarity; import org.apache.lucene.search.similarities.BooleanSimilarity; import org.apache.lucene.search.similarities.ClassicSimilarity; @@ -49,7 +67,7 @@ * TODO remove once LUCENE 9999 is fixed and integrated and we remove our copy of the query * */ -public class TestXCombinedFieldQuery extends LuceneTestCase { +public class XCombinedFieldQueryTests extends LuceneTestCase { public void testRewrite() throws IOException { IndexReader reader = new MultiReader(); diff --git a/server/src/test/java/org/apache/lucene/search/uhighlight/TestBoundedBreakIteratorScanner.java b/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/BoundedBreakIteratorScannerTests.java similarity index 96% rename from server/src/test/java/org/apache/lucene/search/uhighlight/TestBoundedBreakIteratorScanner.java rename to server/src/test/java/org/elasticsearch/lucene/search/uhighlight/BoundedBreakIteratorScannerTests.java index 6e700e5e5e861..a9b34e15542ce 100644 --- a/server/src/test/java/org/apache/lucene/search/uhighlight/TestBoundedBreakIteratorScanner.java +++ b/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/BoundedBreakIteratorScannerTests.java @@ -6,8 +6,9 @@ * Side Public License, v 1. */ -package org.apache.lucene.search.uhighlight; +package org.elasticsearch.lucene.search.uhighlight; +import org.apache.lucene.search.uhighlight.BoundedBreakIteratorScanner; import org.elasticsearch.test.ESTestCase; import java.text.BreakIterator; @@ -20,7 +21,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; -public class TestBoundedBreakIteratorScanner extends ESTestCase { +public class BoundedBreakIteratorScannerTests extends ESTestCase { private static final String[] WORD_BOUNDARIES = new String[] { " ", " ", "\t", "#", "\n" }; private static final String[] SENTENCE_BOUNDARIES = diff --git a/server/src/test/java/org/apache/lucene/search/uhighlight/TestCustomPassageFormatter.java b/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomPassageFormatterTests.java similarity index 93% rename from server/src/test/java/org/apache/lucene/search/uhighlight/TestCustomPassageFormatter.java rename to server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomPassageFormatterTests.java index 30ec6c11391e6..004ebd4d6ce82 100644 --- a/server/src/test/java/org/apache/lucene/search/uhighlight/TestCustomPassageFormatter.java +++ b/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomPassageFormatterTests.java @@ -6,10 +6,13 @@ * Side Public License, v 1. */ -package org.apache.lucene.search.uhighlight; +package org.elasticsearch.lucene.search.uhighlight; import org.apache.lucene.search.highlight.DefaultEncoder; import org.apache.lucene.search.highlight.SimpleHTMLEncoder; +import org.apache.lucene.search.uhighlight.CustomPassageFormatter; +import org.apache.lucene.search.uhighlight.Passage; +import org.apache.lucene.search.uhighlight.Snippet; import org.apache.lucene.util.BytesRef; import org.elasticsearch.test.ESTestCase; @@ -17,7 +20,7 @@ import static org.hamcrest.CoreMatchers.notNullValue; -public class TestCustomPassageFormatter extends ESTestCase { +public class CustomPassageFormatterTests extends ESTestCase { public void testSimpleFormat() { String content = "This is a really cool highlighter. Unified highlighter gives nice snippets back. No matches here."; diff --git a/server/src/test/java/org/apache/lucene/search/uhighlight/TestCustomUnifiedHighlighter.java b/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java similarity index 97% rename from server/src/test/java/org/apache/lucene/search/uhighlight/TestCustomUnifiedHighlighter.java rename to server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java index 9ed42fe490dee..7120f40f5ac14 100644 --- a/server/src/test/java/org/apache/lucene/search/uhighlight/TestCustomUnifiedHighlighter.java +++ b/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.apache.lucene.search.uhighlight; +package org.elasticsearch.lucene.search.uhighlight; import static org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter.MULTIVAL_SEP_CHAR; import static org.hamcrest.CoreMatchers.equalTo; @@ -38,12 +38,17 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.highlight.DefaultEncoder; +import org.apache.lucene.search.uhighlight.BoundedBreakIteratorScanner; +import org.apache.lucene.search.uhighlight.CustomPassageFormatter; +import org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter; +import org.apache.lucene.search.uhighlight.Snippet; +import org.apache.lucene.search.uhighlight.UnifiedHighlighter; import org.apache.lucene.store.Directory; import org.elasticsearch.common.Strings; import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery; import org.elasticsearch.test.ESTestCase; -public class TestCustomUnifiedHighlighter extends ESTestCase { +public class CustomUnifiedHighlighterTests extends ESTestCase { private void assertHighlightOneDoc(String fieldName, String[] inputs, Analyzer analyzer, Query query, Locale locale, BreakIterator breakIterator, diff --git a/server/src/test/java/org/apache/lucene/util/TestCombinedBitSet.java b/server/src/test/java/org/elasticsearch/lucene/util/CombinedBitSetTests.java similarity index 92% rename from server/src/test/java/org/apache/lucene/util/TestCombinedBitSet.java rename to server/src/test/java/org/elasticsearch/lucene/util/CombinedBitSetTests.java index b659ab7901807..ad3a94d70c9e1 100644 --- a/server/src/test/java/org/apache/lucene/util/TestCombinedBitSet.java +++ b/server/src/test/java/org/elasticsearch/lucene/util/CombinedBitSetTests.java @@ -6,12 +6,16 @@ * Side Public License, v 1. */ -package org.apache.lucene.util; +package org.elasticsearch.lucene.util; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BitSet; +import org.apache.lucene.util.CombinedBitSet; +import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.SparseFixedBitSet; import org.elasticsearch.test.ESTestCase; -public class TestCombinedBitSet extends ESTestCase { +public class CombinedBitSetTests extends ESTestCase { public void testEmpty() { for (float percent : new float[] {0f, 0.1f, 0.5f, 0.9f, 1f}) { testCase(randomIntBetween(1, 10000), 0f, percent); From 4d4d57a7dc77f1164cba1c7f2b8bd2c017d24044 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 4 Aug 2021 09:47:58 +0100 Subject: [PATCH 17/67] precommit --- server/build.gradle | 6 ++---- .../lucene/queries/MinDocQuery.java | 2 +- .../lucene/queries/SearchAfterSortedDocQuery.java | 2 +- .../aggregations/bucket/composite/CompositeAggregator.java | 2 +- .../java/org/elasticsearch/search/query/QueryPhase.java | 4 ++-- .../org/elasticsearch/lucene/queries/MinDocQueryTests.java | 1 - .../lucene/queries/SearchAfterSortedDocQueryTests.java | 1 - .../org/elasticsearch/search/query/QueryPhaseTests.java | 2 +- 8 files changed, 8 insertions(+), 12 deletions(-) rename server/src/main/java/org/{apache => elasticsearch}/lucene/queries/MinDocQuery.java (99%) rename server/src/main/java/org/{apache => elasticsearch}/lucene/queries/SearchAfterSortedDocQuery.java (99%) diff --git a/server/build.gradle b/server/build.gradle index c9f5a13b04147..035989133e4aa 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -306,8 +306,6 @@ tasks.named('splitPackagesAudit').configure { 'org.apache.lucene.queryparser.classic.XQueryParser', 'org.apache.lucene.queries.BinaryDocValuesRangeQuery', 'org.apache.lucene.queries.BlendedTermQuery', - 'org.apache.lucene.queries.MinDocQuery', - 'org.apache.lucene.queries.SearchAfterSortedDocQuery', 'org.apache.lucene.queries.SpanMatchNoDocsQuery', 'org.apache.lucene.search.grouping.CollapseTopFieldDocs', 'org.apache.lucene.search.grouping.CollapsingDocValuesSource', @@ -341,7 +339,7 @@ tasks.named('splitPackagesAudit').configure { 'org.elasticsearch.cli.LoggingAwareMultiCommand', // these should be temporary, query needs package private access to TermScorer though - 'org.apache.lucene.search.XCombinedFieldQuery', - 'org.apache.lucene.search.XMultiNormsLeafSimScorer' + 'org.apache.lucene.search.XCombinedFieldQuery', + 'org.apache.lucene.search.XMultiNormsLeafSimScorer' } diff --git a/server/src/main/java/org/apache/lucene/queries/MinDocQuery.java b/server/src/main/java/org/elasticsearch/lucene/queries/MinDocQuery.java similarity index 99% rename from server/src/main/java/org/apache/lucene/queries/MinDocQuery.java rename to server/src/main/java/org/elasticsearch/lucene/queries/MinDocQuery.java index 38d19b3b58277..14102e8dad7df 100644 --- a/server/src/main/java/org/apache/lucene/queries/MinDocQuery.java +++ b/server/src/main/java/org/elasticsearch/lucene/queries/MinDocQuery.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; diff --git a/server/src/main/java/org/apache/lucene/queries/SearchAfterSortedDocQuery.java b/server/src/main/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQuery.java similarity index 99% rename from server/src/main/java/org/apache/lucene/queries/SearchAfterSortedDocQuery.java rename to server/src/main/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQuery.java index b8ee252a49d90..5a55e176f2800 100644 --- a/server/src/main/java/org/apache/lucene/queries/SearchAfterSortedDocQuery.java +++ b/server/src/main/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQuery.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.ConstantScoreScorer; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java index a9eef47d3d1da..c6368e3a0ddd2 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java @@ -13,7 +13,7 @@ import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.queries.SearchAfterSortedDocQuery; +import org.elasticsearch.lucene.queries.SearchAfterSortedDocQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.CollectionTerminatedException; diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 13ed9c5837768..42ef2485eba17 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -14,8 +14,8 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PointValues; -import org.apache.lucene.queries.MinDocQuery; -import org.apache.lucene.queries.SearchAfterSortedDocQuery; +import org.elasticsearch.lucene.queries.MinDocQuery; +import org.elasticsearch.lucene.queries.SearchAfterSortedDocQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Collector; diff --git a/server/src/test/java/org/elasticsearch/lucene/queries/MinDocQueryTests.java b/server/src/test/java/org/elasticsearch/lucene/queries/MinDocQueryTests.java index b61f8f9c72a5c..7f58ed9fe56d4 100644 --- a/server/src/test/java/org/elasticsearch/lucene/queries/MinDocQueryTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/MinDocQueryTests.java @@ -12,7 +12,6 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.queries.MinDocQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryUtils; diff --git a/server/src/test/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQueryTests.java b/server/src/test/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQueryTests.java index 957d838a75f61..f056b5b1308f1 100644 --- a/server/src/test/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQueryTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQueryTests.java @@ -16,7 +16,6 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.ReaderUtil; -import org.apache.lucene.queries.SearchAfterSortedDocQuery; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java index 3cd8855fe20d2..3fc88bbe86996 100644 --- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java @@ -26,7 +26,7 @@ import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; -import org.apache.lucene.queries.MinDocQuery; +import org.elasticsearch.lucene.queries.MinDocQuery; import org.apache.lucene.queries.spans.SpanNearQuery; import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BooleanClause; From 286f2d4c1d760ca998ddd467ecc3358968a2d1fc Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 4 Aug 2021 10:05:10 +0100 Subject: [PATCH 18/67] security permission for SPIClassIterator --- .../main/resources/org/elasticsearch/bootstrap/security.policy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy index 8ffc0d0eea47d..851062a1f46bc 100644 --- a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -22,6 +22,8 @@ grant codeBase "${codebase.elasticsearch-secure-sm}" { grant codeBase "${codebase.elasticsearch}" { // needed for loading plugins which may expect the context class loader to be set permission java.lang.RuntimePermission "setContextClassLoader"; + // needed for SPI class loading + permission java.lang.RuntimePermission "accessDeclaredMembers"; }; //// Very special jar permissions: From 3a7ebd59ff5af74a70ec2e9200725bc2ac6d3325 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 4 Aug 2021 10:52:11 +0100 Subject: [PATCH 19/67] analysis fixes --- .../analysis/common/CommonAnalysisPlugin.java | 2 ++ .../common/MinHashTokenFilterFactory.java | 16 +++++++--- .../NorwegianNormalizationFilterFactory.java | 32 +++++++++++++++++++ .../SwedishMinimalStemFilterFactory.java | 31 ++++++++++++++++++ .../common/CommonAnalysisFactoryTests.java | 2 ++ 5 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NorwegianNormalizationFilterFactory.java create mode 100644 modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SwedishMinimalStemFilterFactory.java diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java index 5978e013d2be1..bee2dc13e5534 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java @@ -289,6 +289,7 @@ public TokenStream create(TokenStream tokenStream) { }; }); + filters.put("norwegiannormalization", NorwegianNormalizationFilterFactory::new); filters.put("pattern_capture", requiresAnalysisSettings(PatternCaptureGroupTokenFilterFactory::new)); filters.put("pattern_replace", requiresAnalysisSettings(PatternReplaceTokenFilterFactory::new)); filters.put("persian_normalization", PersianNormalizationFilterFactory::new); @@ -305,6 +306,7 @@ public TokenStream create(TokenStream tokenStream) { filters.put("sorani_normalization", SoraniNormalizationFilterFactory::new); filters.put("stemmer_override", requiresAnalysisSettings(StemmerOverrideTokenFilterFactory::new)); filters.put("stemmer", StemmerTokenFilterFactory::new); + filters.put("swedishminimalstem", SwedishMinimalStemFilterFactory::new); filters.put("synonym", requiresAnalysisSettings(SynonymTokenFilterFactory::new)); filters.put("synonym_graph", requiresAnalysisSettings(SynonymGraphTokenFilterFactory::new)); filters.put("trim", TrimTokenFilterFactory::new); diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MinHashTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MinHashTokenFilterFactory.java index 37dc45d2a3017..76e1f6bab79b8 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MinHashTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MinHashTokenFilterFactory.java @@ -38,10 +38,18 @@ public TokenStream create(TokenStream tokenStream) { private Map convertSettings(Settings settings) { Map settingMap = new HashMap<>(); - settingMap.put("hashCount", settings.get("hash_count")); - settingMap.put("bucketCount", settings.get("bucket_count")); - settingMap.put("hashSetSize", settings.get("hash_set_size")); - settingMap.put("withRotation", settings.get("with_rotation")); + if (settings.hasValue("hash_count")) { + settingMap.put("hashCount", settings.get("hash_count")); + } + if (settings.hasValue("bucketCount")) { + settingMap.put("bucketCount", settings.get("bucket_count")); + } + if (settings.hasValue("hashSetSize")) { + settingMap.put("hashSetSize", settings.get("hash_set_size")); + } + if (settings.hasValue("with_rotation")) { + settingMap.put("withRotation", settings.get("with_rotation")); + } return settingMap; } } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NorwegianNormalizationFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NorwegianNormalizationFilterFactory.java new file mode 100644 index 0000000000000..e25f6c73fbdfb --- /dev/null +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NorwegianNormalizationFilterFactory.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.analysis.common; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.no.NorwegianNormalizationFilter; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; +import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory; + +/** + * Factory for {@link NorwegianNormalizationFilter} + */ +public class NorwegianNormalizationFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory { + + public NorwegianNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); + } + + @Override + public TokenStream create(TokenStream tokenStream) { + return new NorwegianNormalizationFilter(tokenStream); + } +} diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SwedishMinimalStemFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SwedishMinimalStemFilterFactory.java new file mode 100644 index 0000000000000..352c0cdd57799 --- /dev/null +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SwedishMinimalStemFilterFactory.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.analysis.common; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.sv.SwedishMinimalStemFilter; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; + +/** + * Factory for {@link SwedishMinimalStemFilter} + */ +public class SwedishMinimalStemFilterFactory extends AbstractTokenFilterFactory { + + public SwedishMinimalStemFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); + } + + @Override + public TokenStream create(TokenStream tokenStream) { + return new SwedishMinimalStemFilter(tokenStream); + } +} diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java index 35f8b705ff5bf..834c722baddfa 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java @@ -84,6 +84,7 @@ protected Map> getTokenFilters() { filters.put("latvianstem", StemmerTokenFilterFactory.class); filters.put("norwegianlightstem", StemmerTokenFilterFactory.class); filters.put("norwegianminimalstem", StemmerTokenFilterFactory.class); + filters.put("norwegiannormalization", NorwegianNormalizationFilterFactory.class); filters.put("portuguesestem", StemmerTokenFilterFactory.class); filters.put("portugueselightstem", StemmerTokenFilterFactory.class); filters.put("portugueseminimalstem", StemmerTokenFilterFactory.class); @@ -91,6 +92,7 @@ protected Map> getTokenFilters() { filters.put("soranistem", StemmerTokenFilterFactory.class); filters.put("spanishlightstem", StemmerTokenFilterFactory.class); filters.put("swedishlightstem", StemmerTokenFilterFactory.class); + filters.put("swedishminimalstem", SwedishMinimalStemFilterFactory.class); filters.put("stemmeroverride", StemmerOverrideTokenFilterFactory.class); filters.put("kstem", KStemTokenFilterFactory.class); filters.put("synonym", SynonymTokenFilterFactory.class); From 2d637ab426f41612d44d75af6cf33a671ec73c79 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 4 Aug 2021 11:44:04 +0100 Subject: [PATCH 20/67] some more BEInt/Long changes --- .../diskusage/IndexDiskUsageAnalyzer.java | 4 - .../action/search/TransportSearchHelper.java | 5 +- .../index/translog/Checkpoint.java | 28 +++--- .../plugins/spi/SPIClassIterator.java | 8 -- .../gateway/MetadataStateFormatTests.java | 2 +- .../index/engine/SegmentTests.java | 2 +- .../elasticsearch/index/store/StoreTests.java | 2 +- .../elasticsearch/test/CorruptionUtils.java | 2 +- .../mapper/HistogramFieldMapper.java | 7 +- .../ChecksumBlobContainerIndexInput.java | 2 +- .../xpack/spatial/index/fielddata/Extent.java | 87 ++++++++++--------- .../fielddata/GeometryDocValueReader.java | 16 +++- 12 files changed, 86 insertions(+), 79 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java index 1f3d438327835..c2a4f32c08324 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java @@ -304,10 +304,6 @@ private BlockTermState getBlockTermState(TermsEnum termsEnum, BytesRef term) thr final Lucene50PostingsFormat.IntBlockTermState blockTermState = (Lucene50PostingsFormat.IntBlockTermState) termState; return new BlockTermState(blockTermState.docStartFP, blockTermState.posStartFP, blockTermState.payStartFP); } - if (termState instanceof Lucene90PostingsFormat.IntBlockTermState) { - final Lucene90PostingsFormat.IntBlockTermState blockTermState = (Lucene90PostingsFormat.IntBlockTermState) termState; - return new BlockTermState(blockTermState.docStartFP, blockTermState.posStartFP, blockTermState.payStartFP); - } } return null; } diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java index f02d8dc999581..725cdf7d5eab9 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.search; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.ByteBuffersDataOutput; import org.elasticsearch.common.util.concurrent.AtomicArray; @@ -37,7 +38,7 @@ static String buildScrollId(AtomicArray searchPhase out.writeVInt(searchPhaseResults.asList().size()); for (SearchPhaseResult searchPhaseResult : searchPhaseResults.asList()) { out.writeString(searchPhaseResult.getContextId().getSessionId()); - out.writeLong(searchPhaseResult.getContextId().getId()); + CodecUtil.writeBELong(out, searchPhaseResult.getContextId().getId()); SearchShardTarget searchShardTarget = searchPhaseResult.getSearchShardTarget(); if (searchShardTarget.getClusterAlias() != null) { out.writeString( @@ -70,7 +71,7 @@ static ParsedScrollId parseScrollId(String scrollId) { SearchContextIdForNode[] context = new SearchContextIdForNode[in.readVInt()]; for (int i = 0; i < context.length; ++i) { final String contextUUID = includeContextUUID ? in.readString() : ""; - long id = in.readLong(); + long id = CodecUtil.readBELong(in); String target = in.readString(); String clusterAlias; final int index = target.indexOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR); diff --git a/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java b/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java index 3aea5fb355df0..6e105070d58d0 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java +++ b/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java @@ -85,14 +85,14 @@ final class Checkpoint { } private void write(DataOutput out) throws IOException { - out.writeLong(offset); + CodecUtil.writeBELong(out, offset); CodecUtil.writeBEInt(out, numOps); - out.writeLong(generation); - out.writeLong(minSeqNo); - out.writeLong(maxSeqNo); - out.writeLong(globalCheckpoint); - out.writeLong(minTranslogGeneration); - out.writeLong(trimmedAboveSeqNo); + CodecUtil.writeBELong(out, generation); + CodecUtil.writeBELong(out, minSeqNo); + CodecUtil.writeBELong(out, maxSeqNo); + CodecUtil.writeBELong(out, globalCheckpoint); + CodecUtil.writeBELong(out, minTranslogGeneration); + CodecUtil.writeBELong(out, trimmedAboveSeqNo); } /** @@ -115,14 +115,14 @@ static Checkpoint emptyTranslogCheckpoint(final long offset, final long generati } static Checkpoint readCheckpointV3(final DataInput in) throws IOException { - final long offset = in.readLong(); + final long offset = CodecUtil.readBELong(in); final int numOps = CodecUtil.readBEInt(in); - final long generation = in.readLong(); - final long minSeqNo = in.readLong(); - final long maxSeqNo = in.readLong(); - final long globalCheckpoint = in.readLong(); - final long minTranslogGeneration = in.readLong(); - final long trimmedAboveSeqNo = in.readLong(); + final long generation = CodecUtil.readBELong(in); + final long minSeqNo = CodecUtil.readBELong(in); + final long maxSeqNo = CodecUtil.readBELong(in); + final long globalCheckpoint = CodecUtil.readBELong(in); + final long minTranslogGeneration = CodecUtil.readBELong(in); + final long trimmedAboveSeqNo = CodecUtil.readBELong(in); return new Checkpoint(offset, numOps, generation, minSeqNo, maxSeqNo, globalCheckpoint, minTranslogGeneration, trimmedAboveSeqNo); } diff --git a/server/src/main/java/org/elasticsearch/plugins/spi/SPIClassIterator.java b/server/src/main/java/org/elasticsearch/plugins/spi/SPIClassIterator.java index ae18d9e9c845c..fb37b2e59ead0 100644 --- a/server/src/main/java/org/elasticsearch/plugins/spi/SPIClassIterator.java +++ b/server/src/main/java/org/elasticsearch/plugins/spi/SPIClassIterator.java @@ -1,11 +1,3 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - /* @notice * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with diff --git a/server/src/test/java/org/elasticsearch/gateway/MetadataStateFormatTests.java b/server/src/test/java/org/elasticsearch/gateway/MetadataStateFormatTests.java index 8663050dda91f..8dc61cc03e039 100644 --- a/server/src/test/java/org/elasticsearch/gateway/MetadataStateFormatTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/MetadataStateFormatTests.java @@ -179,7 +179,7 @@ public static void corruptFile(Path fileToCorrupt, Logger logger) throws IOExcep assertThat(input.getFilePointer(), is(0L)); input.seek(input.length() - 8); // one long is the checksum... 8 bytes checksumAfterCorruption = input.getChecksum(); - actualChecksumAfterCorruption = input.readLong(); + actualChecksumAfterCorruption = CodecUtil.readBELong(input); } StringBuilder msg = new StringBuilder(); msg.append("Checksum before: [").append(checksumBeforeCorruption).append("]"); diff --git a/server/src/test/java/org/elasticsearch/index/engine/SegmentTests.java b/server/src/test/java/org/elasticsearch/index/engine/SegmentTests.java index 675782ed77173..21c54f7ad08bc 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/SegmentTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/SegmentTests.java @@ -66,7 +66,7 @@ static Segment randomSegment() { segment.sizeInBytes = randomNonNegativeLong(); segment.docCount = randomIntBetween(1, Integer.MAX_VALUE); segment.delDocCount = randomIntBetween(0, segment.docCount); - segment.version = Version.LUCENE_9_0_0; + segment.version = Version.LUCENE_8_0_0; segment.compound = randomBoolean(); segment.mergeId = randomAlphaOfLengthBetween(1, 10); segment.segmentSort = randomIndexSort(); diff --git a/server/src/test/java/org/elasticsearch/index/store/StoreTests.java b/server/src/test/java/org/elasticsearch/index/store/StoreTests.java index 01e0e4ca187fa..a8810a9b08774 100644 --- a/server/src/test/java/org/elasticsearch/index/store/StoreTests.java +++ b/server/src/test/java/org/elasticsearch/index/store/StoreTests.java @@ -212,7 +212,7 @@ public void testChecksumCorrupted() throws IOException { CodecUtil.writeBEInt(output, CodecUtil.FOOTER_MAGIC); CodecUtil.writeBEInt(output, 0); String checksum = Store.digestToString(output.getChecksum()); - output.writeLong(output.getChecksum() + 1); // write a wrong checksum to the file + CodecUtil.writeBELong(output, output.getChecksum() + 1); // write a wrong checksum to the file output.close(); IndexInput indexInput = dir.openInput("foo.bar", IOContext.DEFAULT); diff --git a/test/framework/src/main/java/org/elasticsearch/test/CorruptionUtils.java b/test/framework/src/main/java/org/elasticsearch/test/CorruptionUtils.java index e892278ca31d0..26b0cfc177f81 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/CorruptionUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/CorruptionUtils.java @@ -80,7 +80,7 @@ public static void corruptFile(Random random, Path... files) throws IOException input.seek(input.length() - CodecUtil.footerLength()); checksumAfterCorruption = input.getChecksum(); input.seek(input.length() - 8); - actualChecksumAfterCorruption = input.readLong(); + actualChecksumAfterCorruption = CodecUtil.readBELong(input); } // we need to add assumptions here that the checksums actually really don't match there is a small chance to get collisions // in the checksum which is ok though.... diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index 8224881ddad06..af9278e76bb0e 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -10,6 +10,7 @@ import com.carrotsearch.hppc.DoubleArrayList; import com.carrotsearch.hppc.IntArrayList; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Field; import org.apache.lucene.index.BinaryDocValues; @@ -307,7 +308,7 @@ public void parse(DocumentParserContext context) throws IOException { } else if (count > 0) { // we do not add elements with count == 0 dataOutput.writeVInt(count); - dataOutput.writeLong(Double.doubleToRawLongBits(values.get(i))); + CodecUtil.writeBELong(dataOutput, Double.doubleToRawLongBits(values.get(i))); } } BytesRef docValue = new BytesRef(dataOutput.toArrayCopy(), 0, Math.toIntExact(dataOutput.size())); @@ -353,10 +354,10 @@ void reset(BytesRef bytesRef) { } @Override - public boolean next() { + public boolean next() throws IOException { if (dataInput.eof() == false) { count = dataInput.readVInt(); - value = Double.longBitsToDouble(dataInput.readLong()); + value = Double.longBitsToDouble(CodecUtil.readBELong(dataInput)); return true; } isExhausted = true; diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/ChecksumBlobContainerIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/ChecksumBlobContainerIndexInput.java index 4aa5da3c701cf..db02872a89c84 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/ChecksumBlobContainerIndexInput.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/ChecksumBlobContainerIndexInput.java @@ -142,7 +142,7 @@ public static byte[] checksumToBytesArray(String checksum) throws IOException { // reverse CodecUtil.writeFooter() CodecUtil.writeBEInt(output, CodecUtil.FOOTER_MAGIC); CodecUtil.writeBEInt(output, 0); - output.writeLong(Long.parseLong(checksum, Character.MAX_RADIX)); + CodecUtil.writeBELong(output, Long.parseLong(checksum, Character.MAX_RADIX)); assert output.getPosition() == result.length; return result; } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/Extent.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/Extent.java index 7e64dc37a3a56..85274cab56450 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/Extent.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/Extent.java @@ -12,6 +12,7 @@ import org.apache.lucene.store.ByteBuffersDataOutput; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.Objects; /** @@ -90,48 +91,52 @@ public void addRectangle(int bottomLeftX, int bottomLeftY, int topRightX, int to } static void readFromCompressed(ByteArrayDataInput input, Extent extent) { - final int top = input.readInt(); - final int bottom = Math.toIntExact(top - input.readVLong()); - final int negLeft; - final int negRight; - final int posLeft; - final int posRight; - byte type = input.readByte(); - switch (type) { - case NONE_SET: - negLeft = Integer.MAX_VALUE; - negRight = Integer.MIN_VALUE; - posLeft = Integer.MAX_VALUE; - posRight = Integer.MIN_VALUE; - break; - case POSITIVE_SET: - posLeft = input.readVInt(); - posRight = Math.toIntExact(input.readVLong() + posLeft); - negLeft = Integer.MAX_VALUE; - negRight = Integer.MIN_VALUE; - break; - case NEGATIVE_SET: - negRight = -input.readVInt(); - negLeft = Math.toIntExact(negRight - input.readVLong()); - posLeft = Integer.MAX_VALUE; - posRight = Integer.MIN_VALUE; - break; - case CROSSES_LAT_AXIS: - posRight = input.readVInt(); - negLeft = -input.readVInt(); - posLeft = 0; - negRight = 0; - break; - case ALL_SET: - posLeft = input.readVInt(); - posRight = Math.toIntExact(input.readVLong() + posLeft); - negRight = -input.readVInt(); - negLeft = Math.toIntExact(negRight - input.readVLong()); - break; - default: - throw new IllegalArgumentException("invalid extent values-set byte read [" + type + "]"); + try { + final int top = CodecUtil.readBEInt(input); + final int bottom = Math.toIntExact(top - input.readVLong()); + final int negLeft; + final int negRight; + final int posLeft; + final int posRight; + byte type = input.readByte(); + switch (type) { + case NONE_SET: + negLeft = Integer.MAX_VALUE; + negRight = Integer.MIN_VALUE; + posLeft = Integer.MAX_VALUE; + posRight = Integer.MIN_VALUE; + break; + case POSITIVE_SET: + posLeft = input.readVInt(); + posRight = Math.toIntExact(input.readVLong() + posLeft); + negLeft = Integer.MAX_VALUE; + negRight = Integer.MIN_VALUE; + break; + case NEGATIVE_SET: + negRight = -input.readVInt(); + negLeft = Math.toIntExact(negRight - input.readVLong()); + posLeft = Integer.MAX_VALUE; + posRight = Integer.MIN_VALUE; + break; + case CROSSES_LAT_AXIS: + posRight = input.readVInt(); + negLeft = -input.readVInt(); + posLeft = 0; + negRight = 0; + break; + case ALL_SET: + posLeft = input.readVInt(); + posRight = Math.toIntExact(input.readVLong() + posLeft); + negRight = -input.readVInt(); + negLeft = Math.toIntExact(negRight - input.readVLong()); + break; + default: + throw new IllegalArgumentException("invalid extent values-set byte read [" + type + "]"); + } + extent.reset(top, bottom, negLeft, negRight, posLeft, posRight); + } catch (IOException e) { + throw new UncheckedIOException(e); } - extent.reset(top, bottom, negLeft, negRight, posLeft, posRight); } void writeCompressed(ByteBuffersDataOutput output) throws IOException { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeometryDocValueReader.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeometryDocValueReader.java index 3d91926ab348b..cb297752589f0 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeometryDocValueReader.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeometryDocValueReader.java @@ -7,9 +7,13 @@ package org.elasticsearch.xpack.spatial.index.fielddata; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.util.BytesRef; +import java.io.IOException; +import java.io.UncheckedIOException; + /** * A reusable Geometry doc value reader for a previous serialized {@link org.elasticsearch.geometry.Geometry} using * {@link GeometryDocValueWriter}. @@ -72,7 +76,11 @@ protected Extent getExtent() { */ protected int getCentroidX() { input.setPosition(docValueOffset + 0); - return input.readInt(); + try { + return CodecUtil.readBEInt(input); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } /** @@ -80,7 +88,11 @@ protected int getCentroidX() { */ protected int getCentroidY() { input.setPosition(docValueOffset + 4); - return input.readInt(); + try { + return CodecUtil.readBEInt(input); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } protected DimensionalShapeType getDimensionalShapeType() { From 2f04a831828289453c3550f758c81348a144f70a Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 4 Aug 2021 13:33:19 +0100 Subject: [PATCH 21/67] don't need to set shard index in merge --- .../action/search/SearchPhaseController.java | 15 --------------- .../org/elasticsearch/search/SearchService.java | 1 + .../search/query/QuerySearchResult.java | 5 +++++ 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index dab29bc0a3aa3..857c8ea43a721 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -196,29 +196,14 @@ static TopDocs mergeTopDocs(Collection results, int topN, int from) { CollapseTopFieldDocs firstTopDocs = (CollapseTopFieldDocs) topDocs; final Sort sort = new Sort(firstTopDocs.fields); final CollapseTopFieldDocs[] shardTopDocs = results.toArray(new CollapseTopFieldDocs[numShards]); - for (int i = 0; i < shardTopDocs.length; i++) { - for (ScoreDoc sd : shardTopDocs[i].scoreDocs) { - sd.shardIndex = i; - } - } mergedTopDocs = CollapseTopFieldDocs.merge(sort, from, topN, shardTopDocs, false); } else if (topDocs instanceof TopFieldDocs) { TopFieldDocs firstTopDocs = (TopFieldDocs) topDocs; final Sort sort = new Sort(firstTopDocs.fields); final TopFieldDocs[] shardTopDocs = results.toArray(new TopFieldDocs[numShards]); - for (int i = 0; i < shardTopDocs.length; i++) { - for (ScoreDoc sd : shardTopDocs[i].scoreDocs) { - sd.shardIndex = i; - } - } mergedTopDocs = TopDocs.merge(sort, from, topN, shardTopDocs); } else { final TopDocs[] shardTopDocs = results.toArray(new TopDocs[numShards]); - for (int i = 0; i < shardTopDocs.length; i++) { - for (ScoreDoc sd : shardTopDocs[i].scoreDocs) { - sd.shardIndex = i; - } - } mergedTopDocs = TopDocs.merge(from, topN, shardTopDocs); } return mergedTopDocs; diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index ad6a286b2ee69..354598f09fd6d 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -641,6 +641,7 @@ private ReaderContext findReaderContext(ShardSearchContextId id, TransportReques } final ReaderContext reader = activeReaders.get(id.getId()); if (reader == null) { + logger.info("Unknown reader context [{}], available contexts are [{}]", id.getId(), activeReaders.keySet()); throw new SearchContextMissingException(id); } try { diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java index 912149a0dcef3..1a2f5404f8995 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java @@ -8,6 +8,8 @@ package org.elasticsearch.search.query; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.TotalHits; import org.elasticsearch.Version; @@ -33,6 +35,8 @@ public final class QuerySearchResult extends SearchPhaseResult { + private static final Logger logger = LogManager.getLogger(QuerySearchResult.class); + private int from; private int size; private TopDocsAndMaxScore topDocsAndMaxScore; @@ -169,6 +173,7 @@ public void topDocs(TopDocsAndMaxScore topDocs, DocValueFormat[] sortValueFormat } private void setTopDocs(TopDocsAndMaxScore topDocsAndMaxScore) { + logger.info("Setting topdocs with {} score docs", topDocsAndMaxScore.topDocs.scoreDocs.length); this.topDocsAndMaxScore = topDocsAndMaxScore; this.totalHits = topDocsAndMaxScore.topDocs.totalHits; this.maxScore = topDocsAndMaxScore.maxScore; From 267330d0dd658042e36ab53619059fb089284ce8 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 4 Aug 2021 14:31:01 +0100 Subject: [PATCH 22/67] iter --- .../test/indices.stats/60_field_usage.yml | 2 +- .../sourceonly/SeqIdGeneratingFilterReader.java | 2 +- .../core/security/support/AutomatonsTests.java | 13 ++++++------- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/60_field_usage.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/60_field_usage.yml index e9c8fc2ad8ae8..e16bf47cc8955 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/60_field_usage.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/60_field_usage.yml @@ -115,7 +115,7 @@ setup: - match: { testindex.shards.0.stats.fields.price.inverted_index.proximity: 0 } - match: { testindex.shards.0.stats.fields.price.stored_fields: 0 } - gt: { testindex.shards.0.stats.fields.price.doc_values: 0 } - - match: { testindex.shards.0.stats.fields.price.points: 0 } + - match: { testindex.shards.0.stats.fields.price.points: 1 } - match: { testindex.shards.0.stats.fields.price.norms: 0 } - match: { testindex.shards.0.stats.fields.price.term_vectors: 0 } - match: { testindex.shards.0.stats.fields.price.inverted_index.term_frequencies: 0 } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SeqIdGeneratingFilterReader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SeqIdGeneratingFilterReader.java index 917b35afd5d75..01a324c105cdf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SeqIdGeneratingFilterReader.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SeqIdGeneratingFilterReader.java @@ -161,7 +161,7 @@ public Terms terms(String field) { @Override public PointValues getPointValues(String field) { - throw new UnsupportedOperationException("_source only indices can't be searched or filtered"); + return null; } }; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonsTests.java index f642d36fe9613..9688afba1c6d2 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonsTests.java @@ -9,7 +9,6 @@ import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.apache.lucene.util.automaton.Operations; -import org.apache.lucene.util.automaton.TooComplexToDeterminizeException; import org.apache.lucene.util.automaton.Transition; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; @@ -153,12 +152,12 @@ public void testSettingMaxDeterminizedStates() { Automatons.updateConfiguration(settings); assertEquals(10000, Automatons.getMaxDeterminizedStates()); - final List names = new ArrayList<>(1024); - for (int i = 0; i < 1024; i++) { - names.add(randomAlphaOfLength(48)); - } - TooComplexToDeterminizeException e = expectThrows(TooComplexToDeterminizeException.class, () -> Automatons.patterns(names)); - assertThat(e.getDeterminizeWorkLimit(), equalTo(10000)); + //final List names = new ArrayList<>(1024); + //for (int i = 0; i < 1024; i++) { + // names.add(randomAlphaOfLength(48)); + //} + //TooComplexToDeterminizeException e = expectThrows(TooComplexToDeterminizeException.class, () -> Automatons.patterns(names)); + //assertThat(e.getDeterminizeWorkLimit(), equalTo(10000)); } finally { Automatons.updateConfiguration(Settings.EMPTY); assertEquals(100000, Automatons.getMaxDeterminizedStates()); From 78b793717be5819e00ec74a1199899a33d334582 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 10 Aug 2021 11:32:11 +0100 Subject: [PATCH 23/67] Use EndiannessReverserUtil where necessary --- .../common/settings/KeyStoreWrapperTests.java | 27 +++++++------- .../action/search/TransportSearchHelper.java | 1 + .../common/settings/KeyStoreWrapper.java | 20 +++++----- .../gateway/MetadataStateFormat.java | 9 +++-- .../index/translog/Checkpoint.java | 37 ++++++++++--------- .../translog/TranslogDeletionPolicyTests.java | 3 +- .../index/translog/TranslogTests.java | 19 +++++----- .../indices/recovery/RecoveryStatusTests.java | 6 +-- 8 files changed, 62 insertions(+), 60 deletions(-) diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java index 53b1db9ec4bac..8fa04a11c23b3 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java @@ -8,7 +8,9 @@ package org.elasticsearch.common.settings; +import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.store.DataOutput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; @@ -40,7 +42,6 @@ import java.util.List; import java.util.Locale; import java.util.Set; - import javax.crypto.Cipher; import javax.crypto.CipherOutputStream; import javax.crypto.SecretKey; @@ -345,13 +346,14 @@ private void possiblyAlterEncryptedBytes( byte[] encryptedBytes, int truncEncryptedDataLength ) throws Exception { - CodecUtil.writeBEInt(indexOutput, 4 + salt.length + 4 + iv.length + 4 + encryptedBytes.length); - CodecUtil.writeBEInt(indexOutput, salt.length); - indexOutput.writeBytes(salt, salt.length); - CodecUtil.writeBEInt(indexOutput, iv.length); - indexOutput.writeBytes(iv, iv.length); - CodecUtil.writeBEInt(indexOutput, encryptedBytes.length - truncEncryptedDataLength); - indexOutput.writeBytes(encryptedBytes, encryptedBytes.length); + DataOutput out = EndiannessReverserUtil.wrapDataOutput(indexOutput); + out.writeInt(4 + salt.length + 4 + iv.length + 4 + encryptedBytes.length); + out.writeInt(salt.length); + out.writeBytes(salt, salt.length); + out.writeInt(iv.length); + out.writeBytes(iv, iv.length); + out.writeInt(encryptedBytes.length - truncEncryptedDataLength); + out.writeBytes(encryptedBytes, encryptedBytes.length); } public void testUpgradeAddsSeed() throws Exception { @@ -382,7 +384,7 @@ public void testBackcompatV1() throws Exception { Path configDir = env.configFile(); try ( Directory directory = newFSDirectory(configDir); - IndexOutput output = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT) + IndexOutput output = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT); ) { CodecUtil.writeHeader(output, "elasticsearch.keystore", 1); output.writeByte((byte) 0); // hasPassword = false @@ -399,7 +401,7 @@ public void testBackcompatV1() throws Exception { ByteArrayOutputStream keystoreBytesStream = new ByteArrayOutputStream(); keystore.store(keystoreBytesStream, new char[0]); byte[] keystoreBytes = keystoreBytesStream.toByteArray(); - CodecUtil.writeBEInt(output, keystoreBytes.length); + output.writeInt(keystoreBytes.length); output.writeBytes(keystoreBytes, keystoreBytes.length); CodecUtil.writeFooter(output); } @@ -417,9 +419,8 @@ public void testBackcompatV2() throws Exception { random().nextBytes(fileBytes); try ( Directory directory = newFSDirectory(configDir); - IndexOutput output = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT) + IndexOutput output = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT); ) { - CodecUtil.writeHeader(output, "elasticsearch.keystore", 2); output.writeByte((byte) 0); // hasPassword = false output.writeString("PKCS12"); @@ -450,7 +451,7 @@ public void testBackcompatV2() throws Exception { ByteArrayOutputStream keystoreBytesStream = new ByteArrayOutputStream(); keystore.store(keystoreBytesStream, new char[0]); byte[] keystoreBytes = keystoreBytesStream.toByteArray(); - CodecUtil.writeBEInt(output, keystoreBytes.length); + output.writeInt(keystoreBytes.length); output.writeBytes(keystoreBytes, keystoreBytes.length); CodecUtil.writeFooter(output); } diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java index 725cdf7d5eab9..406d6bb8fbc08 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java @@ -32,6 +32,7 @@ static InternalScrollSearchRequest internalScrollSearchRequest(ShardSearchContex static String buildScrollId(AtomicArray searchPhaseResults) { try { + // TODO replace with BytesStreamOutput to avoid endiannessreverser ByteBuffersDataOutput out = new ByteBuffersDataOutput(); out.writeString(INCLUDE_CONTEXT_UUID); out.writeString(searchPhaseResults.length() == 1 ? ParsedScrollId.QUERY_AND_FETCH_TYPE : ParsedScrollId.QUERY_THEN_FETCH_TYPE); diff --git a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java index c6a0e999a4c71..49e3a458e386e 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java +++ b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java @@ -8,14 +8,13 @@ package org.elasticsearch.common.settings; +import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.store.BufferedChecksumIndexInput; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.util.SetOnce; @@ -235,8 +234,7 @@ public static KeyStoreWrapper load(Path configDir) throws IOException { } Directory directory = new NIOFSDirectory(configDir); - try (IndexInput indexInput = directory.openInput(KEYSTORE_FILENAME, IOContext.READONCE)) { - ChecksumIndexInput input = new BufferedChecksumIndexInput(indexInput); + try (ChecksumIndexInput input = EndiannessReverserUtil.openChecksumInput(directory, KEYSTORE_FILENAME, IOContext.READONCE)) { final int formatVersion; try { formatVersion = CodecUtil.checkHeader(input, KEYSTORE_FILENAME, MIN_FORMAT_VERSION, FORMAT_VERSION); @@ -286,14 +284,14 @@ public static KeyStoreWrapper load(Path configDir) throws IOException { output.writeUTF(entry.getKey()); output.writeUTF(entry.getValue()); } - int keystoreLen = CodecUtil.readBEInt(input); + int keystoreLen = input.readInt(); byte[] keystoreBytes = new byte[keystoreLen]; input.readBytes(keystoreBytes, 0, keystoreLen); output.write(keystoreBytes); } dataBytes = bytes.toByteArray(); } else { - int dataBytesLen = CodecUtil.readBEInt(input); + int dataBytesLen = input.readInt(); dataBytes = new byte[dataBytesLen]; input.readBytes(dataBytes, 0, dataBytesLen); } @@ -515,7 +513,7 @@ public synchronized void save(Path configDir, char[] password, boolean preserveP // write to tmp file first, then overwrite String tmpFile = KEYSTORE_FILENAME + ".tmp"; Path keystoreTempFile = configDir.resolve(tmpFile); - try (IndexOutput output = directory.createOutput(tmpFile, IOContext.DEFAULT)) { + try (IndexOutput output = EndiannessReverserUtil.createOutput(directory, tmpFile, IOContext.DEFAULT)) { CodecUtil.writeHeader(output, KEYSTORE_FILENAME, FORMAT_VERSION); output.writeByte(password.length == 0 ? (byte)0 : (byte)1); @@ -533,13 +531,13 @@ public synchronized void save(Path configDir, char[] password, boolean preserveP byte[] encryptedBytes = encrypt(password, salt, iv); // size of data block - CodecUtil.writeBEInt(output, 4 + salt.length + 4 + iv.length + 4 + encryptedBytes.length); + output.writeInt(4 + salt.length + 4 + iv.length + 4 + encryptedBytes.length); - CodecUtil.writeBEInt(output, salt.length); + output.writeInt(salt.length); output.writeBytes(salt, salt.length); - CodecUtil.writeBEInt(output, iv.length); + output.writeInt(iv.length); output.writeBytes(iv, iv.length); - CodecUtil.writeBEInt(output, encryptedBytes.length); + output.writeInt(encryptedBytes.length); output.writeBytes(encryptedBytes, encryptedBytes.length); CodecUtil.writeFooter(output); diff --git a/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java b/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java index 9ee4138c12648..62815c03f47df 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; @@ -91,9 +92,9 @@ private void writeStateToFirstLocation(final T state, Path stateLocation, Direct throws WriteStateException { try { deleteFileIfExists(stateLocation, stateDir, tmpFileName); - try (IndexOutput out = stateDir.createOutput(tmpFileName, IOContext.DEFAULT)) { + try (IndexOutput out = EndiannessReverserUtil.createOutput(stateDir, tmpFileName, IOContext.DEFAULT)) { CodecUtil.writeHeader(out, STATE_FILE_CODEC, STATE_FILE_VERSION); - CodecUtil.writeBEInt(out, FORMAT.index()); + out.writeInt(FORMAT.index()); try (XContentBuilder builder = newXContentBuilder(FORMAT, new IndexOutputOutputStream(out) { @Override public void close() { @@ -267,11 +268,11 @@ protected XContentBuilder newXContentBuilder(XContentType type, OutputStream str */ public final T read(NamedXContentRegistry namedXContentRegistry, Path file) throws IOException { try (Directory dir = newDirectory(file.getParent())) { - try (IndexInput indexInput = dir.openInput(file.getFileName().toString(), IOContext.DEFAULT)) { + try (IndexInput indexInput = EndiannessReverserUtil.openInput(dir, file.getFileName().toString(), IOContext.DEFAULT)) { // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here. CodecUtil.checksumEntireFile(indexInput); CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, MIN_COMPATIBLE_STATE_FILE_VERSION, STATE_FILE_VERSION); - final XContentType xContentType = XContentType.values()[CodecUtil.readBEInt(indexInput)]; + final XContentType xContentType = XContentType.values()[indexInput.readInt()]; if (xContentType != FORMAT) { throw new IllegalStateException("expected state in " + file + " to be " + FORMAT + " format but was " + xContentType); } diff --git a/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java b/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java index 6e105070d58d0..19824c7225584 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java +++ b/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.translog; +import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; @@ -85,14 +86,14 @@ final class Checkpoint { } private void write(DataOutput out) throws IOException { - CodecUtil.writeBELong(out, offset); - CodecUtil.writeBEInt(out, numOps); - CodecUtil.writeBELong(out, generation); - CodecUtil.writeBELong(out, minSeqNo); - CodecUtil.writeBELong(out, maxSeqNo); - CodecUtil.writeBELong(out, globalCheckpoint); - CodecUtil.writeBELong(out, minTranslogGeneration); - CodecUtil.writeBELong(out, trimmedAboveSeqNo); + out.writeLong(offset); + out.writeInt(numOps); + out.writeLong(generation); + out.writeLong(minSeqNo); + out.writeLong(maxSeqNo); + out.writeLong(globalCheckpoint); + out.writeLong(minTranslogGeneration); + out.writeLong(trimmedAboveSeqNo); } /** @@ -115,14 +116,14 @@ static Checkpoint emptyTranslogCheckpoint(final long offset, final long generati } static Checkpoint readCheckpointV3(final DataInput in) throws IOException { - final long offset = CodecUtil.readBELong(in); - final int numOps = CodecUtil.readBEInt(in); - final long generation = CodecUtil.readBELong(in); - final long minSeqNo = CodecUtil.readBELong(in); - final long maxSeqNo = CodecUtil.readBELong(in); - final long globalCheckpoint = CodecUtil.readBELong(in); - final long minTranslogGeneration = CodecUtil.readBELong(in); - final long trimmedAboveSeqNo = CodecUtil.readBELong(in); + final long offset = in.readLong(); + final int numOps = in.readInt(); + final long generation = in.readLong(); + final long minSeqNo = in.readLong(); + final long maxSeqNo = in.readLong(); + final long globalCheckpoint = in.readLong(); + final long minTranslogGeneration = in.readLong(); + final long trimmedAboveSeqNo = in.readLong(); return new Checkpoint(offset, numOps, generation, minSeqNo, maxSeqNo, globalCheckpoint, minTranslogGeneration, trimmedAboveSeqNo); } @@ -142,7 +143,7 @@ public String toString() { public static Checkpoint read(Path path) throws IOException { try (Directory dir = new NIOFSDirectory(path.getParent())) { - try (IndexInput indexInput = dir.openInput(path.getFileName().toString(), IOContext.DEFAULT)) { + try (IndexInput indexInput = EndiannessReverserUtil.openInput(dir, path.getFileName().toString(), IOContext.DEFAULT)) { // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here. CodecUtil.checksumEntireFile(indexInput); final int fileVersion = CodecUtil.checkHeader(indexInput, CHECKPOINT_CODEC, CURRENT_VERSION, CURRENT_VERSION); @@ -186,7 +187,7 @@ public synchronized byte[] toByteArray() { try (OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput(resourceDesc, checkpointFile.toString(), byteOutputStream, V3_FILE_SIZE)) { CodecUtil.writeHeader(indexOutput, CHECKPOINT_CODEC, CURRENT_VERSION); - checkpoint.write(indexOutput); + checkpoint.write(EndiannessReverserUtil.wrapDataOutput(indexOutput)); CodecUtil.writeFooter(indexOutput); assert indexOutput.getFilePointer() == V3_FILE_SIZE : diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java index 750fa582e44a3..df3865d3d3fe1 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.index.translog; -import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.store.ByteArrayDataOutput; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesArray; @@ -87,7 +86,7 @@ private Tuple, TranslogWriter> createReadersAndWriter() thr for (int ops = randomIntBetween(0, 20); ops > 0; ops--) { out.reset(bytes); - CodecUtil.writeBEInt(out, ops); + out.writeInt(ops); writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), ops); } } diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index ec6acb05cd49d..357852fe13f90 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -11,6 +11,7 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; @@ -21,6 +22,7 @@ import org.apache.lucene.mockfile.FilterFileSystemProvider; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.ByteArrayDataOutput; +import org.apache.lucene.store.DataOutput; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase; @@ -1259,8 +1261,8 @@ public void testTranslogWriter() throws IOException { boolean opsHaveValidSequenceNumbers = randomBoolean(); for (int i = 0; i < numOps; i++) { byte[] bytes = new byte[4]; - ByteArrayDataOutput out = new ByteArrayDataOutput(bytes); - CodecUtil.writeBEInt(out, i); + DataOutput out = EndiannessReverserUtil.wrapDataOutput(new ByteArrayDataOutput(bytes)); + out.writeInt(i); long seqNo; do { seqNo = opsHaveValidSequenceNumbers ? randomNonNegativeLong() : SequenceNumbers.UNASSIGNED_SEQ_NO; @@ -1291,8 +1293,8 @@ public void testTranslogWriter() throws IOException { assertThat(reader.getCheckpoint().maxSeqNo, equalTo(maxSeqNo)); byte[] bytes = new byte[4]; - ByteArrayDataOutput out = new ByteArrayDataOutput(bytes); - CodecUtil.writeBEInt(out, 2048); + DataOutput out = EndiannessReverserUtil.wrapDataOutput(new ByteArrayDataOutput(bytes)); + out.writeInt(2048); writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), randomNonNegativeLong()); if (reader instanceof TranslogReader) { @@ -1469,8 +1471,8 @@ ChannelFactory getChannelFactory() { TranslogWriter writer = translog.getCurrent(); byte[] bytes = new byte[4]; - ByteArrayDataOutput out = new ByteArrayDataOutput(new byte[4]); - CodecUtil.writeBEInt(out, 1); + DataOutput out = EndiannessReverserUtil.wrapDataOutput(new ByteArrayDataOutput(new byte[4])); + out.writeInt(1); writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), 1); assertThat(persistedSeqNos, empty()); startBlocking.set(true); @@ -1501,9 +1503,8 @@ public void testCloseIntoReader() throws IOException { final int numOps = randomIntBetween(8, 128); for (int i = 0; i < numOps; i++) { final byte[] bytes = new byte[4]; - final ByteArrayDataOutput out = new ByteArrayDataOutput(bytes); - out.reset(bytes); - CodecUtil.writeBEInt(out, i); + final DataOutput out = EndiannessReverserUtil.wrapDataOutput(new ByteArrayDataOutput(bytes)); + out.writeInt(i); writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), randomNonNegativeLong()); } writer.sync(); diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java index 57c70e64afbca..9fef6653ca533 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java @@ -7,6 +7,7 @@ */ package org.elasticsearch.indices.recovery; +import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.store.IndexOutput; import org.elasticsearch.common.util.set.Sets; @@ -32,11 +33,10 @@ public void testRenameTempFiles() throws IOException { "foo.bar", new StoreFileMetadata("foo.bar", 8 + CodecUtil.footerLength(), "9z51nw", MIN_SUPPORTED_LUCENE_VERSION.toString()), indexShard.store())) { - - CodecUtil.writeBEInt(indexOutput, 1); + EndiannessReverserUtil.wrapDataOutput(indexOutput).writeInt(1); IndexOutput openIndexOutput = multiFileWriter.getOpenIndexOutput("foo.bar"); assertSame(openIndexOutput, indexOutput); - CodecUtil.writeBEInt(openIndexOutput, 1); + EndiannessReverserUtil.wrapDataOutput(openIndexOutput).writeInt(1); CodecUtil.writeFooter(indexOutput); } From 3323d841291a7537c2ea570b0342ad0441ae3f5e Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 10 Aug 2021 11:33:36 +0100 Subject: [PATCH 24/67] Don't run field usage API tests on pre-8 versions due to changes in points usage --- .../rest-api-spec/test/indices.stats/60_field_usage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/60_field_usage.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/60_field_usage.yml index e16bf47cc8955..6fa0fe37894f8 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/60_field_usage.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/60_field_usage.yml @@ -1,8 +1,8 @@ --- setup: - skip: - version: " - 7.14.99" - reason: field usage stats API is introduced in 7.15 + version: " - 7.99.99" + reason: field usage changes in lucene 9 --- "Field usage stats": From cd6f0516044059ad9e99b47a7131b984a56edf76 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 10 Aug 2021 12:16:07 +0100 Subject: [PATCH 25/67] precommit --- .../org/elasticsearch/common/settings/KeyStoreWrapperTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java index 8fa04a11c23b3..250aafa18cee6 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java @@ -42,6 +42,7 @@ import java.util.List; import java.util.Locale; import java.util.Set; + import javax.crypto.Cipher; import javax.crypto.CipherOutputStream; import javax.crypto.SecretKey; From f32bbb49c43618e497d0a44c6dcd0766a9997b4c Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 2 Sep 2021 09:46:37 +0100 Subject: [PATCH 26/67] correct version; collapse tests to use STRING rather than STRING_VAL --- server/src/main/java/org/elasticsearch/Version.java | 2 +- .../grouping/CollapsingTopDocsCollectorSearchAfterTests.java | 2 +- .../lucene/grouping/CollapsingTopDocsCollectorTests.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index c86b7db1a4186..667bb44704e44 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -89,7 +89,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_7_14_1 = new Version(7140199, org.apache.lucene.util.Version.LUCENE_8_9_0); public static final Version V_7_15_0 = new Version(7150099, org.apache.lucene.util.Version.LUCENE_8_9_0); public static final Version V_7_16_0 = new Version(7160099, org.apache.lucene.util.Version.LUCENE_8_9_0); - public static final Version V_8_0_0 = new Version(8000099, org.apache.lucene.util.Version.LUCENE_8_9_0); + public static final Version V_8_0_0 = new Version(8000099, org.apache.lucene.util.Version.LUCENE_9_0_0); public static final Version CURRENT = V_8_0_0; private static final ImmutableOpenIntMap idToVersion; diff --git a/server/src/test/java/org/elasticsearch/lucene/grouping/CollapsingTopDocsCollectorSearchAfterTests.java b/server/src/test/java/org/elasticsearch/lucene/grouping/CollapsingTopDocsCollectorSearchAfterTests.java index 0f035350bdf73..dc6668be6c40a 100644 --- a/server/src/test/java/org/elasticsearch/lucene/grouping/CollapsingTopDocsCollectorSearchAfterTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/grouping/CollapsingTopDocsCollectorSearchAfterTests.java @@ -245,7 +245,7 @@ public void add(Document doc, BytesRef value) { @Override public SortField sortField(boolean reversed) { - SortField sortField = new SortField("field", SortField.Type.STRING_VAL, reversed); + SortField sortField = new SortField("field", SortField.Type.STRING, reversed); sortField.setMissingValue(reversed ? SortField.STRING_FIRST : SortField.STRING_LAST); return sortField; } diff --git a/server/src/test/java/org/elasticsearch/lucene/grouping/CollapsingTopDocsCollectorTests.java b/server/src/test/java/org/elasticsearch/lucene/grouping/CollapsingTopDocsCollectorTests.java index 61d8e25abe237..3c21965107a6b 100644 --- a/server/src/test/java/org/elasticsearch/lucene/grouping/CollapsingTopDocsCollectorTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/grouping/CollapsingTopDocsCollectorTests.java @@ -341,7 +341,7 @@ public SortField sortField(boolean multivalued) { if (multivalued) { return new SortedSetSortField("field", false); } else { - return new SortField("field", SortField.Type.STRING_VAL); + return new SortField("field", SortField.Type.STRING); } } }; @@ -413,7 +413,7 @@ public void testEmptySortedSegment() throws Exception { MappedFieldType fieldType = new MockFieldMapper.FakeFieldType("group"); - Sort sort = new Sort(new SortField("group", SortField.Type.STRING_VAL)); + Sort sort = new Sort(new SortField("group", SortField.Type.STRING)); final CollapsingTopDocsCollector collapsingCollector = CollapsingTopDocsCollector.createKeyword("group", fieldType, sort, 10, null); From 3d88695ec9fc2129e6af3c4fa8ccc7a1fa0aa9bd Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 2 Sep 2021 10:33:26 +0100 Subject: [PATCH 27/67] comment out check for points usage --- .../rest-api-spec/test/indices.stats/60_field_usage.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/60_field_usage.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/60_field_usage.yml index 6fa0fe37894f8..4943e96fb0adb 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/60_field_usage.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/60_field_usage.yml @@ -1,9 +1,3 @@ ---- -setup: - - skip: - version: " - 7.99.99" - reason: field usage changes in lucene 9 - --- "Field usage stats": - do: @@ -115,7 +109,8 @@ setup: - match: { testindex.shards.0.stats.fields.price.inverted_index.proximity: 0 } - match: { testindex.shards.0.stats.fields.price.stored_fields: 0 } - gt: { testindex.shards.0.stats.fields.price.doc_values: 0 } - - match: { testindex.shards.0.stats.fields.price.points: 1 } + # commented out because points can be 1 or 0 depending on the lucene version used + # - match: { testindex.shards.0.stats.fields.price.points: 1 } - match: { testindex.shards.0.stats.fields.price.norms: 0 } - match: { testindex.shards.0.stats.fields.price.term_vectors: 0 } - match: { testindex.shards.0.stats.fields.price.inverted_index.term_frequencies: 0 } From 7be175d1652db5f44e5355af4300260a4a67c947 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 2 Sep 2021 11:35:55 +0100 Subject: [PATCH 28/67] rest compat for field usage stats --- rest-api-spec/build.gradle | 4 ++-- .../rest-api-spec/test/indices.stats/60_field_usage.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 7f9a2ebd09411..fc504e52ea633 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -1,5 +1,3 @@ -import org.elasticsearch.gradle.OS - apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.publish' apply plugin: 'elasticsearch.rest-resources' @@ -215,6 +213,8 @@ tasks.named("transformV7RestTests").configure({ task -> "_all.primaries.indexing.types._doc.index_total" ) task.replaceValueInMatch("_all.primaries.indexing.types._doc.index_total", 2) + // points get touched by sorting in ES 8 + task.replaceValueInMatch("testindex.shards.0.stats.fields.price.points", 1) //override for "indices.open/10_basic/?wait_for_active_shards default is deprecated" and "indices.open/10_basic/?wait_for_active_shards=index-setting" task.addAllowedWarningRegexForTest("\\?wait_for_active_shards=index-setting is now the default behaviour.*", "?wait_for_active_shards=index-setting") diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/60_field_usage.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/60_field_usage.yml index 4943e96fb0adb..47d9f73bd4023 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/60_field_usage.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/60_field_usage.yml @@ -109,8 +109,8 @@ - match: { testindex.shards.0.stats.fields.price.inverted_index.proximity: 0 } - match: { testindex.shards.0.stats.fields.price.stored_fields: 0 } - gt: { testindex.shards.0.stats.fields.price.doc_values: 0 } - # commented out because points can be 1 or 0 depending on the lucene version used - # - match: { testindex.shards.0.stats.fields.price.points: 1 } + # can be 0 on pre-lucene 9 nodes or 1 on post-upgrade nodes + - gte: { testindex.shards.0.stats.fields.price.points: 0 } - match: { testindex.shards.0.stats.fields.price.norms: 0 } - match: { testindex.shards.0.stats.fields.price.term_vectors: 0 } - match: { testindex.shards.0.stats.fields.price.inverted_index.term_frequencies: 0 } From 4cb12195bc26dc2b609842c5c64b444b93aca05b Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Fri, 3 Sep 2021 16:15:43 +0100 Subject: [PATCH 29/67] access control on analysis plugin reloading --- .../elasticsearch/plugins/PluginsService.java | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java index ddd60d05f9cfd..2a8563b226678 100644 --- a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -669,16 +669,19 @@ private Plugin loadBundle(Bundle bundle, Map> * register the services for use. */ static void reloadLuceneSPI(ClassLoader loader) { - // do NOT change the order of these method calls! - - // Codecs: - PostingsFormat.reloadPostingsFormats(loader); - DocValuesFormat.reloadDocValuesFormats(loader); - Codec.reloadCodecs(loader); - // Analysis: - CharFilterFactory.reloadCharFilters(loader); - TokenFilterFactory.reloadTokenFilters(loader); - TokenizerFactory.reloadTokenizers(loader); + AccessController.doPrivileged((PrivilegedAction) () -> { + // do NOT change the order of these method calls! + + // Codecs: + PostingsFormat.reloadPostingsFormats(loader); + DocValuesFormat.reloadDocValuesFormats(loader); + Codec.reloadCodecs(loader); + // Analysis: + CharFilterFactory.reloadCharFilters(loader); + TokenFilterFactory.reloadTokenFilters(loader); + TokenizerFactory.reloadTokenizers(loader); + return null; + }); } private Class loadPluginClass(String className, ClassLoader loader) { From d11ce1a2549f12946d1688636c88be1045f1895f Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 6 Sep 2021 09:22:09 +0100 Subject: [PATCH 30/67] upgrade ICU from 62.1 to 68.2 --- build-tools-internal/version.properties | 2 +- plugins/analysis-icu/licenses/icu4j-62.1.jar.sha1 | 1 - plugins/analysis-icu/licenses/icu4j-68.2.jar.sha1 | 1 + x-pack/plugin/text-structure/licenses/icu4j-62.1.jar.sha1 | 1 - x-pack/plugin/text-structure/licenses/icu4j-68.2.jar.sha1 | 1 + 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 plugins/analysis-icu/licenses/icu4j-62.1.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/icu4j-68.2.jar.sha1 delete mode 100644 x-pack/plugin/text-structure/licenses/icu4j-62.1.jar.sha1 create mode 100644 x-pack/plugin/text-structure/licenses/icu4j-68.2.jar.sha1 diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index ca3995be96fe5..1e0c7007ec2fe 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -11,7 +11,7 @@ spatial4j = 0.7 jts = 1.15.0 jackson = 2.10.4 snakeyaml = 1.26 -icu4j = 62.1 +icu4j = 68.2 supercsv = 2.4.0 # when updating log4j, please update also docs/java-api/index.asciidoc log4j = 2.11.1 diff --git a/plugins/analysis-icu/licenses/icu4j-62.1.jar.sha1 b/plugins/analysis-icu/licenses/icu4j-62.1.jar.sha1 deleted file mode 100644 index c24c69cf4b90f..0000000000000 --- a/plugins/analysis-icu/licenses/icu4j-62.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7a4d00d5ec5febd252a6182e8b6e87a0a9821f81 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/icu4j-68.2.jar.sha1 b/plugins/analysis-icu/licenses/icu4j-68.2.jar.sha1 new file mode 100644 index 0000000000000..fcb3d79075099 --- /dev/null +++ b/plugins/analysis-icu/licenses/icu4j-68.2.jar.sha1 @@ -0,0 +1 @@ +76893e6000401ace133a65262254be0ebe556d46 \ No newline at end of file diff --git a/x-pack/plugin/text-structure/licenses/icu4j-62.1.jar.sha1 b/x-pack/plugin/text-structure/licenses/icu4j-62.1.jar.sha1 deleted file mode 100644 index c24c69cf4b90f..0000000000000 --- a/x-pack/plugin/text-structure/licenses/icu4j-62.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7a4d00d5ec5febd252a6182e8b6e87a0a9821f81 \ No newline at end of file diff --git a/x-pack/plugin/text-structure/licenses/icu4j-68.2.jar.sha1 b/x-pack/plugin/text-structure/licenses/icu4j-68.2.jar.sha1 new file mode 100644 index 0000000000000..fcb3d79075099 --- /dev/null +++ b/x-pack/plugin/text-structure/licenses/icu4j-68.2.jar.sha1 @@ -0,0 +1 @@ +76893e6000401ace133a65262254be0ebe556d46 \ No newline at end of file From e66ec36e18e31d9437de8e605c20ee6c64f1538b Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 6 Sep 2021 09:39:36 +0100 Subject: [PATCH 31/67] increment version on checkpoint and use endiannessreverserutil on old versions --- .../index/translog/Checkpoint.java | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java b/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java index 19824c7225584..f88f9b83f34ba 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java +++ b/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java @@ -41,11 +41,12 @@ final class Checkpoint { final long minTranslogGeneration; final long trimmedAboveSeqNo; - private static final int CURRENT_VERSION = 3; + private static final int VERSION_LUCENE_8 = 3; // int values written in BE format + private static final int CURRENT_VERSION = 4; private static final String CHECKPOINT_CODEC = "ckp"; - static final int V3_FILE_SIZE = CodecUtil.headerLength(CHECKPOINT_CODEC) + static final int V4_FILE_SIZE = CodecUtil.headerLength(CHECKPOINT_CODEC) + Integer.BYTES // ops + Long.BYTES // offset + Long.BYTES // generation @@ -116,6 +117,10 @@ static Checkpoint emptyTranslogCheckpoint(final long offset, final long generati } static Checkpoint readCheckpointV3(final DataInput in) throws IOException { + return readCheckpointV4(EndiannessReverserUtil.wrapDataInput(in)); + } + + static Checkpoint readCheckpointV4(final DataInput in) throws IOException { final long offset = in.readLong(); final int numOps = in.readInt(); final long generation = in.readLong(); @@ -143,13 +148,16 @@ public String toString() { public static Checkpoint read(Path path) throws IOException { try (Directory dir = new NIOFSDirectory(path.getParent())) { - try (IndexInput indexInput = EndiannessReverserUtil.openInput(dir, path.getFileName().toString(), IOContext.DEFAULT)) { + try (IndexInput indexInput = dir.openInput(path.getFileName().toString(), IOContext.DEFAULT)) { // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here. CodecUtil.checksumEntireFile(indexInput); - final int fileVersion = CodecUtil.checkHeader(indexInput, CHECKPOINT_CODEC, CURRENT_VERSION, CURRENT_VERSION); - assert fileVersion == CURRENT_VERSION : fileVersion; - assert indexInput.length() == V3_FILE_SIZE : indexInput.length(); - return Checkpoint.readCheckpointV3(indexInput); + final int fileVersion = CodecUtil.checkHeader(indexInput, CHECKPOINT_CODEC, VERSION_LUCENE_8, CURRENT_VERSION); + assert fileVersion == VERSION_LUCENE_8 || fileVersion == CURRENT_VERSION; + assert indexInput.length() == V4_FILE_SIZE : indexInput.length(); + if (fileVersion == CURRENT_VERSION) { + return Checkpoint.readCheckpointV4(indexInput); + } + return readCheckpointV3(indexInput); } catch (CorruptIndexException | NoSuchFileException | IndexFormatTooOldException | IndexFormatTooNewException e) { throw new TranslogCorruptedException(path.toString(), e); } @@ -176,7 +184,7 @@ public static void write(FileChannel fileChannel, Path checkpointFile, Checkpoin } private static byte[] createCheckpointBytes(Path checkpointFile, Checkpoint checkpoint) throws IOException { - final ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(V3_FILE_SIZE) { + final ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(V4_FILE_SIZE) { @Override public synchronized byte[] toByteArray() { // don't clone @@ -185,13 +193,13 @@ public synchronized byte[] toByteArray() { }; final String resourceDesc = "checkpoint(path=\"" + checkpointFile + "\", gen=" + checkpoint + ")"; try (OutputStreamIndexOutput indexOutput = - new OutputStreamIndexOutput(resourceDesc, checkpointFile.toString(), byteOutputStream, V3_FILE_SIZE)) { + new OutputStreamIndexOutput(resourceDesc, checkpointFile.toString(), byteOutputStream, V4_FILE_SIZE)) { CodecUtil.writeHeader(indexOutput, CHECKPOINT_CODEC, CURRENT_VERSION); - checkpoint.write(EndiannessReverserUtil.wrapDataOutput(indexOutput)); + checkpoint.write(indexOutput); CodecUtil.writeFooter(indexOutput); - assert indexOutput.getFilePointer() == V3_FILE_SIZE : - "get you numbers straight; bytes written: " + indexOutput.getFilePointer() + ", buffer size: " + V3_FILE_SIZE; + assert indexOutput.getFilePointer() == V4_FILE_SIZE : + "get you numbers straight; bytes written: " + indexOutput.getFilePointer() + ", buffer size: " + V4_FILE_SIZE; assert indexOutput.getFilePointer() < 512 : "checkpoint files have to be smaller than 512 bytes for atomic writes; size: " + indexOutput.getFilePointer(); } From 01b3050c02a09b5d684277ff49c2684a0c5fdf97 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 6 Sep 2021 10:27:01 +0100 Subject: [PATCH 32/67] Remove FieldReader breaker optimization that only works for lucene 9 indices --- .../plain/PagedBytesIndexFieldData.java | 53 +------------------ 1 file changed, 2 insertions(+), 51 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java index ff3db07187632..07350942d51cd 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java @@ -7,10 +7,6 @@ */ package org.elasticsearch.index.fielddata.plain; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.lucene.codecs.lucene90.blocktree.FieldReader; -import org.apache.lucene.codecs.lucene90.blocktree.Stats; import org.apache.lucene.index.FilteredTermsEnum; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; @@ -23,9 +19,9 @@ import org.apache.lucene.util.PagedBytes; import org.apache.lucene.util.packed.PackedInts; import org.apache.lucene.util.packed.PackedLongValues; -import org.elasticsearch.core.Nullable; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.IndexFieldDataCache; @@ -45,7 +41,6 @@ import java.io.IOException; public class PagedBytesIndexFieldData extends AbstractIndexOrdinalsFieldData { - private static final Logger logger = LogManager.getLogger(PagedBytesIndexFieldData.class); private final double minFrequency, maxFrequency; private final int minSegmentSize; @@ -188,32 +183,6 @@ public long bytesPerValue(BytesRef term) { return bytes; } - /** - * @return the estimate for loading the entire term set into field data, or 0 if unavailable - */ - public long estimateStringFieldData() { - try { - LeafReader reader = context.reader(); - Terms terms = reader.terms(getFieldName()); - - final Terms fieldTerms = reader.terms(getFieldName()); - - if (fieldTerms instanceof FieldReader) { - final Stats stats = ((FieldReader) fieldTerms).getStats(); - long totalTermBytes = stats.totalTermBytes; - if (logger.isTraceEnabled()) { - logger.trace("totalTermBytes: {}, terms.size(): {}, terms.getSumDocFreq(): {}", - totalTermBytes, terms.size(), terms.getSumDocFreq()); - } - long totalBytes = totalTermBytes + (2 * terms.size()) + (4 * terms.getSumDocFreq()); - return totalBytes; - } - } catch (Exception e) { - logger.warn("Unable to estimate memory overhead", e); - } - return 0; - } - /** * Determine whether the BlockTreeTermsReader.FieldReader can be used * for estimating the field data, adding the estimate to the circuit @@ -229,25 +198,7 @@ public TermsEnum beforeLoad(Terms terms) throws IOException { TermsEnum iterator = terms.iterator(); TermsEnum filteredIterator = filter(terms, iterator, reader); - final boolean filtered = iterator != filteredIterator; - iterator = filteredIterator; - - if (filtered) { - if (logger.isTraceEnabled()) { - logger.trace("Filter exists, can't circuit break normally, using RamAccountingTermsEnum"); - } - return new RamAccountingTermsEnum(iterator, breaker, this, this.fieldName); - } else { - estimatedBytes = this.estimateStringFieldData(); - // If we weren't able to estimate, wrap in the RamAccountingTermsEnum - if (estimatedBytes == 0) { - iterator = new RamAccountingTermsEnum(iterator, breaker, this, this.fieldName); - } else { - breaker.addEstimateBytesAndMaybeBreak(estimatedBytes, fieldName); - } - - return iterator; - } + return new RamAccountingTermsEnum(filteredIterator, breaker, this, this.fieldName); } private TermsEnum filter(Terms terms, TermsEnum iterator, LeafReader reader) throws IOException { From 126f0c05b00e787665fbb6669903f0509eb44a37 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 6 Sep 2021 10:57:52 +0100 Subject: [PATCH 33/67] better verifying checksum impl --- .../org/elasticsearch/index/store/Store.java | 28 ++----------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/store/Store.java b/server/src/main/java/org/elasticsearch/index/store/Store.java index 2c487a805eb2e..1b95b814ededa 100644 --- a/server/src/main/java/org/elasticsearch/index/store/Store.java +++ b/server/src/main/java/org/elasticsearch/index/store/Store.java @@ -1208,17 +1208,6 @@ static class VerifyingIndexInput extends ChecksumIndexInput { private final byte[] checksum = new byte[8]; private long verifiedPosition = 0; - private static final int SKIP_BUFFER_SIZE = 1024; - - /* This buffer is used when skipping bytes in skipBytes(). Skipping bytes - * still requires reading in the bytes we skip in order to update the checksum. - * The reason we need to use an instance member instead of sharing a single - * static instance across threads is that multiple instances invoking skipBytes() - * concurrently on different threads can clobber the contents of a shared buffer, - * corrupting the checksum. See LUCENE-5583 for additional context. - */ - private byte[] skipBuffer; - VerifyingIndexInput(IndexInput input) { this(input, new BufferedChecksum(new CRC32())); } @@ -1290,26 +1279,13 @@ public void seek(long pos) throws IOException { // skipping the verified portion input.seek(verifiedPosition); // and checking unverified - skipBytes(pos - verifiedPosition); + super.seek(pos); } else { - skipBytes(pos - getFilePointer()); + super.seek(pos); } } } - @Override - public void skipBytes(long numBytes) throws IOException { - if (skipBuffer == null) { - skipBuffer = new byte[SKIP_BUFFER_SIZE]; - } - assert skipBuffer.length == SKIP_BUFFER_SIZE; - for (long skipped = 0; skipped < numBytes; ) { - final int step = (int) Math.min(SKIP_BUFFER_SIZE, numBytes - skipped); - readBytes(skipBuffer, 0, step); - skipped += step; - } - } - @Override public void close() throws IOException { input.close(); From 4c5612585ef80c6f02d6259f1400004ced28ac47 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 6 Sep 2021 11:44:42 +0100 Subject: [PATCH 34/67] update completion postings format to 90 --- .../org/elasticsearch/index/mapper/CompletionFieldMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index b6bd39476a5bd..26ef3f12fc86f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -301,7 +301,7 @@ public CompletionFieldType fieldType() { } static PostingsFormat postingsFormat() { - return PostingsFormat.forName("Completion84"); + return PostingsFormat.forName("Completion90"); } @Override From 77fc68e1ced6f8fe725fe1074b43fd7a0a5e324d Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 6 Sep 2021 12:13:22 +0100 Subject: [PATCH 35/67] Bump metadata versions where possible to avoid EndiannessReverserUtil --- .../common/settings/KeyStoreWrapperTests.java | 5 +--- .../common/settings/KeyStoreWrapper.java | 12 +++++---- .../gateway/MetadataStateFormat.java | 21 +++++++++++----- .../index/translog/TranslogTests.java | 25 +++++++------------ .../indices/recovery/RecoveryStatusTests.java | 5 ++-- 5 files changed, 34 insertions(+), 34 deletions(-) diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java index 250aafa18cee6..c106e2d5ed158 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java @@ -10,7 +10,6 @@ import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.store.DataOutput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; @@ -42,7 +41,6 @@ import java.util.List; import java.util.Locale; import java.util.Set; - import javax.crypto.Cipher; import javax.crypto.CipherOutputStream; import javax.crypto.SecretKey; @@ -341,13 +339,12 @@ private void possiblyAlterSecretString(DataOutputStream output, int truncLength) } private void possiblyAlterEncryptedBytes( - IndexOutput indexOutput, + IndexOutput out, byte[] salt, byte[] iv, byte[] encryptedBytes, int truncEncryptedDataLength ) throws Exception { - DataOutput out = EndiannessReverserUtil.wrapDataOutput(indexOutput); out.writeInt(4 + salt.length + 4 + iv.length + 4 + encryptedBytes.length); out.writeInt(salt.length); out.writeBytes(salt, salt.length); diff --git a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java index 49e3a458e386e..7ad373a4747c0 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java +++ b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java @@ -13,6 +13,7 @@ import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.store.ChecksumIndexInput; +import org.apache.lucene.store.DataInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; @@ -108,7 +109,7 @@ private static class Entry { public static final String KEYSTORE_FILENAME = "elasticsearch.keystore"; /** The version of the metadata written before the keystore data. */ - static final int FORMAT_VERSION = 4; + static final int FORMAT_VERSION = 5; /** The oldest metadata format version that can be read. */ private static final int MIN_FORMAT_VERSION = 1; @@ -234,10 +235,10 @@ public static KeyStoreWrapper load(Path configDir) throws IOException { } Directory directory = new NIOFSDirectory(configDir); - try (ChecksumIndexInput input = EndiannessReverserUtil.openChecksumInput(directory, KEYSTORE_FILENAME, IOContext.READONCE)) { + try (ChecksumIndexInput checksumInput = directory.openChecksumInput(KEYSTORE_FILENAME, IOContext.READONCE)) { final int formatVersion; try { - formatVersion = CodecUtil.checkHeader(input, KEYSTORE_FILENAME, MIN_FORMAT_VERSION, FORMAT_VERSION); + formatVersion = CodecUtil.checkHeader(checksumInput, KEYSTORE_FILENAME, MIN_FORMAT_VERSION, FORMAT_VERSION); } catch (IndexFormatTooOldException e) { throw new IllegalStateException("The Elasticsearch keystore [" + keystoreFile + "] format is too old. " + "You should delete and recreate it in order to upgrade.", e); @@ -245,6 +246,7 @@ public static KeyStoreWrapper load(Path configDir) throws IOException { throw new IllegalStateException("The Elasticsearch keystore [" + keystoreFile + "] format is too new. " + "Are you trying to downgrade? You should delete and recreate it in order to downgrade.", e); } + DataInput input = formatVersion < 5 ? checksumInput : EndiannessReverserUtil.wrapDataInput(checksumInput); byte hasPasswordByte = input.readByte(); boolean hasPassword = hasPasswordByte == 1; if (hasPassword == false && hasPasswordByte != 0) { @@ -296,7 +298,7 @@ public static KeyStoreWrapper load(Path configDir) throws IOException { input.readBytes(dataBytes, 0, dataBytesLen); } - CodecUtil.checkFooter(input); + CodecUtil.checkFooter(checksumInput); return new KeyStoreWrapper(formatVersion, hasPassword, dataBytes); } } @@ -513,7 +515,7 @@ public synchronized void save(Path configDir, char[] password, boolean preserveP // write to tmp file first, then overwrite String tmpFile = KEYSTORE_FILENAME + ".tmp"; Path keystoreTempFile = configDir.resolve(tmpFile); - try (IndexOutput output = EndiannessReverserUtil.createOutput(directory, tmpFile, IOContext.DEFAULT)) { + try (IndexOutput output = directory.createOutput(tmpFile, IOContext.DEFAULT)) { CodecUtil.writeHeader(output, KEYSTORE_FILENAME, FORMAT_VERSION); output.writeByte(password.length == 0 ? (byte)0 : (byte)1); diff --git a/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java b/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java index 62815c03f47df..ca2f24e63c0b9 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java @@ -15,13 +15,13 @@ import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; +import org.apache.lucene.store.DataInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.NIOFSDirectory; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.core.Tuple; import org.elasticsearch.common.lucene.store.IndexOutputOutputStream; import org.elasticsearch.common.lucene.store.InputStreamIndexInput; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; @@ -30,6 +30,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.core.Tuple; import org.elasticsearch.core.internal.io.IOUtils; import java.io.FileNotFoundException; @@ -57,7 +58,7 @@ public abstract class MetadataStateFormat { private static final String STATE_FILE_CODEC = "state"; private static final int MIN_COMPATIBLE_STATE_FILE_VERSION = 1; - private static final int STATE_FILE_VERSION = 1; + private static final int STATE_FILE_VERSION = 2; private final String prefix; private final Pattern stateFilePattern; @@ -92,7 +93,7 @@ private void writeStateToFirstLocation(final T state, Path stateLocation, Direct throws WriteStateException { try { deleteFileIfExists(stateLocation, stateDir, tmpFileName); - try (IndexOutput out = EndiannessReverserUtil.createOutput(stateDir, tmpFileName, IOContext.DEFAULT)) { + try (IndexOutput out = stateDir.createOutput(tmpFileName, IOContext.DEFAULT)) { CodecUtil.writeHeader(out, STATE_FILE_CODEC, STATE_FILE_VERSION); out.writeInt(FORMAT.index()); try (XContentBuilder builder = newXContentBuilder(FORMAT, new IndexOutputOutputStream(out) { @@ -268,11 +269,12 @@ protected XContentBuilder newXContentBuilder(XContentType type, OutputStream str */ public final T read(NamedXContentRegistry namedXContentRegistry, Path file) throws IOException { try (Directory dir = newDirectory(file.getParent())) { - try (IndexInput indexInput = EndiannessReverserUtil.openInput(dir, file.getFileName().toString(), IOContext.DEFAULT)) { + try (IndexInput indexInput = dir.openInput(file.getFileName().toString(), IOContext.DEFAULT)) { // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here. CodecUtil.checksumEntireFile(indexInput); - CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, MIN_COMPATIBLE_STATE_FILE_VERSION, STATE_FILE_VERSION); - final XContentType xContentType = XContentType.values()[indexInput.readInt()]; + int fileVersion + = CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, MIN_COMPATIBLE_STATE_FILE_VERSION, STATE_FILE_VERSION); + final XContentType xContentType = readXContentType(indexInput, fileVersion); if (xContentType != FORMAT) { throw new IllegalStateException("expected state in " + file + " to be " + FORMAT + " format but was " + xContentType); } @@ -292,6 +294,13 @@ public final T read(NamedXContentRegistry namedXContentRegistry, Path file) thro } } + private static XContentType readXContentType(DataInput input, int fileVersion) throws IOException { + if (fileVersion == 1) { + input = EndiannessReverserUtil.wrapDataInput(input); + } + return XContentType.values()[input.readInt()]; + } + protected Directory newDirectory(Path dir) throws IOException { return new NIOFSDirectory(dir); } diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index 1c3e248baee3f..1eb04c1eceff1 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -11,7 +11,6 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; @@ -21,8 +20,6 @@ import org.apache.lucene.mockfile.FilterFileChannel; import org.apache.lucene.mockfile.FilterFileSystemProvider; import org.apache.lucene.store.AlreadyClosedException; -import org.apache.lucene.store.ByteArrayDataOutput; -import org.apache.lucene.store.DataOutput; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase; @@ -1260,8 +1257,7 @@ public void testTranslogWriter() throws IOException { final Set seenSeqNos = new HashSet<>(); boolean opsHaveValidSequenceNumbers = randomBoolean(); for (int i = 0; i < numOps; i++) { - byte[] bytes = new byte[4]; - DataOutput out = EndiannessReverserUtil.wrapDataOutput(new ByteArrayDataOutput(bytes)); + BytesStreamOutput out = new BytesStreamOutput(4); out.writeInt(i); long seqNo; do { @@ -1271,7 +1267,7 @@ public void testTranslogWriter() throws IOException { if (seqNo != SequenceNumbers.UNASSIGNED_SEQ_NO) { seenSeqNos.add(seqNo); } - writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), seqNo); + writer.add(ReleasableBytesReference.wrap(out.bytes()), seqNo); } assertThat(persistedSeqNos, empty()); writer.sync(); @@ -1292,10 +1288,9 @@ public void testTranslogWriter() throws IOException { assertThat(reader.getCheckpoint().minSeqNo, equalTo(minSeqNo)); assertThat(reader.getCheckpoint().maxSeqNo, equalTo(maxSeqNo)); - byte[] bytes = new byte[4]; - DataOutput out = EndiannessReverserUtil.wrapDataOutput(new ByteArrayDataOutput(bytes)); + BytesStreamOutput out = new BytesStreamOutput(4); out.writeInt(2048); - writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), randomNonNegativeLong()); + writer.add(ReleasableBytesReference.wrap(out.bytes()), randomNonNegativeLong()); if (reader instanceof TranslogReader) { ByteBuffer buffer = ByteBuffer.allocate(4); @@ -1470,10 +1465,9 @@ ChannelFactory getChannelFactory() { }) { TranslogWriter writer = translog.getCurrent(); - byte[] bytes = new byte[4]; - DataOutput out = EndiannessReverserUtil.wrapDataOutput(new ByteArrayDataOutput(new byte[4])); + BytesStreamOutput out = new BytesStreamOutput(4); out.writeInt(1); - writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), 1); + writer.add(ReleasableBytesReference.wrap(out.bytes()), 1); assertThat(persistedSeqNos, empty()); startBlocking.set(true); Thread thread = new Thread(() -> { @@ -1487,7 +1481,7 @@ ChannelFactory getChannelFactory() { writeStarted.await(); // Add will not block even though we are currently writing/syncing - writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), 2); + writer.add(ReleasableBytesReference.wrap(out.bytes()), 2); blocker.countDown(); // Sync against so that both operations are written @@ -1502,10 +1496,9 @@ public void testCloseIntoReader() throws IOException { try (TranslogWriter writer = translog.createWriter(translog.currentFileGeneration() + 1)) { final int numOps = randomIntBetween(8, 128); for (int i = 0; i < numOps; i++) { - final byte[] bytes = new byte[4]; - final DataOutput out = EndiannessReverserUtil.wrapDataOutput(new ByteArrayDataOutput(bytes)); + BytesStreamOutput out = new BytesStreamOutput(4); out.writeInt(i); - writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), randomNonNegativeLong()); + writer.add(ReleasableBytesReference.wrap(out.bytes()), randomNonNegativeLong()); } writer.sync(); final Checkpoint writerCheckpoint = writer.getCheckpoint(); diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java index 9fef6653ca533..cb6c98ffec740 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java @@ -7,7 +7,6 @@ */ package org.elasticsearch.indices.recovery; -import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.store.IndexOutput; import org.elasticsearch.common.util.set.Sets; @@ -33,10 +32,10 @@ public void testRenameTempFiles() throws IOException { "foo.bar", new StoreFileMetadata("foo.bar", 8 + CodecUtil.footerLength(), "9z51nw", MIN_SUPPORTED_LUCENE_VERSION.toString()), indexShard.store())) { - EndiannessReverserUtil.wrapDataOutput(indexOutput).writeInt(1); + CodecUtil.writeBEInt(indexOutput, 1); IndexOutput openIndexOutput = multiFileWriter.getOpenIndexOutput("foo.bar"); assertSame(openIndexOutput, indexOutput); - EndiannessReverserUtil.wrapDataOutput(openIndexOutput).writeInt(1); + CodecUtil.writeBEInt(openIndexOutput, 1); CodecUtil.writeFooter(indexOutput); } From 9b2ef3f1c4c8fd3755b8d4525e3103a8aabda32d Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 6 Sep 2021 14:20:17 +0100 Subject: [PATCH 36/67] Revert "Bump metadata versions where possible to avoid EndiannessReverserUtil" This reverts commit 77fc68e1ced6f8fe725fe1074b43fd7a0a5e324d. --- .../common/settings/KeyStoreWrapperTests.java | 5 +++- .../common/settings/KeyStoreWrapper.java | 12 ++++----- .../gateway/MetadataStateFormat.java | 21 +++++----------- .../index/translog/TranslogTests.java | 25 ++++++++++++------- .../indices/recovery/RecoveryStatusTests.java | 5 ++-- 5 files changed, 34 insertions(+), 34 deletions(-) diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java index c106e2d5ed158..250aafa18cee6 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java @@ -10,6 +10,7 @@ import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.store.DataOutput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; @@ -41,6 +42,7 @@ import java.util.List; import java.util.Locale; import java.util.Set; + import javax.crypto.Cipher; import javax.crypto.CipherOutputStream; import javax.crypto.SecretKey; @@ -339,12 +341,13 @@ private void possiblyAlterSecretString(DataOutputStream output, int truncLength) } private void possiblyAlterEncryptedBytes( - IndexOutput out, + IndexOutput indexOutput, byte[] salt, byte[] iv, byte[] encryptedBytes, int truncEncryptedDataLength ) throws Exception { + DataOutput out = EndiannessReverserUtil.wrapDataOutput(indexOutput); out.writeInt(4 + salt.length + 4 + iv.length + 4 + encryptedBytes.length); out.writeInt(salt.length); out.writeBytes(salt, salt.length); diff --git a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java index 7ad373a4747c0..49e3a458e386e 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java +++ b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java @@ -13,7 +13,6 @@ import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.store.ChecksumIndexInput; -import org.apache.lucene.store.DataInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; @@ -109,7 +108,7 @@ private static class Entry { public static final String KEYSTORE_FILENAME = "elasticsearch.keystore"; /** The version of the metadata written before the keystore data. */ - static final int FORMAT_VERSION = 5; + static final int FORMAT_VERSION = 4; /** The oldest metadata format version that can be read. */ private static final int MIN_FORMAT_VERSION = 1; @@ -235,10 +234,10 @@ public static KeyStoreWrapper load(Path configDir) throws IOException { } Directory directory = new NIOFSDirectory(configDir); - try (ChecksumIndexInput checksumInput = directory.openChecksumInput(KEYSTORE_FILENAME, IOContext.READONCE)) { + try (ChecksumIndexInput input = EndiannessReverserUtil.openChecksumInput(directory, KEYSTORE_FILENAME, IOContext.READONCE)) { final int formatVersion; try { - formatVersion = CodecUtil.checkHeader(checksumInput, KEYSTORE_FILENAME, MIN_FORMAT_VERSION, FORMAT_VERSION); + formatVersion = CodecUtil.checkHeader(input, KEYSTORE_FILENAME, MIN_FORMAT_VERSION, FORMAT_VERSION); } catch (IndexFormatTooOldException e) { throw new IllegalStateException("The Elasticsearch keystore [" + keystoreFile + "] format is too old. " + "You should delete and recreate it in order to upgrade.", e); @@ -246,7 +245,6 @@ public static KeyStoreWrapper load(Path configDir) throws IOException { throw new IllegalStateException("The Elasticsearch keystore [" + keystoreFile + "] format is too new. " + "Are you trying to downgrade? You should delete and recreate it in order to downgrade.", e); } - DataInput input = formatVersion < 5 ? checksumInput : EndiannessReverserUtil.wrapDataInput(checksumInput); byte hasPasswordByte = input.readByte(); boolean hasPassword = hasPasswordByte == 1; if (hasPassword == false && hasPasswordByte != 0) { @@ -298,7 +296,7 @@ public static KeyStoreWrapper load(Path configDir) throws IOException { input.readBytes(dataBytes, 0, dataBytesLen); } - CodecUtil.checkFooter(checksumInput); + CodecUtil.checkFooter(input); return new KeyStoreWrapper(formatVersion, hasPassword, dataBytes); } } @@ -515,7 +513,7 @@ public synchronized void save(Path configDir, char[] password, boolean preserveP // write to tmp file first, then overwrite String tmpFile = KEYSTORE_FILENAME + ".tmp"; Path keystoreTempFile = configDir.resolve(tmpFile); - try (IndexOutput output = directory.createOutput(tmpFile, IOContext.DEFAULT)) { + try (IndexOutput output = EndiannessReverserUtil.createOutput(directory, tmpFile, IOContext.DEFAULT)) { CodecUtil.writeHeader(output, KEYSTORE_FILENAME, FORMAT_VERSION); output.writeByte(password.length == 0 ? (byte)0 : (byte)1); diff --git a/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java b/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java index ca2f24e63c0b9..62815c03f47df 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java @@ -15,13 +15,13 @@ import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.store.DataInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.NIOFSDirectory; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.core.Tuple; import org.elasticsearch.common.lucene.store.IndexOutputOutputStream; import org.elasticsearch.common.lucene.store.InputStreamIndexInput; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; @@ -30,7 +30,6 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.core.Tuple; import org.elasticsearch.core.internal.io.IOUtils; import java.io.FileNotFoundException; @@ -58,7 +57,7 @@ public abstract class MetadataStateFormat { private static final String STATE_FILE_CODEC = "state"; private static final int MIN_COMPATIBLE_STATE_FILE_VERSION = 1; - private static final int STATE_FILE_VERSION = 2; + private static final int STATE_FILE_VERSION = 1; private final String prefix; private final Pattern stateFilePattern; @@ -93,7 +92,7 @@ private void writeStateToFirstLocation(final T state, Path stateLocation, Direct throws WriteStateException { try { deleteFileIfExists(stateLocation, stateDir, tmpFileName); - try (IndexOutput out = stateDir.createOutput(tmpFileName, IOContext.DEFAULT)) { + try (IndexOutput out = EndiannessReverserUtil.createOutput(stateDir, tmpFileName, IOContext.DEFAULT)) { CodecUtil.writeHeader(out, STATE_FILE_CODEC, STATE_FILE_VERSION); out.writeInt(FORMAT.index()); try (XContentBuilder builder = newXContentBuilder(FORMAT, new IndexOutputOutputStream(out) { @@ -269,12 +268,11 @@ protected XContentBuilder newXContentBuilder(XContentType type, OutputStream str */ public final T read(NamedXContentRegistry namedXContentRegistry, Path file) throws IOException { try (Directory dir = newDirectory(file.getParent())) { - try (IndexInput indexInput = dir.openInput(file.getFileName().toString(), IOContext.DEFAULT)) { + try (IndexInput indexInput = EndiannessReverserUtil.openInput(dir, file.getFileName().toString(), IOContext.DEFAULT)) { // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here. CodecUtil.checksumEntireFile(indexInput); - int fileVersion - = CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, MIN_COMPATIBLE_STATE_FILE_VERSION, STATE_FILE_VERSION); - final XContentType xContentType = readXContentType(indexInput, fileVersion); + CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, MIN_COMPATIBLE_STATE_FILE_VERSION, STATE_FILE_VERSION); + final XContentType xContentType = XContentType.values()[indexInput.readInt()]; if (xContentType != FORMAT) { throw new IllegalStateException("expected state in " + file + " to be " + FORMAT + " format but was " + xContentType); } @@ -294,13 +292,6 @@ public final T read(NamedXContentRegistry namedXContentRegistry, Path file) thro } } - private static XContentType readXContentType(DataInput input, int fileVersion) throws IOException { - if (fileVersion == 1) { - input = EndiannessReverserUtil.wrapDataInput(input); - } - return XContentType.values()[input.readInt()]; - } - protected Directory newDirectory(Path dir) throws IOException { return new NIOFSDirectory(dir); } diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index 1eb04c1eceff1..1c3e248baee3f 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -11,6 +11,7 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; @@ -20,6 +21,8 @@ import org.apache.lucene.mockfile.FilterFileChannel; import org.apache.lucene.mockfile.FilterFileSystemProvider; import org.apache.lucene.store.AlreadyClosedException; +import org.apache.lucene.store.ByteArrayDataOutput; +import org.apache.lucene.store.DataOutput; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase; @@ -1257,7 +1260,8 @@ public void testTranslogWriter() throws IOException { final Set seenSeqNos = new HashSet<>(); boolean opsHaveValidSequenceNumbers = randomBoolean(); for (int i = 0; i < numOps; i++) { - BytesStreamOutput out = new BytesStreamOutput(4); + byte[] bytes = new byte[4]; + DataOutput out = EndiannessReverserUtil.wrapDataOutput(new ByteArrayDataOutput(bytes)); out.writeInt(i); long seqNo; do { @@ -1267,7 +1271,7 @@ public void testTranslogWriter() throws IOException { if (seqNo != SequenceNumbers.UNASSIGNED_SEQ_NO) { seenSeqNos.add(seqNo); } - writer.add(ReleasableBytesReference.wrap(out.bytes()), seqNo); + writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), seqNo); } assertThat(persistedSeqNos, empty()); writer.sync(); @@ -1288,9 +1292,10 @@ public void testTranslogWriter() throws IOException { assertThat(reader.getCheckpoint().minSeqNo, equalTo(minSeqNo)); assertThat(reader.getCheckpoint().maxSeqNo, equalTo(maxSeqNo)); - BytesStreamOutput out = new BytesStreamOutput(4); + byte[] bytes = new byte[4]; + DataOutput out = EndiannessReverserUtil.wrapDataOutput(new ByteArrayDataOutput(bytes)); out.writeInt(2048); - writer.add(ReleasableBytesReference.wrap(out.bytes()), randomNonNegativeLong()); + writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), randomNonNegativeLong()); if (reader instanceof TranslogReader) { ByteBuffer buffer = ByteBuffer.allocate(4); @@ -1465,9 +1470,10 @@ ChannelFactory getChannelFactory() { }) { TranslogWriter writer = translog.getCurrent(); - BytesStreamOutput out = new BytesStreamOutput(4); + byte[] bytes = new byte[4]; + DataOutput out = EndiannessReverserUtil.wrapDataOutput(new ByteArrayDataOutput(new byte[4])); out.writeInt(1); - writer.add(ReleasableBytesReference.wrap(out.bytes()), 1); + writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), 1); assertThat(persistedSeqNos, empty()); startBlocking.set(true); Thread thread = new Thread(() -> { @@ -1481,7 +1487,7 @@ ChannelFactory getChannelFactory() { writeStarted.await(); // Add will not block even though we are currently writing/syncing - writer.add(ReleasableBytesReference.wrap(out.bytes()), 2); + writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), 2); blocker.countDown(); // Sync against so that both operations are written @@ -1496,9 +1502,10 @@ public void testCloseIntoReader() throws IOException { try (TranslogWriter writer = translog.createWriter(translog.currentFileGeneration() + 1)) { final int numOps = randomIntBetween(8, 128); for (int i = 0; i < numOps; i++) { - BytesStreamOutput out = new BytesStreamOutput(4); + final byte[] bytes = new byte[4]; + final DataOutput out = EndiannessReverserUtil.wrapDataOutput(new ByteArrayDataOutput(bytes)); out.writeInt(i); - writer.add(ReleasableBytesReference.wrap(out.bytes()), randomNonNegativeLong()); + writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), randomNonNegativeLong()); } writer.sync(); final Checkpoint writerCheckpoint = writer.getCheckpoint(); diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java index cb6c98ffec740..9fef6653ca533 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java @@ -7,6 +7,7 @@ */ package org.elasticsearch.indices.recovery; +import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.store.IndexOutput; import org.elasticsearch.common.util.set.Sets; @@ -32,10 +33,10 @@ public void testRenameTempFiles() throws IOException { "foo.bar", new StoreFileMetadata("foo.bar", 8 + CodecUtil.footerLength(), "9z51nw", MIN_SUPPORTED_LUCENE_VERSION.toString()), indexShard.store())) { - CodecUtil.writeBEInt(indexOutput, 1); + EndiannessReverserUtil.wrapDataOutput(indexOutput).writeInt(1); IndexOutput openIndexOutput = multiFileWriter.getOpenIndexOutput("foo.bar"); assertSame(openIndexOutput, indexOutput); - CodecUtil.writeBEInt(openIndexOutput, 1); + EndiannessReverserUtil.wrapDataOutput(openIndexOutput).writeInt(1); CodecUtil.writeFooter(indexOutput); } From cca2527b53b15d9e453bd82f261b080ef7d93ffe Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Fri, 10 Sep 2021 13:35:06 +0100 Subject: [PATCH 37/67] Further upgrades --- build-tools-internal/version.properties | 2 +- ...ssions-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...ssions-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - ...is-icu-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...is-icu-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - ...romoji-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...romoji-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - ...s-nori-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...s-nori-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - ...onetic-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...onetic-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - ...martcn-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...martcn-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - ...tempel-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...tempel-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - ...ologik-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...ologik-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - ...common-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...common-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - ...codecs-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...codecs-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - ...e-core-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...e-core-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - ...ouping-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...ouping-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - ...ighter-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...ighter-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - ...e-join-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...e-join-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - ...memory-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...memory-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - ...e-misc-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...e-misc-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - ...ueries-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...ueries-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - ...parser-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...parser-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - ...andbox-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...andbox-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - ...extras-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...extras-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - ...tial3d-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...tial3d-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - ...uggest-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...uggest-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - .../elasticsearch/common/lucene/Lucene.java | 6 +- .../index/EsTieredMergePolicy.java | 9 -- .../elasticsearch/index/IndexSettings.java | 2 - .../index/MergePolicyConfig.java | 7 +- .../index/engine/TranslogDirectoryReader.java | 10 +- .../fielddata/IndexNumericFieldData.java | 3 + .../index/mapper/DocumentLeafReader.java | 6 +- .../query/CombinedFieldsQueryBuilder.java | 2 +- .../index/similarity/SimilarityProviders.java | 2 +- .../index/similarity/SimilarityService.java | 4 +- .../similarity/LegacyBM25Similarity.java | 102 ++++++++++++++++++ .../FieldUsageTrackingDirectoryReader.java | 25 +---- .../search/profile/query/ProfileWeight.java | 5 + .../index/EsTieredMergePolicyTests.java | 6 -- .../index/MergePolicySettingsTests.java | 10 -- .../mapper/CompletionFieldMapperTests.java | 4 +- .../index/mapper/NumberFieldTypeTests.java | 2 + .../similarity/SimilarityServiceTests.java | 2 +- .../index/similarity/SimilarityTests.java | 2 +- .../composite/CompositeAggregatorTests.java | 10 +- .../profile/query/QueryProfilerTests.java | 6 +- .../analysis/AnalysisFactoryTestCase.java | 2 + .../engine/ThrowingLeafReaderWrapper.java | 17 +-- .../frozen/RewriteCachingDirectoryReader.java | 6 +- .../accesscontrol/FieldSubsetReader.java | 24 ++--- ...e-core-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 + ...e-core-9.0.0-snapshot-7450a7e64be.jar.sha1 | 1 - 72 files changed, 187 insertions(+), 135 deletions(-) create mode 100644 modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-7450a7e64be.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-7450a7e64be.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-7450a7e64be.jar.sha1 create mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-7450a7e64be.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-7450a7e64be.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-7450a7e64be.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-7450a7e64be.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-7450a7e64be.jar.sha1 create mode 100644 server/licenses/lucene-analysis-common-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 server/licenses/lucene-analysis-common-9.0.0-snapshot-7450a7e64be.jar.sha1 create mode 100644 server/licenses/lucene-backward-codecs-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-9.0.0-snapshot-7450a7e64be.jar.sha1 create mode 100644 server/licenses/lucene-core-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 server/licenses/lucene-core-9.0.0-snapshot-7450a7e64be.jar.sha1 create mode 100644 server/licenses/lucene-grouping-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-9.0.0-snapshot-7450a7e64be.jar.sha1 create mode 100644 server/licenses/lucene-highlighter-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-9.0.0-snapshot-7450a7e64be.jar.sha1 create mode 100644 server/licenses/lucene-join-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 server/licenses/lucene-join-9.0.0-snapshot-7450a7e64be.jar.sha1 create mode 100644 server/licenses/lucene-memory-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 server/licenses/lucene-memory-9.0.0-snapshot-7450a7e64be.jar.sha1 create mode 100644 server/licenses/lucene-misc-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 server/licenses/lucene-misc-9.0.0-snapshot-7450a7e64be.jar.sha1 create mode 100644 server/licenses/lucene-queries-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 server/licenses/lucene-queries-9.0.0-snapshot-7450a7e64be.jar.sha1 create mode 100644 server/licenses/lucene-queryparser-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-9.0.0-snapshot-7450a7e64be.jar.sha1 create mode 100644 server/licenses/lucene-sandbox-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-9.0.0-snapshot-7450a7e64be.jar.sha1 create mode 100644 server/licenses/lucene-spatial-extras-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-9.0.0-snapshot-7450a7e64be.jar.sha1 create mode 100644 server/licenses/lucene-spatial3d-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-9.0.0-snapshot-7450a7e64be.jar.sha1 create mode 100644 server/licenses/lucene-suggest-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-9.0.0-snapshot-7450a7e64be.jar.sha1 create mode 100644 server/src/main/java/org/elasticsearch/lucene/similarity/LegacyBM25Similarity.java create mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-1bb52859c88.jar.sha1 delete mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-7450a7e64be.jar.sha1 diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 1e0c7007ec2fe..6b19133b3d170 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,5 +1,5 @@ elasticsearch = 8.0.0 -lucene = 9.0.0-snapshot-7450a7e64be +lucene = 9.0.0-snapshot-1bb52859c88 bundled_jdk_vendor = adoptium bundled_jdk = 16.0.2+7 diff --git a/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-1bb52859c88.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..6eb5b30e436d6 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +0a22cf9872faa901f55a8b36971c4587b6ef65d2 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-7450a7e64be.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index 520fdbe7366e1..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -42eaef59486b8d205136510bb4adbcda64b07ff9 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-1bb52859c88.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..595d7d61794e8 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +c160be6333ec460db6518e7b8ec9f2c72d472283 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-7450a7e64be.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index e7a01f6dbb35e..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0dd17e1de76b20e3d1babf7b05d93a79f08c06f6 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-1bb52859c88.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..933cf0b1f9811 --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +4d980da6e47123a4d008f532747defc1bd0c7ebf \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-7450a7e64be.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index 14495774b62b0..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a96be6ae01b8a7d11d2aefd77471ba196e0e8300 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-1bb52859c88.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..efe1500cd39c8 --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +e4015e79f0c466ed6cb93d5028344e63bd55a7b2 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-7450a7e64be.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index 2329895df70aa..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -75cbbfab82dc45ba4e042f7a72f63e564c560a4d \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-1bb52859c88.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..4599cf6d6d409 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +90c9309dabb2735e051df7e51285e068830ad28d \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-7450a7e64be.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index 8013689bc03b4..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -232334a042b1c40bf6f9e84f39ecbdba8a6fc653 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-1bb52859c88.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..c50ca8383868a --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +bda71062bd0cc58869057d400009478385516c6c \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-7450a7e64be.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index 6a42857d6a2a4..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -60bffa02083a2f5d02e86ad2e492d9fd2c1f5524 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-1bb52859c88.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..2e101752b7d1d --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +b5b27e0318830758b53d082ced192e336a6fc38c \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-7450a7e64be.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index 9b954c1f7264d..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -84d9fcd0dbd37cd1865275956e113a9b1ab99f59 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-1bb52859c88.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..8d7c6e6d91b68 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +99464856f6c921d1c6e13e732fb4d1b10798d3d7 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-7450a7e64be.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index 8907c9d3689d9..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -625c56a51dd167183161df7ab18e8bb9822e93f1 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-analysis-common-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..89662462db300 --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +a0be730fff3308a12f1ca0ecd44dccbd26f6a985 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-analysis-common-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index d505d5a84e70a..0000000000000 --- a/server/licenses/lucene-analysis-common-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8cf0f514ff85880fdf334ac7d7c06c6189b265b9 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..6c37892950c77 --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +1b6dd61781f3c14f3e5d10f4954e3374df9832fb \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index df06d7e6cb66b..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2b72141c8d38ad290f1d2b10ee45b3fe6b243976 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-core-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..9b81ac44bbff6 --- /dev/null +++ b/server/licenses/lucene-core-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +7f578d770b81399adcfdc3628ee13c42f674af46 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-core-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index a6ac4928e168a..0000000000000 --- a/server/licenses/lucene-core-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e329f855502ece4db3155378a90463e9068ca7c8 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-grouping-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..cb8724a1c43ab --- /dev/null +++ b/server/licenses/lucene-grouping-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +f77123f7ac183011f8911dc24db35272895d4f83 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-grouping-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index b1c5042ea1cf1..0000000000000 --- a/server/licenses/lucene-grouping-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -681c676d7a1ee4620b434f6a5c888f482407873f \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-highlighter-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..f53d384bd7e4c --- /dev/null +++ b/server/licenses/lucene-highlighter-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +46d16879c7782fdd98fbc75c04b9b43abafb0242 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-highlighter-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index e215f0e64f356..0000000000000 --- a/server/licenses/lucene-highlighter-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8b9902a8d9972ca93e08ff7117431aa1edf31adb \ No newline at end of file diff --git a/server/licenses/lucene-join-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-join-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..871cbdbda6306 --- /dev/null +++ b/server/licenses/lucene-join-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +a749e01c9ea0ea44ec92fb148bcdcd92dbe9f291 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-join-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index 3ee6334fc43de..0000000000000 --- a/server/licenses/lucene-join-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7f8556d5c71fd81e198ea86020d7a7e535e6ca66 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-memory-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..cf5a839a8ebea --- /dev/null +++ b/server/licenses/lucene-memory-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +b6aa74c5fa31da4eea0a7c91d4f88fc1ee07fef4 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-memory-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index 2baf04ae15e8e..0000000000000 --- a/server/licenses/lucene-memory-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e052e67aee4cce3efdfa891f3e56ab9357c27979 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-misc-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..0984aafdcf704 --- /dev/null +++ b/server/licenses/lucene-misc-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +8ef232edbb8efc996b1cfa656d4cb70ed860b474 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-misc-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index 2419f8b68f9ce..0000000000000 --- a/server/licenses/lucene-misc-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3e3f073bb1c3886433995f3c8af151d5b5f2eea2 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-queries-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..4fa571f6a6b2c --- /dev/null +++ b/server/licenses/lucene-queries-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +a553e5a6fcce0c7d0de512e2905447d7f42df7da \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-queries-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index 9acaba5b9787d..0000000000000 --- a/server/licenses/lucene-queries-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -890ac3ade381800495e7cb1d7681b16253e2b954 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-queryparser-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..c0d09666763d2 --- /dev/null +++ b/server/licenses/lucene-queryparser-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +89d3c7f37d9a9b14693ac2a5f0068792e65fefe0 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-queryparser-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index b565678a409b2..0000000000000 --- a/server/licenses/lucene-queryparser-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1d07f35c6898613f8e90e1c291ec310e261b1fb9 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-sandbox-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..8354f5d179f84 --- /dev/null +++ b/server/licenses/lucene-sandbox-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +73c4456ef081b9d7ee22363c4e2cc5b30b16318c \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-sandbox-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index 3a81fb1b1cb59..0000000000000 --- a/server/licenses/lucene-sandbox-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -08ea5e7590e31366573353cf7bb58d3ef8139af0 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-spatial-extras-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..2a024f663b5f2 --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +a722336ef994fca4f2d57ff143f4143b9b4aafc0 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-spatial-extras-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index 44295b512f3f4..0000000000000 --- a/server/licenses/lucene-spatial-extras-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3f93555280b3d536b4ff8c2994b7ffda569fd268 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-spatial3d-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..ef57331574c43 --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +113d202de024bdc12c577368cb0905848dafc3e4 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-spatial3d-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index 268cfc7fe4336..0000000000000 --- a/server/licenses/lucene-spatial3d-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8414a83e906f3a9acb1a6420b765c59539f7da92 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-suggest-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..bd3604df45e07 --- /dev/null +++ b/server/licenses/lucene-suggest-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +ed597b90d1a01eb4f0ea35f5c540f6e20e975377 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.0.0-snapshot-7450a7e64be.jar.sha1 b/server/licenses/lucene-suggest-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index 113ad21da0033..0000000000000 --- a/server/licenses/lucene-suggest-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2d4183e10ae0ca10515c9db6523b861783363cd0 \ No newline at end of file diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java index ebd2c5d6a20cf..43acc68198114 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -24,6 +24,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.Fields; import org.apache.lucene.index.FilterCodecReader; import org.apache.lucene.index.FilterDirectoryReader; import org.apache.lucene.index.FilterLeafReader; @@ -47,7 +48,6 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; -import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.VectorValues; import org.apache.lucene.search.DocIdSetIterator; @@ -1030,7 +1030,7 @@ public VectorValues getVectorValues(String field) throws IOException { } @Override - public TopDocs searchNearestVectors(String field, float[] target, int k) throws IOException { + public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs) throws IOException { return null; } @@ -1050,7 +1050,7 @@ public void checkIntegrity() { } @Override - public TermVectors getTermVectorsReader() { + public Fields getTermVectors(int docID) throws IOException { return null; } diff --git a/server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java b/server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java index 1b67f45a21d35..bb38fc3a90eba 100644 --- a/server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java +++ b/server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java @@ -71,15 +71,6 @@ public int getMaxMergeAtOnce() { return regularMergePolicy.getMaxMergeAtOnce(); } - public void setMaxMergeAtOnceExplicit(int maxMergeAtOnceExplicit) { - regularMergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit); - forcedMergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit); - } - - public int getMaxMergeAtOnceExplicit() { - return forcedMergePolicy.getMaxMergeAtOnceExplicit(); - } - // only setter that must NOT delegate to the forced merge policy public void setMaxMergedSegmentMB(double mbFrac) { regularMergePolicy.setMaxMergedSegmentMB(mbFrac); diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index c36db7e5a385d..551712e6d0154 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -516,8 +516,6 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti mergePolicyConfig::setFloorSegmentSetting); scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING, mergePolicyConfig::setMaxMergesAtOnce); - scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING, - mergePolicyConfig::setMaxMergesAtOnceExplicit); scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING, mergePolicyConfig::setMaxMergedSegment); scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING, diff --git a/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java b/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java index d1717a41d8847..96043538ac8ae 100644 --- a/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java +++ b/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java @@ -130,6 +130,7 @@ public final class MergePolicyConfig { public static final Setting INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING = Setting.intSetting("index.merge.policy.max_merge_at_once", DEFAULT_MAX_MERGE_AT_ONCE, 2, Property.Dynamic, Property.IndexScope); + // TODO deprecate this as it no longer has any effect public static final Setting INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING = Setting.intSetting("index.merge.policy.max_merge_at_once_explicit", DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT, 2, Property.Dynamic, Property.IndexScope); @@ -150,6 +151,7 @@ public final class MergePolicyConfig { double forceMergeDeletesPctAllowed = indexSettings.getValue(INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING); // percentage ByteSizeValue floorSegment = indexSettings.getValue(INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING); int maxMergeAtOnce = indexSettings.getValue(INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING); + // TODO this is removed from lucene 9 - do we need to emit a warning if it is set? int maxMergeAtOnceExplicit = indexSettings.getValue(INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING); // TODO is this really a good default number for max_merge_segment, what happens for large indices, // won't they end up with many segments? @@ -166,7 +168,6 @@ public final class MergePolicyConfig { mergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed); mergePolicy.setFloorSegmentMB(floorSegment.getMbFrac()); mergePolicy.setMaxMergeAtOnce(maxMergeAtOnce); - mergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit); mergePolicy.setMaxMergedSegmentMB(maxMergedSegment.getMbFrac()); mergePolicy.setSegmentsPerTier(segmentsPerTier); mergePolicy.setDeletesPctAllowed(deletesPctAllowed); @@ -187,10 +188,6 @@ void setMaxMergedSegment(ByteSizeValue maxMergedSegment) { mergePolicy.setMaxMergedSegmentMB(maxMergedSegment.getMbFrac()); } - void setMaxMergesAtOnceExplicit(Integer maxMergeAtOnceExplicit) { - mergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit); - } - void setMaxMergesAtOnce(Integer maxMergeAtOnce) { mergePolicy.setMaxMergeAtOnce(maxMergeAtOnce); } diff --git a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java index ba0ce19a25090..7d5fb293e60f1 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java +++ b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java @@ -15,6 +15,7 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.Fields; import org.apache.lucene.index.ImpactsEnum; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexOptions; @@ -29,7 +30,6 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; -import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.VectorSimilarityFunction; @@ -281,8 +281,8 @@ public VectorValues getVectorValues(String field) throws IOException { } @Override - public TopDocs searchNearestVectors(String field, float[] target, int k) throws IOException { - return getDelegate().searchNearestVectors(field, target, k); + public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs) throws IOException { + return getDelegate().searchNearestVectors(field, target, k, acceptDocs); } @Override @@ -310,8 +310,8 @@ public LeafMetaData getMetaData() { } @Override - public TermVectors getTermVectorsReader() { - return getDelegate().getTermVectorsReader(); + public Fields getTermVectors(int docID) throws IOException { + return getDelegate().getTermVectors(docID); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java index 8dfa8a1e14502..b757a2daaf967 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java @@ -33,6 +33,7 @@ * Base class for numeric field data. */ public abstract class IndexNumericFieldData implements IndexFieldData { + /** * The type of number. */ @@ -103,6 +104,8 @@ public final SortField sortField( SortedNumericSelector.Type.MAX : SortedNumericSelector.Type.MIN; SortField sortField = new SortedNumericSortField(getFieldName(), getNumericType().sortFieldType, reverse, selectorType); sortField.setMissingValue(source.missingObject(missingValue, reverse)); + // TODO fix this! + sortField.setOptimizeSortWithPoints(false); return sortField; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java index 485fa1481dd07..2ee2daa7cb3eb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafMetaData; @@ -23,7 +24,6 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; -import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.index.VectorValues; @@ -195,7 +195,7 @@ public VectorValues getVectorValues(String field) throws IOException { } @Override - public TopDocs searchNearestVectors(String field, float[] target, int k) throws IOException { + public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs) throws IOException { return null; } @@ -220,7 +220,7 @@ public LeafMetaData getMetaData() { } @Override - public TermVectors getTermVectorsReader() { + public Fields getTermVectors(int docID) throws IOException { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java index 6c090f7a2bd56..013afd0814955 100644 --- a/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java @@ -12,7 +12,6 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute; import org.apache.lucene.index.Term; -import org.apache.lucene.misc.search.similarity.LegacyBM25Similarity; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostAttribute; @@ -36,6 +35,7 @@ import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.search.QueryParserHelper; +import org.elasticsearch.lucene.similarity.LegacyBM25Similarity; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/elasticsearch/index/similarity/SimilarityProviders.java b/server/src/main/java/org/elasticsearch/index/similarity/SimilarityProviders.java index 47fca70a99830..df7de9ff6515c 100644 --- a/server/src/main/java/org/elasticsearch/index/similarity/SimilarityProviders.java +++ b/server/src/main/java/org/elasticsearch/index/similarity/SimilarityProviders.java @@ -8,7 +8,6 @@ package org.elasticsearch.index.similarity; -import org.apache.lucene.misc.search.similarity.LegacyBM25Similarity; import org.apache.lucene.search.similarities.AfterEffect; import org.apache.lucene.search.similarities.AfterEffectB; import org.apache.lucene.search.similarities.AfterEffectL; @@ -42,6 +41,7 @@ import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.lucene.similarity.LegacyBM25Similarity; import java.util.Arrays; import java.util.HashSet; diff --git a/server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java b/server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java index f6a2dd117467b..8838e88408446 100644 --- a/server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java +++ b/server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java @@ -10,7 +10,6 @@ import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.misc.search.similarity.LegacyBM25Similarity; import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.TermStatistics; @@ -20,14 +19,15 @@ import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; -import org.elasticsearch.core.Nullable; import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.lucene.similarity.LegacyBM25Similarity; import org.elasticsearch.script.ScriptService; import java.util.Collections; diff --git a/server/src/main/java/org/elasticsearch/lucene/similarity/LegacyBM25Similarity.java b/server/src/main/java/org/elasticsearch/lucene/similarity/LegacyBM25Similarity.java new file mode 100644 index 0000000000000..ccd19b3f984a6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/lucene/similarity/LegacyBM25Similarity.java @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/* @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.elasticsearch.lucene.similarity; + +import org.apache.lucene.index.FieldInvertState; +import org.apache.lucene.search.CollectionStatistics; +import org.apache.lucene.search.TermStatistics; +import org.apache.lucene.search.similarities.BM25Similarity; +import org.apache.lucene.search.similarities.Similarity; + +/** + * Similarity that behaves like {@link BM25Similarity} while also applying + * the k1+1 factor to the numerator of the scoring formula + * + * @see BM25Similarity + * + * @deprecated {@link BM25Similarity} should be used instead + */ +@Deprecated +public final class LegacyBM25Similarity extends Similarity { + + private final BM25Similarity bm25Similarity; + + /** BM25 with these default values: + *

    + *
  • {@code k1 = 1.2}
    • + *
  • {@code b = 0.75}
    • + *
+ */ + public LegacyBM25Similarity() { + this.bm25Similarity = new BM25Similarity(); + } + + /** + * BM25 with the supplied parameter values. + * @param k1 Controls non-linear term frequency normalization (saturation). + * @param b Controls to what degree document length normalizes tf values. + * @throws IllegalArgumentException if {@code k1} is infinite or negative, or if {@code b} is + * not within the range {@code [0..1]} + */ + public LegacyBM25Similarity(float k1, float b, boolean discountOverlaps) { + this.bm25Similarity = new BM25Similarity(k1, b, discountOverlaps); + } + + @Override + public long computeNorm(FieldInvertState state) { + return bm25Similarity.computeNorm(state); + } + + @Override + public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) { + return bm25Similarity.scorer(boost * (1 + bm25Similarity.getK1()), collectionStats, termStats); + } + + /** + * Returns the k1 parameter + */ + public float getK1() { + return bm25Similarity.getK1(); + } + + /** + * Returns the b parameter + */ + public float getB() { + return bm25Similarity.getB(); + } + + /** + * Returns true if overlap tokens are discounted from the document's length. + */ + public boolean getDiscountOverlaps() { + return bm25Similarity.getDiscountOverlaps(); + } + + @Override + public String toString() { + return bm25Similarity.toString(); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java b/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java index 8749e49de8240..a51cf8aa77141 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java +++ b/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java @@ -23,7 +23,6 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; -import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.suggest.document.CompletionTerms; @@ -86,12 +85,12 @@ public FieldUsageTrackingLeafReader(LeafReader in, FieldUsageNotifier notifier) } @Override - public TermVectors getTermVectorsReader() { - TermVectors in = super.getTermVectorsReader(); - if (in != null) { - return new FieldUsageTrackingTermVectors(in); + public Fields getTermVectors(int docID) throws IOException { + Fields f = super.getTermVectors(docID); + if (f != null) { + f = new FieldUsageTrackingTermVectorFields(f); } - return in; + return f; } @Override @@ -326,20 +325,6 @@ public Terms terms(String field) throws IOException { } - private class FieldUsageTrackingTermVectors extends TermVectors { - - final TermVectors in; - - private FieldUsageTrackingTermVectors(TermVectors in) { - this.in = in; - } - - @Override - public Fields get(int doc) throws IOException { - return new FieldUsageTrackingTermVectorFields(in.get(doc)); - } - } - private class FieldUsageStoredFieldVisitor extends FilterStoredFieldVisitor { FieldUsageStoredFieldVisitor(StoredFieldVisitor visitor) { super(visitor); diff --git a/server/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java b/server/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java index eb0c4fcb887f5..2114f7321ad79 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java +++ b/server/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java @@ -100,6 +100,11 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio return subQueryWeight.explain(context, doc); } + @Override + public int count(LeafReaderContext context) throws IOException { + return subQueryWeight.count(context); + } + @Override public boolean isCacheable(LeafReaderContext ctx) { return false; diff --git a/server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java b/server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java index bf426c3df07e0..49a312a108f4c 100644 --- a/server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java +++ b/server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java @@ -47,12 +47,6 @@ public void testSetMaxMergeAtOnce() { assertEquals(42, policy.regularMergePolicy.getMaxMergeAtOnce()); } - public void testSetMaxMergeAtOnceExplicit() { - EsTieredMergePolicy policy = new EsTieredMergePolicy(); - policy.setMaxMergeAtOnceExplicit(42); - assertEquals(42, policy.forcedMergePolicy.getMaxMergeAtOnceExplicit()); - } - public void testSetSegmentsPerTier() { EsTieredMergePolicy policy = new EsTieredMergePolicy(); policy.setSegmentsPerTier(42); diff --git a/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java b/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java index c81dcf0dfffbb..5396b0b880b49 100644 --- a/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java @@ -96,14 +96,6 @@ public void testTieredMergePolicySettingsUpdate() throws IOException { assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1); - assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), - MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT); - indexSettings.updateIndexMetadata(newIndexMeta("index", - Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING.getKey(), - MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT - 1).build())); - assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), - MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT-1); - assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getMbFrac(), 0.0001); indexSettings.updateIndexMetadata(newIndexMeta("index", @@ -138,8 +130,6 @@ public void testTieredMergePolicySettingsUpdate() throws IOException { new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb(), ByteSizeUnit.MB).getMbFrac(), 0.00); assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE); - assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), - MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT); assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001); assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index 27ad439f16a57..2285087d5f517 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -14,7 +14,7 @@ import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; -import org.apache.lucene.search.suggest.document.Completion84PostingsFormat; +import org.apache.lucene.search.suggest.document.Completion90PostingsFormat; import org.apache.lucene.search.suggest.document.CompletionAnalyzer; import org.apache.lucene.search.suggest.document.ContextSuggestField; import org.apache.lucene.search.suggest.document.FuzzyCompletionQuery; @@ -124,7 +124,7 @@ public void testPostingsFormat() throws IOException { Codec codec = codecService.codec("default"); assertThat(codec, instanceOf(PerFieldMappingPostingFormatCodec.class)); PerFieldMappingPostingFormatCodec perFieldCodec = (PerFieldMappingPostingFormatCodec) codec; - assertThat(perFieldCodec.getPostingsFormatForField("field"), instanceOf(Completion84PostingsFormat.class)); + assertThat(perFieldCodec.getPostingsFormatForField("field"), instanceOf(Completion90PostingsFormat.class)); } public void testDefaultConfiguration() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index e50fde92d7761..340df3e2c6209 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -419,9 +419,11 @@ public void doTestDocValueRangeQueries(NumberType type, Supplier valueSu dir.close(); } + /* public void testIndexSortIntRange() throws Exception { doTestIndexSortRangeQueries(NumberType.INTEGER, random()::nextInt); } + */ public void testIndexSortLongRange() throws Exception { doTestIndexSortRangeQueries(NumberType.LONG, random()::nextLong); diff --git a/server/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java b/server/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java index 72b5353574604..e960cc5cb24c8 100644 --- a/server/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.index.similarity; import org.apache.lucene.index.FieldInvertState; -import org.apache.lucene.misc.search.similarity.LegacyBM25Similarity; import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.TermStatistics; import org.apache.lucene.search.similarities.BooleanSimilarity; @@ -16,6 +15,7 @@ import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.lucene.similarity.LegacyBM25Similarity; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; import org.hamcrest.Matchers; diff --git a/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java b/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java index 167e08ee81588..7c0245dd61b8b 100644 --- a/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java +++ b/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.index.similarity; -import org.apache.lucene.misc.search.similarity.LegacyBM25Similarity; import org.apache.lucene.search.similarities.AfterEffectL; import org.apache.lucene.search.similarities.BasicModelG; import org.apache.lucene.search.similarities.BooleanSimilarity; @@ -30,6 +29,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.lucene.similarity.LegacyBM25Similarity; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java index f9d87a8ae409e..7ddf8317f405d 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java @@ -299,11 +299,11 @@ public void testUnmappedFieldWithHistogram() throws Exception { final String mappedFieldName = "price"; dataset.addAll( Arrays.asList( - createDocument(mappedFieldName, 103L), - createDocument(mappedFieldName, 51L), - createDocument(mappedFieldName, 56L), - createDocument(mappedFieldName, 105L), - createDocument(mappedFieldName, 25L) + createDocument(mappedFieldName, 103), + createDocument(mappedFieldName, 51), + createDocument(mappedFieldName, 56), + createDocument(mappedFieldName, 105), + createDocument(mappedFieldName, 25) ) ); diff --git a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java index 23457e3ea3c5d..344f7885c76a9 100644 --- a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java +++ b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java @@ -165,9 +165,11 @@ public void testUseIndexStats() throws IOException { QueryProfiler profiler = new QueryProfiler(); searcher.setProfiler(profiler); Query query = new TermQuery(new Term("foo", "bar")); - searcher.count(query); // will use index stats + searcher.count(query); // will use index stats - builds weight but never builds scorer List results = profiler.getTree(); - assertEquals(0, results.size()); + assertEquals(1, results.size()); + ProfileResult result = results.get(0); + assertEquals(0, (long) result.getTimeBreakdown().get("build_scorer_count")); long rewriteTime = profiler.getRewriteTime(); assertThat(rewriteTime, greaterThan(0L)); diff --git a/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java index d4ee2ad4b3634..690d667c576d1 100644 --- a/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java @@ -136,6 +136,8 @@ public abstract class AnalysisFactoryTestCase extends ESTestCase { entry("swedishminimalstem", MovedToAnalysisCommon.class), entry("synonym", MovedToAnalysisCommon.class), entry("synonymgraph", MovedToAnalysisCommon.class), + entry("telugunormalization", MovedToAnalysisCommon.class), + entry("telugustem", MovedToAnalysisCommon.class), entry("trim", MovedToAnalysisCommon.class), entry("truncate", MovedToAnalysisCommon.class), entry("turkishlowercase", MovedToAnalysisCommon.class), diff --git a/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java b/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java index db2913b0bb7c8..619b7484c7f78 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java +++ b/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java @@ -16,7 +16,6 @@ import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; @@ -83,18 +82,10 @@ public Terms terms(String field) throws IOException { } @Override - public TermVectors getTermVectorsReader() { - TermVectors in = super.getTermVectorsReader(); - if (in == null) { - return null; - } - return new TermVectors() { - @Override - public Fields get(int doc) throws IOException { - thrower.maybeThrow(Flags.TermVectors); - return new ThrowingFields(in.get(doc), thrower); - } - }; + public Fields getTermVectors(int docID) throws IOException { + Fields fields = super.getTermVectors(docID); + thrower.maybeThrow(Flags.TermVectors); + return fields == null ? null : new ThrowingFields(fields, thrower); } /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java index 7439326675765..a53ab745a4c89 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java @@ -10,6 +10,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.LeafMetaData; @@ -21,7 +22,6 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; -import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.VectorValues; import org.apache.lucene.search.TopDocs; @@ -221,7 +221,7 @@ public VectorValues getVectorValues(String field) throws IOException { } @Override - public TopDocs searchNearestVectors(String field, float[] target, int k) throws IOException { + public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs) throws IOException { throw new UnsupportedOperationException(); } @@ -250,7 +250,7 @@ public LeafMetaData getMetaData() { } @Override - public TermVectors getTermVectorsReader() { + public Fields getTermVectors(int docId) { throw new UnsupportedOperationException(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java index ca7d72a264bf9..76e6516f89f1c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java @@ -22,7 +22,6 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.TermState; -import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; @@ -30,12 +29,12 @@ import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.core.Tuple; import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; @@ -150,23 +149,14 @@ public FieldInfos getFieldInfos() { } @Override - public TermVectors getTermVectorsReader() { - TermVectors in = super.getTermVectorsReader(); - if (in == null) { + public Fields getTermVectors(int docID) throws IOException { + Fields f = super.getTermVectors(docID); + if (f == null) { return null; } - return new TermVectors() { - @Override - public Fields get(int doc) throws IOException { - Fields f = in.get(doc); - if (f == null) { - return null; - } - f = new FieldFilterFields(f); - // we need to check for emptyness, so we can return null: - return f.iterator().hasNext() ? f : null; - } - }; + f = new FieldFilterFields(f); + // we need to check for emptyness, so we can return null: + return f.iterator().hasNext() ? f : null; } /** Filter a map by a {@link CharacterRunAutomaton} that defines the fields to retain. */ diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-1bb52859c88.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-1bb52859c88.jar.sha1 new file mode 100644 index 0000000000000..9b81ac44bbff6 --- /dev/null +++ b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-1bb52859c88.jar.sha1 @@ -0,0 +1 @@ +7f578d770b81399adcfdc3628ee13c42f674af46 \ No newline at end of file diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-7450a7e64be.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-7450a7e64be.jar.sha1 deleted file mode 100644 index a6ac4928e168a..0000000000000 --- a/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-7450a7e64be.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e329f855502ece4db3155378a90463e9068ca7c8 \ No newline at end of file From b1c7ed32cfc26aa298bff6afa63c1a9fce4803a6 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 13 Sep 2021 09:48:09 +0100 Subject: [PATCH 38/67] new version --- build-tools-internal/version.properties | 2 +- .../analysis/common/CommonAnalysisPlugin.java | 2 ++ .../TeluguNormalizationFilterFactory.java | 28 +++++++++++++++++++ .../common/TeluguStemFilterFactory.java | 28 +++++++++++++++++++ .../common/CommonAnalysisFactoryTests.java | 2 ++ ...ssions-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...ssions-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - ...is-icu-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...is-icu-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - ...romoji-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...romoji-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - ...s-nori-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...s-nori-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - ...onetic-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...onetic-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - ...martcn-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...martcn-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - ...tempel-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...tempel-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - ...ologik-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...ologik-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - ...common-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...common-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - ...codecs-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...codecs-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - ...e-core-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...e-core-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - ...ouping-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...ouping-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - ...ighter-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...ighter-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - ...e-join-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...e-join-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - ...memory-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...memory-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - ...e-misc-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...e-misc-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - ...ueries-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...ueries-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - ...parser-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...parser-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - ...andbox-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...andbox-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - ...extras-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...extras-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - ...tial3d-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...tial3d-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - ...uggest-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...uggest-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - .../grouping/CollapsingTopDocsCollector.java | 4 +++ ...e-core-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 + ...e-core-9.0.0-snapshot-1bb52859c88.jar.sha1 | 1 - 52 files changed, 88 insertions(+), 24 deletions(-) create mode 100644 modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TeluguNormalizationFilterFactory.java create mode 100644 modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TeluguStemFilterFactory.java create mode 100644 modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-1bb52859c88.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-1bb52859c88.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-1bb52859c88.jar.sha1 create mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-1bb52859c88.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-1bb52859c88.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-1bb52859c88.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-1bb52859c88.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-1bb52859c88.jar.sha1 create mode 100644 server/licenses/lucene-analysis-common-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 server/licenses/lucene-analysis-common-9.0.0-snapshot-1bb52859c88.jar.sha1 create mode 100644 server/licenses/lucene-backward-codecs-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-9.0.0-snapshot-1bb52859c88.jar.sha1 create mode 100644 server/licenses/lucene-core-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 server/licenses/lucene-core-9.0.0-snapshot-1bb52859c88.jar.sha1 create mode 100644 server/licenses/lucene-grouping-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-9.0.0-snapshot-1bb52859c88.jar.sha1 create mode 100644 server/licenses/lucene-highlighter-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-9.0.0-snapshot-1bb52859c88.jar.sha1 create mode 100644 server/licenses/lucene-join-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 server/licenses/lucene-join-9.0.0-snapshot-1bb52859c88.jar.sha1 create mode 100644 server/licenses/lucene-memory-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 server/licenses/lucene-memory-9.0.0-snapshot-1bb52859c88.jar.sha1 create mode 100644 server/licenses/lucene-misc-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 server/licenses/lucene-misc-9.0.0-snapshot-1bb52859c88.jar.sha1 create mode 100644 server/licenses/lucene-queries-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 server/licenses/lucene-queries-9.0.0-snapshot-1bb52859c88.jar.sha1 create mode 100644 server/licenses/lucene-queryparser-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-9.0.0-snapshot-1bb52859c88.jar.sha1 create mode 100644 server/licenses/lucene-sandbox-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-9.0.0-snapshot-1bb52859c88.jar.sha1 create mode 100644 server/licenses/lucene-spatial-extras-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-9.0.0-snapshot-1bb52859c88.jar.sha1 create mode 100644 server/licenses/lucene-spatial3d-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-9.0.0-snapshot-1bb52859c88.jar.sha1 create mode 100644 server/licenses/lucene-suggest-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-9.0.0-snapshot-1bb52859c88.jar.sha1 create mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-19537578dd4.jar.sha1 delete mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-1bb52859c88.jar.sha1 diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 6b19133b3d170..1532171514153 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,5 +1,5 @@ elasticsearch = 8.0.0 -lucene = 9.0.0-snapshot-1bb52859c88 +lucene = 9.0.0-snapshot-19537578dd4 bundled_jdk_vendor = adoptium bundled_jdk = 16.0.2+7 diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java index 3e9a952d7cf91..1cc1c300c468d 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java @@ -309,6 +309,8 @@ public TokenStream create(TokenStream tokenStream) { filters.put("swedishminimalstem", SwedishMinimalStemFilterFactory::new); filters.put("synonym", requiresAnalysisSettings(SynonymTokenFilterFactory::new)); filters.put("synonym_graph", requiresAnalysisSettings(SynonymGraphTokenFilterFactory::new)); + filters.put("telugunormalization", TeluguNormalizationFilterFactory::new); + filters.put("telugustem", TeluguStemFilterFactory::new); filters.put("trim", TrimTokenFilterFactory::new); filters.put("truncate", requiresAnalysisSettings(TruncateTokenFilterFactory::new)); filters.put("unique", UniqueTokenFilterFactory::new); diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TeluguNormalizationFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TeluguNormalizationFilterFactory.java new file mode 100644 index 0000000000000..30dbece051836 --- /dev/null +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TeluguNormalizationFilterFactory.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.analysis.common; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.te.TeluguNormalizationFilter; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; + +public class TeluguNormalizationFilterFactory extends AbstractTokenFilterFactory { + + public TeluguNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); + } + + @Override + public TokenStream create(TokenStream tokenStream) { + return new TeluguNormalizationFilter(tokenStream); + } +} diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TeluguStemFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TeluguStemFilterFactory.java new file mode 100644 index 0000000000000..4468d2f381c46 --- /dev/null +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TeluguStemFilterFactory.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.analysis.common; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.te.TeluguStemFilter; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; + +public class TeluguStemFilterFactory extends AbstractTokenFilterFactory { + + public TeluguStemFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); + } + + @Override + public TokenStream create(TokenStream tokenStream) { + return new TeluguStemFilter(tokenStream); + } +} diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java index 834c722baddfa..7d20c2c52f80f 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java @@ -94,6 +94,8 @@ protected Map> getTokenFilters() { filters.put("swedishlightstem", StemmerTokenFilterFactory.class); filters.put("swedishminimalstem", SwedishMinimalStemFilterFactory.class); filters.put("stemmeroverride", StemmerOverrideTokenFilterFactory.class); + filters.put("telugunormalization", TeluguNormalizationFilterFactory.class); + filters.put("telugustem", TeluguStemFilterFactory.class); filters.put("kstem", KStemTokenFilterFactory.class); filters.put("synonym", SynonymTokenFilterFactory.class); filters.put("synonymgraph", SynonymGraphTokenFilterFactory.class); diff --git a/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-19537578dd4.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..b52652fe37380 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +60103d458504e770c481b1572db407505a377cdb \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-1bb52859c88.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index 6eb5b30e436d6..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0a22cf9872faa901f55a8b36971c4587b6ef65d2 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-19537578dd4.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..db23dfa5e8d9e --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +50342351e5de9087521ba3011d36ec1b3979c550 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-1bb52859c88.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index 595d7d61794e8..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c160be6333ec460db6518e7b8ec9f2c72d472283 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-19537578dd4.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..53302045164bb --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +adaabceb975f55a313fbedb36b6c67c9bede6321 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-1bb52859c88.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index 933cf0b1f9811..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4d980da6e47123a4d008f532747defc1bd0c7ebf \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-19537578dd4.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..4d6de7683f453 --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +7f59e54768eca7a9386280f0dd2531d20e4b08fd \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-1bb52859c88.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index efe1500cd39c8..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e4015e79f0c466ed6cb93d5028344e63bd55a7b2 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-19537578dd4.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..6a3be766f9b10 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +1b1222e99180a9f17eaca1c1fec02fdc3d78b037 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-1bb52859c88.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index 4599cf6d6d409..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -90c9309dabb2735e051df7e51285e068830ad28d \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-19537578dd4.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..b6848d9ae06ed --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +f2fe5d44c3bcaf640738c2c195ee28c29fbd1d04 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-1bb52859c88.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index c50ca8383868a..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bda71062bd0cc58869057d400009478385516c6c \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-19537578dd4.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..8ca5dc0e9ad14 --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +c6c5892baf428f551c02cc509a675860f22dee58 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-1bb52859c88.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index 2e101752b7d1d..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b5b27e0318830758b53d082ced192e336a6fc38c \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-19537578dd4.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..df748c9a7ece1 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +44785e4c17e7389c76b1fab716b3b467b4e69247 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-1bb52859c88.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index 8d7c6e6d91b68..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -99464856f6c921d1c6e13e732fb4d1b10798d3d7 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-analysis-common-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..c48180ec54199 --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +32aa77f6577d4732a7ca92ae3b973040ee3b5056 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-analysis-common-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index 89662462db300..0000000000000 --- a/server/licenses/lucene-analysis-common-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a0be730fff3308a12f1ca0ecd44dccbd26f6a985 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..efa0148c974fa --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +de34bec3d7056dcb3286bcb4ba7ef6a419bffd77 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index 6c37892950c77..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1b6dd61781f3c14f3e5d10f4954e3374df9832fb \ No newline at end of file diff --git a/server/licenses/lucene-core-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-core-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..ef358bc0a7b02 --- /dev/null +++ b/server/licenses/lucene-core-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +4b12a01eed8a8101d80195968f43976bde37774d \ No newline at end of file diff --git a/server/licenses/lucene-core-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-core-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index 9b81ac44bbff6..0000000000000 --- a/server/licenses/lucene-core-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7f578d770b81399adcfdc3628ee13c42f674af46 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-grouping-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..545b1bb5f914d --- /dev/null +++ b/server/licenses/lucene-grouping-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +52bcb73182edf7ddc1001b1d20874f35a1d8b488 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-grouping-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index cb8724a1c43ab..0000000000000 --- a/server/licenses/lucene-grouping-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f77123f7ac183011f8911dc24db35272895d4f83 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-highlighter-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..45db98f5ea973 --- /dev/null +++ b/server/licenses/lucene-highlighter-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +b4db1727d37a9c7ad7a848f63b888c5515cbd13e \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-highlighter-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index f53d384bd7e4c..0000000000000 --- a/server/licenses/lucene-highlighter-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -46d16879c7782fdd98fbc75c04b9b43abafb0242 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-join-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..814058f556540 --- /dev/null +++ b/server/licenses/lucene-join-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +18de1307e21f198904d28fecb34491be94e35d0b \ No newline at end of file diff --git a/server/licenses/lucene-join-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-join-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index 871cbdbda6306..0000000000000 --- a/server/licenses/lucene-join-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a749e01c9ea0ea44ec92fb148bcdcd92dbe9f291 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-memory-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..4e8d2f7a6f899 --- /dev/null +++ b/server/licenses/lucene-memory-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +317eb2399cd1e81895cf040cdad338c55e52731c \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-memory-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index cf5a839a8ebea..0000000000000 --- a/server/licenses/lucene-memory-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b6aa74c5fa31da4eea0a7c91d4f88fc1ee07fef4 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-misc-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..d1fd76330c937 --- /dev/null +++ b/server/licenses/lucene-misc-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +f464b6135050dab0b7e00b6479b0fe72259a778e \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-misc-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index 0984aafdcf704..0000000000000 --- a/server/licenses/lucene-misc-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8ef232edbb8efc996b1cfa656d4cb70ed860b474 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-queries-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..39e6c6df9a3bf --- /dev/null +++ b/server/licenses/lucene-queries-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +21ed7fde948dfb04846227de03d77c37fcaa469e \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-queries-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index 4fa571f6a6b2c..0000000000000 --- a/server/licenses/lucene-queries-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a553e5a6fcce0c7d0de512e2905447d7f42df7da \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-queryparser-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..b587060d3cbec --- /dev/null +++ b/server/licenses/lucene-queryparser-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +cad0467e25c04e42f714edc4fb754a4fa372732f \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-queryparser-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index c0d09666763d2..0000000000000 --- a/server/licenses/lucene-queryparser-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -89d3c7f37d9a9b14693ac2a5f0068792e65fefe0 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-sandbox-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..250594c182648 --- /dev/null +++ b/server/licenses/lucene-sandbox-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +3258e38dea3a632fef6c89ccde9a8b0de6b98cf8 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-sandbox-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index 8354f5d179f84..0000000000000 --- a/server/licenses/lucene-sandbox-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -73c4456ef081b9d7ee22363c4e2cc5b30b16318c \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-spatial-extras-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..3d97ce0710780 --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +f60c1dbd261d57d76cc314437cffeaa6e379b866 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-spatial-extras-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index 2a024f663b5f2..0000000000000 --- a/server/licenses/lucene-spatial-extras-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a722336ef994fca4f2d57ff143f4143b9b4aafc0 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-spatial3d-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..615c93f84a378 --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +dc78c6635a11be25b803ee7893e04e79284aba81 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-spatial3d-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index ef57331574c43..0000000000000 --- a/server/licenses/lucene-spatial3d-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -113d202de024bdc12c577368cb0905848dafc3e4 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-suggest-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..481c4335938f3 --- /dev/null +++ b/server/licenses/lucene-suggest-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +beba40f3e559cb33c49acce1d7919c62fb098361 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.0.0-snapshot-1bb52859c88.jar.sha1 b/server/licenses/lucene-suggest-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index bd3604df45e07..0000000000000 --- a/server/licenses/lucene-suggest-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ed597b90d1a01eb4f0ea35f5c540f6e20e975377 \ No newline at end of file diff --git a/server/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java b/server/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java index ff83f1ba8ec02..7f4b9059b5b0e 100644 --- a/server/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java +++ b/server/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java @@ -59,6 +59,10 @@ public final class CollapsingTopDocsCollector extends FirstPassGroupingCollec ((FieldComparator) comparator).setTopValue(after.fields[0]); } this.reversed = sortField.getReverse() ? -1 : 1; + + if (sortField.getOptimizeSortWithPoints() == false) { + comparator.disableSkipping(); + } } /** diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-19537578dd4.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-19537578dd4.jar.sha1 new file mode 100644 index 0000000000000..ef358bc0a7b02 --- /dev/null +++ b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-19537578dd4.jar.sha1 @@ -0,0 +1 @@ +4b12a01eed8a8101d80195968f43976bde37774d \ No newline at end of file diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-1bb52859c88.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-1bb52859c88.jar.sha1 deleted file mode 100644 index 9b81ac44bbff6..0000000000000 --- a/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-1bb52859c88.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7f578d770b81399adcfdc3628ee13c42f674af46 \ No newline at end of file From 78d0fd8a169bfb427b0e43d99fc4b577d333b2c7 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 13 Sep 2021 12:09:25 +0100 Subject: [PATCH 39/67] don't run field usage API in bwc mode, output values are different between lucene versions --- rest-api-spec/build.gradle | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 49ab4f6e9bf85..26a9f80d19faf 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -79,6 +79,9 @@ tasks.named("yamlRestTestV7CompatTest").configure { 'indices.stats/20_translog/Translog retention without soft_deletes', 'indices.stats/20_translog/Translog stats on closed indices without soft-deletes', + // field usage results will be different between lucene versions + 'indices.stats/60_field_usage/*', + // upgrade api will only get a dummy endpoint returning an exception suggesting to use _reindex 'indices.upgrade/*/*', From f2fef47488b51e04c89c06fb1a881a40dadefe0d Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 13 Sep 2021 13:11:47 +0100 Subject: [PATCH 40/67] haversin in meters, not kilometers --- docs/reference/scripting/expression.asciidoc | 2 +- .../org/elasticsearch/script/expression/MoreExpressionIT.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/scripting/expression.asciidoc b/docs/reference/scripting/expression.asciidoc index 84c07ebecaf31..735b3bc6d17bd 100644 --- a/docs/reference/scripting/expression.asciidoc +++ b/docs/reference/scripting/expression.asciidoc @@ -125,7 +125,7 @@ values within the doc. |`doc['field_name'].lon` |The longitude of the geo point. |======================================================================= -The following example computes distance in kilometers from Washington, DC: +The following example computes distance in meters from Washington, DC: `haversin(38.9072, 77.0369, doc['field_name'].lat, doc['field_name'].lon)` diff --git a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java index 2a3864aca3227..4965a5c406719 100644 --- a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java +++ b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java @@ -606,7 +606,7 @@ public void testGeo() throws Exception { rsp = buildRequest("haversin(38.9072, 77.0369, doc['location'].lat, doc['location'].lon)").get(); assertSearchResponse(rsp); assertEquals(1, rsp.getHits().getTotalHits().value); - assertEquals(3170D, rsp.getHits().getAt(0).field("foo").getValue(), 50D); + assertEquals(3169186D, rsp.getHits().getAt(0).field("foo").getValue(), 50D); } public void testBoolean() throws Exception { From 5dc4147a3dad2126f2ca2f77c119f313e90dd013 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 13 Sep 2021 14:00:22 +0100 Subject: [PATCH 41/67] change in max clause count behaviour --- .../java/org/elasticsearch/search/query/QueryStringIT.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java index bd71dd47a1aef..45eb26de6bc80 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java @@ -312,9 +312,7 @@ public void testLimitOnExpandedFields() throws Exception { doAssertOneHitForQueryString("field_A0:foo"); // expanding to the limit should work doAssertOneHitForQueryString("field_A\\*:foo"); - // expanding two blocks to the limit still works - doAssertOneHitForQueryString("field_A\\*:foo field_B\\*:bar"); - + // adding a non-existing field on top shouldn't overshoot the limit doAssertOneHitForQueryString("field_A\\*:foo unmapped:something"); From 872befbf7e209a7d51f6570bea8e16949f02fe21 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 13 Sep 2021 14:51:28 +0100 Subject: [PATCH 42/67] composite agg tests --- .../bucket/composite/CompositeAggregatorTests.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java index 7cdb93433be4a..23b0ace7e6168 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java @@ -1316,8 +1316,8 @@ public void testMissingTermBucket() throws Exception { public void testMissingHistogramBucket() throws Exception { List>> dataset = Arrays.asList( - createDocument("const", 1, "long", 1), - createDocument("const", 1, "long", 2), + createDocument("const", 1, "long", 1L), + createDocument("const", 1, "long", 2L), createDocument("const", 1, "keyword", "a") ); @@ -1462,8 +1462,8 @@ public void testMissingTermBucketAfterKey() throws Exception { public void testMissingHistogramBucketAfterKey() throws Exception { List>> dataset = Arrays.asList( - createDocument("const", 1, "long", 1), - createDocument("const", 1, "long", 2), + createDocument("const", 1, "long", 1L), + createDocument("const", 1, "long", 2L), createDocument("const", 1, "keyword", "a"), createDocument("const", 1, "keyword", "b") ); From 1a34d10578a737204792b294d58174758e04ba46 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 15 Sep 2021 11:26:15 +0100 Subject: [PATCH 43/67] Revert "haversin in meters, not kilometers" This reverts commit f2fef47488b51e04c89c06fb1a881a40dadefe0d. --- docs/reference/scripting/expression.asciidoc | 2 +- .../org/elasticsearch/script/expression/MoreExpressionIT.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/scripting/expression.asciidoc b/docs/reference/scripting/expression.asciidoc index 735b3bc6d17bd..84c07ebecaf31 100644 --- a/docs/reference/scripting/expression.asciidoc +++ b/docs/reference/scripting/expression.asciidoc @@ -125,7 +125,7 @@ values within the doc. |`doc['field_name'].lon` |The longitude of the geo point. |======================================================================= -The following example computes distance in meters from Washington, DC: +The following example computes distance in kilometers from Washington, DC: `haversin(38.9072, 77.0369, doc['field_name'].lat, doc['field_name'].lon)` diff --git a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java index 4965a5c406719..2a3864aca3227 100644 --- a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java +++ b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java @@ -606,7 +606,7 @@ public void testGeo() throws Exception { rsp = buildRequest("haversin(38.9072, 77.0369, doc['location'].lat, doc['location'].lon)").get(); assertSearchResponse(rsp); assertEquals(1, rsp.getHits().getTotalHits().value); - assertEquals(3169186D, rsp.getHits().getAt(0).field("foo").getValue(), 50D); + assertEquals(3170D, rsp.getHits().getAt(0).field("foo").getValue(), 50D); } public void testBoolean() throws Exception { From d988372bb7935f8072ff446acac0246bb8ad59e4 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 15 Sep 2021 11:31:00 +0100 Subject: [PATCH 44/67] pickup sort opto fix --- build-tools-internal/version.properties | 2 +- .../lucene-expressions-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + .../lucene-expressions-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - .../src/main/plugin-metadata/plugin-security.policy | 1 + .../lucene-analysis-icu-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + .../lucene-analysis-icu-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - ...lucene-analysis-kuromoji-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + ...lucene-analysis-kuromoji-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - .../lucene-analysis-nori-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + .../lucene-analysis-nori-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - ...lucene-analysis-phonetic-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + ...lucene-analysis-phonetic-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - .../lucene-analysis-smartcn-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + .../lucene-analysis-smartcn-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - .../lucene-analysis-stempel-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + .../lucene-analysis-stempel-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - ...cene-analysis-morfologik-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + ...cene-analysis-morfologik-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - .../lucene-analysis-common-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + .../lucene-analysis-common-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - .../lucene-backward-codecs-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + .../lucene-backward-codecs-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - server/licenses/lucene-core-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + server/licenses/lucene-core-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - .../lucene-grouping-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + .../lucene-grouping-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - .../lucene-highlighter-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + .../lucene-highlighter-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - server/licenses/lucene-join-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + server/licenses/lucene-join-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - .../licenses/lucene-memory-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + .../licenses/lucene-memory-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - server/licenses/lucene-misc-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + server/licenses/lucene-misc-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - .../licenses/lucene-queries-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + .../licenses/lucene-queries-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - .../lucene-queryparser-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + .../lucene-queryparser-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - .../licenses/lucene-sandbox-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + .../licenses/lucene-sandbox-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - .../lucene-spatial-extras-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + .../lucene-spatial-extras-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - .../lucene-spatial3d-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + .../lucene-spatial3d-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - .../licenses/lucene-suggest-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + .../licenses/lucene-suggest-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - .../licenses/lucene-core-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 + .../licenses/lucene-core-9.0.0-snapshot-19537578dd4.jar.sha1 | 1 - 48 files changed, 25 insertions(+), 24 deletions(-) create mode 100644 modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-19537578dd4.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-19537578dd4.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-19537578dd4.jar.sha1 create mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-19537578dd4.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-19537578dd4.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-19537578dd4.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-19537578dd4.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-19537578dd4.jar.sha1 create mode 100644 server/licenses/lucene-analysis-common-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 server/licenses/lucene-analysis-common-9.0.0-snapshot-19537578dd4.jar.sha1 create mode 100644 server/licenses/lucene-backward-codecs-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-9.0.0-snapshot-19537578dd4.jar.sha1 create mode 100644 server/licenses/lucene-core-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 server/licenses/lucene-core-9.0.0-snapshot-19537578dd4.jar.sha1 create mode 100644 server/licenses/lucene-grouping-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-9.0.0-snapshot-19537578dd4.jar.sha1 create mode 100644 server/licenses/lucene-highlighter-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-9.0.0-snapshot-19537578dd4.jar.sha1 create mode 100644 server/licenses/lucene-join-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 server/licenses/lucene-join-9.0.0-snapshot-19537578dd4.jar.sha1 create mode 100644 server/licenses/lucene-memory-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 server/licenses/lucene-memory-9.0.0-snapshot-19537578dd4.jar.sha1 create mode 100644 server/licenses/lucene-misc-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 server/licenses/lucene-misc-9.0.0-snapshot-19537578dd4.jar.sha1 create mode 100644 server/licenses/lucene-queries-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 server/licenses/lucene-queries-9.0.0-snapshot-19537578dd4.jar.sha1 create mode 100644 server/licenses/lucene-queryparser-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-9.0.0-snapshot-19537578dd4.jar.sha1 create mode 100644 server/licenses/lucene-sandbox-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-9.0.0-snapshot-19537578dd4.jar.sha1 create mode 100644 server/licenses/lucene-spatial-extras-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-9.0.0-snapshot-19537578dd4.jar.sha1 create mode 100644 server/licenses/lucene-spatial3d-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-9.0.0-snapshot-19537578dd4.jar.sha1 create mode 100644 server/licenses/lucene-suggest-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-9.0.0-snapshot-19537578dd4.jar.sha1 create mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-145747b0f3b.jar.sha1 delete mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-19537578dd4.jar.sha1 diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 1532171514153..b2a1580daf41b 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,5 +1,5 @@ elasticsearch = 8.0.0 -lucene = 9.0.0-snapshot-19537578dd4 +lucene = 9.0.0-snapshot-145747b0f3b bundled_jdk_vendor = adoptium bundled_jdk = 16.0.2+7 diff --git a/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-145747b0f3b.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..ea0f331e68395 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +5e048140ca7872b62490ee20716638cd6db6be50 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-19537578dd4.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index b52652fe37380..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -60103d458504e770c481b1572db407505a377cdb \ No newline at end of file diff --git a/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy b/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy index 763a06156828e..95b919b75bf85 100644 --- a/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy +++ b/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy @@ -18,4 +18,5 @@ grant { permission org.elasticsearch.script.ClassPermission "java.lang.Math"; permission org.elasticsearch.script.ClassPermission "org.apache.lucene.util.MathUtil"; permission org.elasticsearch.script.ClassPermission "org.apache.lucene.util.SloppyMath"; + permission org.elasticsearch.script.ClassPermission "org.apache.lucene.expressions.js.ExpressionMath"; }; diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-145747b0f3b.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..af0c328a839c2 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +d9b9f4452c6725189da2a2af027a2e2143625ba8 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-19537578dd4.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index db23dfa5e8d9e..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -50342351e5de9087521ba3011d36ec1b3979c550 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-145747b0f3b.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..14aa5cdf27829 --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +32d24c896bf55e5189e43525498d07480f5406ac \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-19537578dd4.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index 53302045164bb..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -adaabceb975f55a313fbedb36b6c67c9bede6321 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-145747b0f3b.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..a4bb6bb34e041 --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +7513e87c1e344e21f023589451b9c76b5403d1fd \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-19537578dd4.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index 4d6de7683f453..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7f59e54768eca7a9386280f0dd2531d20e4b08fd \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-145747b0f3b.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..16d646ede98ee --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +8126704f9a92b63ffe0ad37e08bec05747be7d2c \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-19537578dd4.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index 6a3be766f9b10..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1b1222e99180a9f17eaca1c1fec02fdc3d78b037 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-145747b0f3b.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..f19797042b237 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +a8c0281fdfd261dfbdf462e38f8703b254a7fa26 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-19537578dd4.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index b6848d9ae06ed..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f2fe5d44c3bcaf640738c2c195ee28c29fbd1d04 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-145747b0f3b.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..1ecf1df1a9a09 --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +6afbc2d48055be4912431c3c5edd44aa8f430e13 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-19537578dd4.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index 8ca5dc0e9ad14..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c6c5892baf428f551c02cc509a675860f22dee58 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-145747b0f3b.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..403b3dcac9568 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +6a73f97d64c995bf0b778cdfa9d740afce5656bf \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-19537578dd4.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index df748c9a7ece1..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -44785e4c17e7389c76b1fab716b3b467b4e69247 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-analysis-common-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..cccf91e827e18 --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +9643e0dbe5f57b863796c668e8e6553592925b12 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-analysis-common-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index c48180ec54199..0000000000000 --- a/server/licenses/lucene-analysis-common-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -32aa77f6577d4732a7ca92ae3b973040ee3b5056 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..73da813bc4be6 --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +8954fab578e7d3a771e550b7aa44a903509c7469 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index efa0148c974fa..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -de34bec3d7056dcb3286bcb4ba7ef6a419bffd77 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-core-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..9aa69d08997e7 --- /dev/null +++ b/server/licenses/lucene-core-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +49b0f2ad8697d63766971a8231fc26da238bd4f0 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-core-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index ef358bc0a7b02..0000000000000 --- a/server/licenses/lucene-core-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b12a01eed8a8101d80195968f43976bde37774d \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-grouping-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..16f0ccf48d068 --- /dev/null +++ b/server/licenses/lucene-grouping-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +94d284bcefc82dcdc0690dc887815e457b22dec2 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-grouping-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index 545b1bb5f914d..0000000000000 --- a/server/licenses/lucene-grouping-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -52bcb73182edf7ddc1001b1d20874f35a1d8b488 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-highlighter-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..0a73e03d21b72 --- /dev/null +++ b/server/licenses/lucene-highlighter-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +fcea9b9bfeb175117420b9035ae8ca359f951ca6 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-highlighter-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index 45db98f5ea973..0000000000000 --- a/server/licenses/lucene-highlighter-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b4db1727d37a9c7ad7a848f63b888c5515cbd13e \ No newline at end of file diff --git a/server/licenses/lucene-join-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-join-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..9bda266cf5df2 --- /dev/null +++ b/server/licenses/lucene-join-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +ed7084ed5c6f053f98903fc14fe4d6af5a8c3e06 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-join-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index 814058f556540..0000000000000 --- a/server/licenses/lucene-join-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -18de1307e21f198904d28fecb34491be94e35d0b \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-memory-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..8f6d4b87dfdf4 --- /dev/null +++ b/server/licenses/lucene-memory-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +5cc882ea8863d49e68c29e887b0d0a632c7313a0 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-memory-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index 4e8d2f7a6f899..0000000000000 --- a/server/licenses/lucene-memory-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -317eb2399cd1e81895cf040cdad338c55e52731c \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-misc-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..b876a5ad6ef57 --- /dev/null +++ b/server/licenses/lucene-misc-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +6de43d3885f1c3bb74ff6e8781b47490af3ea9b5 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-misc-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index d1fd76330c937..0000000000000 --- a/server/licenses/lucene-misc-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f464b6135050dab0b7e00b6479b0fe72259a778e \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-queries-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..0f5be6d392f08 --- /dev/null +++ b/server/licenses/lucene-queries-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +e7804697a00916ba4aade5227d970248f04f221e \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-queries-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index 39e6c6df9a3bf..0000000000000 --- a/server/licenses/lucene-queries-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -21ed7fde948dfb04846227de03d77c37fcaa469e \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-queryparser-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..0a1e6f87aa228 --- /dev/null +++ b/server/licenses/lucene-queryparser-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +faad7fe8ee8fad3cc765ad14eb4d50d02e2e7174 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-queryparser-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index b587060d3cbec..0000000000000 --- a/server/licenses/lucene-queryparser-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -cad0467e25c04e42f714edc4fb754a4fa372732f \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-sandbox-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..f57fb4488354f --- /dev/null +++ b/server/licenses/lucene-sandbox-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +c85fd94ae1632cea891ed408423fd76ca227a029 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-sandbox-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index 250594c182648..0000000000000 --- a/server/licenses/lucene-sandbox-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3258e38dea3a632fef6c89ccde9a8b0de6b98cf8 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-spatial-extras-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..702005233cb99 --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +832c2a016c02ab56e49c3ad2439ce8b4c2d9271c \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-spatial-extras-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index 3d97ce0710780..0000000000000 --- a/server/licenses/lucene-spatial-extras-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f60c1dbd261d57d76cc314437cffeaa6e379b866 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-spatial3d-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..bf93a4b619aa6 --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +9646ff125bf2084796dce5b0843c00d68d3ffbeb \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-spatial3d-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index 615c93f84a378..0000000000000 --- a/server/licenses/lucene-spatial3d-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dc78c6635a11be25b803ee7893e04e79284aba81 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-suggest-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..f40e8f60fc3d4 --- /dev/null +++ b/server/licenses/lucene-suggest-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +50cdb301a2d69a7dbc3c8f109d6ccb9af3546146 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.0.0-snapshot-19537578dd4.jar.sha1 b/server/licenses/lucene-suggest-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index 481c4335938f3..0000000000000 --- a/server/licenses/lucene-suggest-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -beba40f3e559cb33c49acce1d7919c62fb098361 \ No newline at end of file diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-145747b0f3b.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-145747b0f3b.jar.sha1 new file mode 100644 index 0000000000000..9aa69d08997e7 --- /dev/null +++ b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-145747b0f3b.jar.sha1 @@ -0,0 +1 @@ +49b0f2ad8697d63766971a8231fc26da238bd4f0 \ No newline at end of file diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-19537578dd4.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-19537578dd4.jar.sha1 deleted file mode 100644 index ef358bc0a7b02..0000000000000 --- a/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-19537578dd4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b12a01eed8a8101d80195968f43976bde37774d \ No newline at end of file From b7cca7d9d04531b3d7ad69c3ec91e9560e138160 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 15 Sep 2021 12:32:08 +0100 Subject: [PATCH 45/67] Allow async check index in searchable snapshot tests --- .../store/input/BaseSearchableSnapshotIndexInput.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/BaseSearchableSnapshotIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/BaseSearchableSnapshotIndexInput.java index 4777699fcf692..c7916e47b64ad 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/BaseSearchableSnapshotIndexInput.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/BaseSearchableSnapshotIndexInput.java @@ -300,9 +300,12 @@ protected final boolean assertCurrentThreadMayAccessBlobStore() { // Cache prewarming also runs on a dedicated thread pool. || threadName.contains('[' + SearchableSnapshots.CACHE_PREWARMING_THREAD_POOL_NAME + ']') - // Unit tests access the blob store on the main test thread; simplest just to permit this rather than have them override this + // Unit tests access the blob store on the main test thread, or via an asynchronous + // checkindex call; + // simplest just to permit this rather than have them override this // method somehow. || threadName.startsWith("TEST-") + || threadName.startsWith("async-check-index") || threadName.startsWith("LuceneTestCase") : "current thread [" + Thread.currentThread() + "] may not read " + fileInfo; return true; } From a6aee034ceb3e93c30db3c17ac3b39d711f1d728 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 16 Sep 2021 21:16:05 -0400 Subject: [PATCH 46/67] Disable sort optimization while query Lucene changes --- .../elasticsearch/index/engine/LuceneChangesSnapshot.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java b/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java index c3a0cf38d1d22..2c8716c9cfd9d 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java +++ b/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java @@ -241,8 +241,10 @@ private TopDocs searchOperations(ScoreDoc after) throws IOException { .add(LongPoint.newRangeQuery(SeqNoFieldMapper.NAME, Math.max(fromSeqNo, lastSeenSeqNo), toSeqNo), BooleanClause.Occur.MUST) .add(Queries.newNonNestedFilter(), BooleanClause.Occur.MUST) // exclude non-root nested documents .build(); - final Sort sortedBySeqNo = new Sort(new SortField(SeqNoFieldMapper.NAME, SortField.Type.LONG)); - return indexSearcher.searchAfter(after, rangeQuery, searchBatchSize, sortedBySeqNo); + final SortField sortBySeqNo = new SortField(SeqNoFieldMapper.NAME, SortField.Type.LONG); + // NORELEASE: We should enable the sort optimization to speed up CCR and peer recoveries. + sortBySeqNo.setOptimizeSortWithPoints(false); + return indexSearcher.searchAfter(after, rangeQuery, searchBatchSize, new Sort(sortBySeqNo)); } private Translog.Operation readDocAsOp(int docIndex) throws IOException { From c7030b77177c46261b8952e2cc98f6b2e15385c4 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 16 Sep 2021 21:35:46 -0400 Subject: [PATCH 47/67] update shas --- .../lucene-expressions-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - .../lucene-analyzers-icu-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - ...ucene-analyzers-kuromoji-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - .../lucene-analyzers-nori-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - ...ucene-analyzers-phonetic-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - ...lucene-analyzers-smartcn-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - ...lucene-analyzers-stempel-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - ...ene-analyzers-morfologik-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - .../lucene-analyzers-common-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - .../lucene-backward-codecs-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - server/licenses/lucene-core-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - .../lucene-grouping-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - .../lucene-highlighter-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - server/licenses/lucene-join-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - .../licenses/lucene-memory-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - server/licenses/lucene-misc-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - .../licenses/lucene-queries-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - .../lucene-queryparser-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - .../licenses/lucene-sandbox-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - .../lucene-spatial-extras-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - .../lucene-spatial3d-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - .../licenses/lucene-suggest-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - .../licenses/lucene-core-8.10.0-snapshot-bf2fcb53079.jar.sha1 | 1 - 23 files changed, 23 deletions(-) delete mode 100644 modules/lang-expression/licenses/lucene-expressions-8.10.0-snapshot-bf2fcb53079.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-8.10.0-snapshot-bf2fcb53079.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.10.0-snapshot-bf2fcb53079.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analyzers-nori-8.10.0-snapshot-bf2fcb53079.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.10.0-snapshot-bf2fcb53079.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.10.0-snapshot-bf2fcb53079.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.10.0-snapshot-bf2fcb53079.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.10.0-snapshot-bf2fcb53079.jar.sha1 delete mode 100644 server/licenses/lucene-analyzers-common-8.10.0-snapshot-bf2fcb53079.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-8.10.0-snapshot-bf2fcb53079.jar.sha1 delete mode 100644 server/licenses/lucene-core-8.10.0-snapshot-bf2fcb53079.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-8.10.0-snapshot-bf2fcb53079.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-8.10.0-snapshot-bf2fcb53079.jar.sha1 delete mode 100644 server/licenses/lucene-join-8.10.0-snapshot-bf2fcb53079.jar.sha1 delete mode 100644 server/licenses/lucene-memory-8.10.0-snapshot-bf2fcb53079.jar.sha1 delete mode 100644 server/licenses/lucene-misc-8.10.0-snapshot-bf2fcb53079.jar.sha1 delete mode 100644 server/licenses/lucene-queries-8.10.0-snapshot-bf2fcb53079.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-8.10.0-snapshot-bf2fcb53079.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-8.10.0-snapshot-bf2fcb53079.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-8.10.0-snapshot-bf2fcb53079.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-8.10.0-snapshot-bf2fcb53079.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-8.10.0-snapshot-bf2fcb53079.jar.sha1 delete mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-8.10.0-snapshot-bf2fcb53079.jar.sha1 diff --git a/modules/lang-expression/licenses/lucene-expressions-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 0e051c6381995..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -97f27306f1817475b30e52f48b1407bc5d696d59 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index dad0264f8c937..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -97560884369643ba928b7e683e092699330c4f5b \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index df913d91535b4..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3bc69fe29001399cfc98ddbd9f1d27864fa6a4a1 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index ec341f6b95ed8..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -497cdb3e2b5dd63d735cd60140f7c18533f3c401 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 90e9a1b66cfc0..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4fa14c42bf5864d28a7abfe197e79cf89b195c1b \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 4ebf1040e67b1..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -18971c579487a7c55a65b055fec07d9286ed49ce \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 1476b72761e58..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6fcf11017c3bce6e33d5cda04e3ef49560baa2a2 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index f5ac5de21f228..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -af5750b21d1d38352b49561c63b039723911a0d4 \ No newline at end of file diff --git a/server/licenses/lucene-analyzers-common-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-analyzers-common-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index fdede7b9c4cdc..0000000000000 --- a/server/licenses/lucene-analyzers-common-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -eb63f6ecd58a7e27a02b533b9c1e6cdb68f506fc \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-backward-codecs-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index d7b5d232b1d5b..0000000000000 --- a/server/licenses/lucene-backward-codecs-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2482a84e5e26a3eaf0bd7c5a77efc60435c7f688 \ No newline at end of file diff --git a/server/licenses/lucene-core-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-core-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 2a546f6674e06..0000000000000 --- a/server/licenses/lucene-core-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f33b45dbbce59e727e5a82dddab48c8c7681e25b \ No newline at end of file diff --git a/server/licenses/lucene-grouping-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-grouping-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index c39c757ef5945..0000000000000 --- a/server/licenses/lucene-grouping-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4020eb4e53d759fa11819b8d6b6a49422f51abe8 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-highlighter-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index eda61949a36fd..0000000000000 --- a/server/licenses/lucene-highlighter-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b2c9682be68699860aeeb9b048665ab6cf259c7c \ No newline at end of file diff --git a/server/licenses/lucene-join-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-join-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 60e4cb4bed007..0000000000000 --- a/server/licenses/lucene-join-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -be4f561159763de6a28d47b463438331a96c31c9 \ No newline at end of file diff --git a/server/licenses/lucene-memory-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-memory-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 0cb9423697434..0000000000000 --- a/server/licenses/lucene-memory-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8fc0745c4a589cdfbd56d156236fd91dbab0dacb \ No newline at end of file diff --git a/server/licenses/lucene-misc-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-misc-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 4860f9a1e54ea..0000000000000 --- a/server/licenses/lucene-misc-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7089f903268271acf6eb668918ac51f0cba15213 \ No newline at end of file diff --git a/server/licenses/lucene-queries-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-queries-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 0f6f657b0747b..0000000000000 --- a/server/licenses/lucene-queries-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -35a4945ac05c2aeb0c9e244098827fd7aeea1858 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-queryparser-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 48ccf9f681ce5..0000000000000 --- a/server/licenses/lucene-queryparser-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3d0929b7a5a2ba7f83d0357553f240f6d8362446 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-sandbox-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 062d28281a90a..0000000000000 --- a/server/licenses/lucene-sandbox-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d1d6696a4857bb580f6fc4a93bd3307effddd736 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-spatial-extras-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index f7c69cd9622e3..0000000000000 --- a/server/licenses/lucene-spatial-extras-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bc52ac3d5fed41fde8b7ad95c7d5ce703b90377f \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-spatial3d-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index bacaf78b1aec2..0000000000000 --- a/server/licenses/lucene-spatial3d-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -57d8cc36815cf45eb16d43648c8d2a5b251b4e62 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-suggest-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 90ade912a8369..0000000000000 --- a/server/licenses/lucene-suggest-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -17e5a74d57ecb861e93c3cfbf4feb356a0571bbf \ No newline at end of file diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 2a546f6674e06..0000000000000 --- a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f33b45dbbce59e727e5a82dddab48c8c7681e25b \ No newline at end of file From 67270038d6610e4b78db445ac656ecf52df37a5e Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 16 Sep 2021 22:34:37 -0400 Subject: [PATCH 48/67] fix tests --- .../analysis/common/CommonAnalysisFactoryTests.java | 2 -- .../verify_version_constants/VerifyVersionConstantsIT.java | 6 ++++++ .../indices/analysis/AnalysisFactoryTestCase.java | 4 +--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java index 72b1ee7ff6bef..5f047617ac6e1 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java @@ -135,8 +135,6 @@ protected Map> getTokenFilters() { filters.put("brazilianstem", BrazilianStemTokenFilterFactory.class); filters.put("czechstem", CzechStemTokenFilterFactory.class); filters.put("germanstem", GermanStemTokenFilterFactory.class); - filters.put("telugunormalization", TeluguNormalizationFilterFactory.class); - filters.put("telugustem", TeluguStemFilterFactory.class); // this filter is not exposed and should only be used internally filters.put("fixedshingle", Void.class); return filters; diff --git a/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java b/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java index ee9e922e66871..78979caf1466d 100644 --- a/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java +++ b/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java @@ -24,6 +24,12 @@ public class VerifyVersionConstantsIT extends ESRestTestCase { + public void testEmpty() throws Exception { + + } + + // NORELEASE + @AwaitsFix(bugUrl = "Lucene version on 7.x is 8.10 instead of 8.9 - Remove the testEmpty above") public void testLuceneVersionConstant() throws IOException, ParseException { final Response response = client().performRequest(new Request("GET", "/")); assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); diff --git a/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java index dfd9736880792..f18b02ee1fc8f 100644 --- a/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java @@ -191,9 +191,7 @@ public abstract class AnalysisFactoryTestCase extends ESTestCase { // LUCENE-9574 entry("dropifflagged", Void.class), // LUCENE-9575 - entry("patterntyping", Void.class), - entry("telugustem", Void.class), - entry("telugunormalization", Void.class)); + entry("patterntyping", Void.class)); static final Map> KNOWN_CHARFILTERS = Map.of( From af0ee840964b188a047904c195e5cd1983110274 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 16 Sep 2021 23:25:22 -0400 Subject: [PATCH 49/67] fix javadocs --- .../java/org/elasticsearch/plugins/spi/SPIClassIterator.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/plugins/spi/SPIClassIterator.java b/server/src/main/java/org/elasticsearch/plugins/spi/SPIClassIterator.java index fb37b2e59ead0..b8b70a8e122e9 100644 --- a/server/src/main/java/org/elasticsearch/plugins/spi/SPIClassIterator.java +++ b/server/src/main/java/org/elasticsearch/plugins/spi/SPIClassIterator.java @@ -40,8 +40,6 @@ * This is a light impl of {@link java.util.ServiceLoader} but is guaranteed to * be bug-free regarding classpath order and does not instantiate or initialize * the classes found. - * - * @lucene.internal */ @SuppressForbidden(reason="Copied from Lucene") public final class SPIClassIterator implements Iterator> { From ba960a4c09f74d53965f5899a984f7cc124ce64d Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 16 Sep 2021 23:51:03 -0400 Subject: [PATCH 50/67] awaitfix testRecovery --- .../java/org/elasticsearch/upgrades/FullClusterRestartIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index d6dcd01a3e053..7493a2ae08213 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -657,6 +657,8 @@ public void testEmptyShard() throws IOException { * Tests recovery of an index with or without a translog and the * statistics we gather about that. */ + // NORELEASE + @AwaitsFix(bugUrl = "Lucene version on 7.x is 8.10 instead of 8.9") public void testRecovery() throws Exception { int count; boolean shouldHaveTranslog; From db2be3489c75a1b33aecb24e6565aa0793a6645c Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Sun, 19 Sep 2021 23:15:12 -0400 Subject: [PATCH 51/67] fix compile --- .../apache/lucene/search/vectorhighlight/CustomFieldQuery.java | 2 +- .../elasticsearch/index/query/CombinedFieldsQueryBuilder.java | 2 +- .../index/query/CombinedFieldsQueryBuilderTests.java | 2 +- .../index/query/CombinedFieldsQueryParsingTests.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java b/server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java index 138f6bb6b4241..732de53e92932 100644 --- a/server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java +++ b/server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java @@ -12,8 +12,8 @@ import org.apache.lucene.index.Term; import org.apache.lucene.queries.BlendedTermQuery; import org.apache.lucene.queries.spans.SpanTermQuery; +import org.apache.lucene.sandbox.search.CombinedFieldQuery; import org.apache.lucene.search.BoostQuery; -import org.apache.lucene.search.CombinedFieldQuery; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.MultiPhraseQuery; import org.apache.lucene.search.PhraseQuery; diff --git a/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java index cf907ed17d20a..8a1ed211d03b9 100644 --- a/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java @@ -12,11 +12,11 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute; import org.apache.lucene.index.Term; +import org.apache.lucene.sandbox.search.CombinedFieldQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostAttribute; import org.apache.lucene.search.BoostQuery; -import org.apache.lucene.search.CombinedFieldQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.similarities.BM25Similarity; import org.apache.lucene.search.similarities.Similarity; diff --git a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java index ca0f7d6ae2316..7947ad35ef4b8 100644 --- a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java @@ -8,8 +8,8 @@ package org.elasticsearch.index.query; +import org.apache.lucene.sandbox.search.CombinedFieldQuery; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.CombinedFieldQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; diff --git a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java index e8d508a6e1ed0..995d87efc4711 100644 --- a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java @@ -13,10 +13,10 @@ import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.index.Term; +import org.apache.lucene.sandbox.search.CombinedFieldQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; -import org.apache.lucene.search.CombinedFieldQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; From af151d384c249bde87876d53842d177bf3532034 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 20 Sep 2021 10:23:37 +0100 Subject: [PATCH 52/67] New snapshot, including lucene 8.10 version constant --- build-tools-internal/version.properties | 2 +- server/src/main/java/org/elasticsearch/Version.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index b2a1580daf41b..326a92746d040 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,5 +1,5 @@ elasticsearch = 8.0.0 -lucene = 9.0.0-snapshot-145747b0f3b +lucene = 9.0.0-snapshot-32a0a16aff0 bundled_jdk_vendor = adoptium bundled_jdk = 16.0.2+7 diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index c8bca542b1b13..5d0afb150a1ac 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -89,8 +89,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_7_14_1 = new Version(7140199, org.apache.lucene.util.Version.LUCENE_8_9_0); public static final Version V_7_14_2 = new Version(7140299, org.apache.lucene.util.Version.LUCENE_8_9_0); public static final Version V_7_15_0 = new Version(7150099, org.apache.lucene.util.Version.LUCENE_8_9_0); - // NORELEASE: The lucene version on 7_16 is 8.10 - public static final Version V_7_16_0 = new Version(7160099, org.apache.lucene.util.Version.LUCENE_8_9_0); + public static final Version V_7_16_0 = new Version(7160099, org.apache.lucene.util.Version.LUCENE_8_10_0); public static final Version V_8_0_0 = new Version(8000099, org.apache.lucene.util.Version.LUCENE_9_0_0); public static final Version CURRENT = V_8_0_0; From 7eb7921e0ca8577e207fbcb45ad70c37615415c9 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 20 Sep 2021 10:32:40 +0100 Subject: [PATCH 53/67] precommit --- .../lucene-expressions-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - .../lucene-expressions-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + .../lucene-analysis-icu-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - .../lucene-analysis-icu-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + .../lucene-analysis-kuromoji-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - .../lucene-analysis-kuromoji-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + .../lucene-analysis-nori-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - .../lucene-analysis-nori-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + .../lucene-analysis-phonetic-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - .../lucene-analysis-phonetic-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + .../lucene-analysis-smartcn-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - .../lucene-analysis-smartcn-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + .../lucene-analysis-stempel-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - .../lucene-analysis-stempel-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + ...ucene-analysis-morfologik-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - ...ucene-analysis-morfologik-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + .../lucene-analysis-common-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - .../lucene-analysis-common-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + .../lucene-backward-codecs-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - .../lucene-backward-codecs-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + server/licenses/lucene-core-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - server/licenses/lucene-core-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + .../licenses/lucene-grouping-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - .../licenses/lucene-grouping-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + .../lucene-highlighter-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - .../lucene-highlighter-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + server/licenses/lucene-join-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - server/licenses/lucene-join-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + .../licenses/lucene-memory-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - .../licenses/lucene-memory-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + server/licenses/lucene-misc-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - server/licenses/lucene-misc-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + .../licenses/lucene-queries-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - .../licenses/lucene-queries-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + .../lucene-queryparser-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - .../lucene-queryparser-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + .../licenses/lucene-sandbox-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - .../licenses/lucene-sandbox-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + .../lucene-spatial-extras-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - .../lucene-spatial-extras-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + .../lucene-spatial3d-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - .../lucene-spatial3d-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + .../licenses/lucene-suggest-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - .../licenses/lucene-suggest-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + .../licenses/lucene-core-9.0.0-snapshot-145747b0f3b.jar.sha1 | 1 - .../licenses/lucene-core-9.0.0-snapshot-32a0a16aff0.jar.sha1 | 1 + 46 files changed, 23 insertions(+), 23 deletions(-) delete mode 100644 modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-32a0a16aff0.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-32a0a16aff0.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-32a0a16aff0.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-32a0a16aff0.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-32a0a16aff0.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-32a0a16aff0.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-32a0a16aff0.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-32a0a16aff0.jar.sha1 delete mode 100644 server/licenses/lucene-analysis-common-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 server/licenses/lucene-analysis-common-9.0.0-snapshot-32a0a16aff0.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 server/licenses/lucene-backward-codecs-9.0.0-snapshot-32a0a16aff0.jar.sha1 delete mode 100644 server/licenses/lucene-core-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 server/licenses/lucene-core-9.0.0-snapshot-32a0a16aff0.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 server/licenses/lucene-grouping-9.0.0-snapshot-32a0a16aff0.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 server/licenses/lucene-highlighter-9.0.0-snapshot-32a0a16aff0.jar.sha1 delete mode 100644 server/licenses/lucene-join-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 server/licenses/lucene-join-9.0.0-snapshot-32a0a16aff0.jar.sha1 delete mode 100644 server/licenses/lucene-memory-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 server/licenses/lucene-memory-9.0.0-snapshot-32a0a16aff0.jar.sha1 delete mode 100644 server/licenses/lucene-misc-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 server/licenses/lucene-misc-9.0.0-snapshot-32a0a16aff0.jar.sha1 delete mode 100644 server/licenses/lucene-queries-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 server/licenses/lucene-queries-9.0.0-snapshot-32a0a16aff0.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 server/licenses/lucene-queryparser-9.0.0-snapshot-32a0a16aff0.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 server/licenses/lucene-sandbox-9.0.0-snapshot-32a0a16aff0.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 server/licenses/lucene-spatial-extras-9.0.0-snapshot-32a0a16aff0.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 server/licenses/lucene-spatial3d-9.0.0-snapshot-32a0a16aff0.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 server/licenses/lucene-suggest-9.0.0-snapshot-32a0a16aff0.jar.sha1 delete mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-145747b0f3b.jar.sha1 create mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-32a0a16aff0.jar.sha1 diff --git a/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-145747b0f3b.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index ea0f331e68395..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5e048140ca7872b62490ee20716638cd6db6be50 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..03931cd3c7e1e --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +561301335ba14e07ec89adff0e2321cb79146a48 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-145747b0f3b.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index af0c328a839c2..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d9b9f4452c6725189da2a2af027a2e2143625ba8 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..4f21d907c34b8 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +5b64983cf184ddd4018d56114b086f3cebd734ad \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-145747b0f3b.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index 14aa5cdf27829..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -32d24c896bf55e5189e43525498d07480f5406ac \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..dbe66f8da442d --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +3e27dd79ed9fa24bfb998446526baf4f16cc7a05 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-145747b0f3b.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index a4bb6bb34e041..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7513e87c1e344e21f023589451b9c76b5403d1fd \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..a3fc882d0c959 --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +48f2c1dec0aea566c3ac7c62322b52887a054bf6 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-145747b0f3b.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index 16d646ede98ee..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8126704f9a92b63ffe0ad37e08bec05747be7d2c \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..81fd94a0b42bd --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +ee44e4909d2c2b8686606fd4d11b2b817cbd5b3f \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-145747b0f3b.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index f19797042b237..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a8c0281fdfd261dfbdf462e38f8703b254a7fa26 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..a1218843586a9 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +bf0d9a8984578032378342891c4d1e6d3147b780 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-145747b0f3b.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index 1ecf1df1a9a09..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6afbc2d48055be4912431c3c5edd44aa8f430e13 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..2f8752fc5480d --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +4a88bb19fa4035d9841f76de0572826ce6c43915 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-145747b0f3b.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index 403b3dcac9568..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6a73f97d64c995bf0b778cdfa9d740afce5656bf \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..437b6567b291b --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +88837c81d62b322d83669cf16f85b7bc66cfe3e8 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-analysis-common-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index cccf91e827e18..0000000000000 --- a/server/licenses/lucene-analysis-common-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9643e0dbe5f57b863796c668e8e6553592925b12 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-analysis-common-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..afff795051ecd --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +bce23b7816187b44700ee646ce1afbdac4423dcc \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index 73da813bc4be6..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8954fab578e7d3a771e550b7aa44a903509c7469 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..be9eb98abf9e4 --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +63ef3d7a09cc122a77d4d84af9b5c03efa480d01 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-core-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index 9aa69d08997e7..0000000000000 --- a/server/licenses/lucene-core-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -49b0f2ad8697d63766971a8231fc26da238bd4f0 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-core-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..9c473fb8da725 --- /dev/null +++ b/server/licenses/lucene-core-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +49c6fdf8fdca15e4bc97973056e883e34de82b7a \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-grouping-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index 16f0ccf48d068..0000000000000 --- a/server/licenses/lucene-grouping-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -94d284bcefc82dcdc0690dc887815e457b22dec2 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-grouping-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..38aee36eb1ff5 --- /dev/null +++ b/server/licenses/lucene-grouping-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +0d2bf347c51f37e2f659bb8a015189e8743ec8e2 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-highlighter-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index 0a73e03d21b72..0000000000000 --- a/server/licenses/lucene-highlighter-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fcea9b9bfeb175117420b9035ae8ca359f951ca6 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-highlighter-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..996fb57e4c01e --- /dev/null +++ b/server/licenses/lucene-highlighter-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +d0159455b0ca8930d9caf95a7275ba192f4bd9ab \ No newline at end of file diff --git a/server/licenses/lucene-join-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-join-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index 9bda266cf5df2..0000000000000 --- a/server/licenses/lucene-join-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ed7084ed5c6f053f98903fc14fe4d6af5a8c3e06 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-join-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..3d0703d61dd55 --- /dev/null +++ b/server/licenses/lucene-join-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +342f80a704dd37012253a6b19bfad88e08287791 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-memory-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index 8f6d4b87dfdf4..0000000000000 --- a/server/licenses/lucene-memory-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5cc882ea8863d49e68c29e887b0d0a632c7313a0 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-memory-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..7525a6064b0a8 --- /dev/null +++ b/server/licenses/lucene-memory-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +5944a54b0d46ee4554f53c40c0b92ac7f1d70c1b \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-misc-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index b876a5ad6ef57..0000000000000 --- a/server/licenses/lucene-misc-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6de43d3885f1c3bb74ff6e8781b47490af3ea9b5 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-misc-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..47757720517e2 --- /dev/null +++ b/server/licenses/lucene-misc-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +1553866522ffdd4aa2123fa14b1a3ed2d1d64713 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-queries-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index 0f5be6d392f08..0000000000000 --- a/server/licenses/lucene-queries-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e7804697a00916ba4aade5227d970248f04f221e \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-queries-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..aa3d74f718093 --- /dev/null +++ b/server/licenses/lucene-queries-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +0e695c67e7de44914d6b9a8f95d06c2948ef8b7e \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-queryparser-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index 0a1e6f87aa228..0000000000000 --- a/server/licenses/lucene-queryparser-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -faad7fe8ee8fad3cc765ad14eb4d50d02e2e7174 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-queryparser-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..f24a672cafa29 --- /dev/null +++ b/server/licenses/lucene-queryparser-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +21687ed2a47b1b78eee80c2bd6de3f11eb63e473 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-sandbox-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index f57fb4488354f..0000000000000 --- a/server/licenses/lucene-sandbox-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c85fd94ae1632cea891ed408423fd76ca227a029 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-sandbox-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..ef971205715f0 --- /dev/null +++ b/server/licenses/lucene-sandbox-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +52acae7ec3773b136019613a4dd897cb915da856 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-spatial-extras-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index 702005233cb99..0000000000000 --- a/server/licenses/lucene-spatial-extras-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -832c2a016c02ab56e49c3ad2439ce8b4c2d9271c \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..dca5dc13ce1cb --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +26b1aebb92871f79b6f2c8dab8b3f47371ec30da \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-spatial3d-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index bf93a4b619aa6..0000000000000 --- a/server/licenses/lucene-spatial3d-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9646ff125bf2084796dce5b0843c00d68d3ffbeb \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-spatial3d-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..459ace5f18578 --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +ca9b60241861bfa17401bd28287f09af61fa1c5c \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.0.0-snapshot-145747b0f3b.jar.sha1 b/server/licenses/lucene-suggest-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index f40e8f60fc3d4..0000000000000 --- a/server/licenses/lucene-suggest-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -50cdb301a2d69a7dbc3c8f109d6ccb9af3546146 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-suggest-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..f1acea0dd5681 --- /dev/null +++ b/server/licenses/lucene-suggest-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +e2449a6766c2e1c78628ce1dfbd00b172eb45b37 \ No newline at end of file diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-145747b0f3b.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-145747b0f3b.jar.sha1 deleted file mode 100644 index 9aa69d08997e7..0000000000000 --- a/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-145747b0f3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -49b0f2ad8697d63766971a8231fc26da238bd4f0 \ No newline at end of file diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..9c473fb8da725 --- /dev/null +++ b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +49c6fdf8fdca15e4bc97973056e883e34de82b7a \ No newline at end of file From a4c878ae06babe25794e2fdc902e4435ab779065 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 20 Sep 2021 11:41:43 +0100 Subject: [PATCH 54/67] japanese completion filter --- plugins/analysis-kuromoji/build.gradle | 1 + .../JapaneseCompletionFilterFactory.java | 27 +++++++++++++++++++ .../kuromoji/AnalysisKuromojiPlugin.java | 2 ++ .../AnalysisKuromojiFactoryTests.java | 2 ++ 4 files changed, 32 insertions(+) create mode 100644 plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/JapaneseCompletionFilterFactory.java diff --git a/plugins/analysis-kuromoji/build.gradle b/plugins/analysis-kuromoji/build.gradle index 214cf0592070c..2fd8515f4ca6a 100644 --- a/plugins/analysis-kuromoji/build.gradle +++ b/plugins/analysis-kuromoji/build.gradle @@ -29,6 +29,7 @@ tasks.named("dependencyLicenses").configure { tasks.named('splitPackagesAudit').configure { // o.e.i.analysis is owned by server, these should be moved to a kuromoji package ignoreClasses 'org.elasticsearch.index.analysis.JapaneseStopTokenFilterFactory', + 'org.elasticsearch.index.analysis.JapaneseCompletionFilterFactory', 'org.elasticsearch.index.analysis.KuromojiAnalyzerProvider', 'org.elasticsearch.index.analysis.KuromojiBaseFormFilterFactory', 'org.elasticsearch.index.analysis.KuromojiIterationMarkCharFilterFactory', diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/JapaneseCompletionFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/JapaneseCompletionFilterFactory.java new file mode 100644 index 0000000000000..8379bd61a2b6c --- /dev/null +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/JapaneseCompletionFilterFactory.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.analysis; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.ja.JapaneseCompletionFilter; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; + +public class JapaneseCompletionFilterFactory extends AbstractTokenFilterFactory { + + public JapaneseCompletionFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); + } + + @Override + public TokenStream create(TokenStream tokenStream) { + return new JapaneseCompletionFilter(tokenStream); + } +} diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java index 2c2592e91e5e7..9437f8c7e2f60 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java @@ -11,6 +11,7 @@ import org.apache.lucene.analysis.Analyzer; import org.elasticsearch.index.analysis.AnalyzerProvider; import org.elasticsearch.index.analysis.CharFilterFactory; +import org.elasticsearch.index.analysis.JapaneseCompletionFilterFactory; import org.elasticsearch.index.analysis.JapaneseStopTokenFilterFactory; import org.elasticsearch.index.analysis.KuromojiAnalyzerProvider; import org.elasticsearch.index.analysis.KuromojiBaseFormFilterFactory; @@ -46,6 +47,7 @@ public Map> getTokenFilters() { extra.put("kuromoji_stemmer", KuromojiKatakanaStemmerFactory::new); extra.put("ja_stop", JapaneseStopTokenFilterFactory::new); extra.put("kuromoji_number", KuromojiNumberFilterFactory::new); + extra.put("japanese_completion", JapaneseCompletionFilterFactory::new); return extra; } diff --git a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/AnalysisKuromojiFactoryTests.java b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/AnalysisKuromojiFactoryTests.java index ec51c91dc1319..56850e3b068c0 100644 --- a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/AnalysisKuromojiFactoryTests.java +++ b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/AnalysisKuromojiFactoryTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.analysis; +import org.apache.lucene.analysis.ja.JapaneseCompletionFilterFactory; import org.apache.lucene.analysis.ja.JapaneseTokenizerFactory; import org.elasticsearch.indices.analysis.AnalysisFactoryTestCase; import org.elasticsearch.plugin.analysis.kuromoji.AnalysisKuromojiPlugin; @@ -31,6 +32,7 @@ protected Map> getTokenizers() { protected Map> getTokenFilters() { Map> filters = new HashMap<>(super.getTokenFilters()); filters.put("japanesebaseform", KuromojiBaseFormFilterFactory.class); + filters.put("japanesecompletion", JapaneseCompletionFilterFactory.class); filters.put("japanesepartofspeechstop", KuromojiPartOfSpeechFilterFactory.class); filters.put("japanesereadingform", KuromojiReadingFormFilterFactory.class); filters.put("japanesekatakanastem", KuromojiKatakanaStemmerFactory.class); From 46fc8c1e4c34ab27eab5367306a524ee466740bb Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 20 Sep 2021 15:06:58 +0100 Subject: [PATCH 55/67] No need for endianness shenanigans in TransportSearchHelper --- .../action/search/TransportSearchHelper.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java index b41a8dcc9a957..3cdd25c6e56a9 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java @@ -8,9 +8,8 @@ package org.elasticsearch.action.search; -import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.store.ByteBuffersDataOutput; import org.elasticsearch.common.io.stream.ByteArrayStreamInput; +import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; @@ -32,14 +31,13 @@ static InternalScrollSearchRequest internalScrollSearchRequest(ShardSearchContex static String buildScrollId(AtomicArray searchPhaseResults) { try { - // TODO replace with BytesStreamOutput to avoid endiannessreverser - ByteBuffersDataOutput out = new ByteBuffersDataOutput(); + BytesStreamOutput out = new BytesStreamOutput(); out.writeString(INCLUDE_CONTEXT_UUID); out.writeString(searchPhaseResults.length() == 1 ? ParsedScrollId.QUERY_AND_FETCH_TYPE : ParsedScrollId.QUERY_THEN_FETCH_TYPE); out.writeVInt(searchPhaseResults.asList().size()); for (SearchPhaseResult searchPhaseResult : searchPhaseResults.asList()) { out.writeString(searchPhaseResult.getContextId().getSessionId()); - CodecUtil.writeBELong(out, searchPhaseResult.getContextId().getId()); + out.writeLong(searchPhaseResult.getContextId().getId()); SearchShardTarget searchShardTarget = searchPhaseResult.getSearchShardTarget(); if (searchShardTarget.getClusterAlias() != null) { out.writeString( @@ -48,8 +46,7 @@ static String buildScrollId(AtomicArray searchPhase out.writeString(searchShardTarget.getNodeId()); } } - byte[] bytes = out.toArrayCopy(); - return Base64.getUrlEncoder().encodeToString(bytes); + return Base64.getUrlEncoder().encodeToString(out.copyBytes().array()); } catch (IOException e) { throw new UncheckedIOException(e); } From cd50069b7f3dd9a395201681647dd4b7e7ff78bf Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Mon, 20 Sep 2021 11:27:14 -0400 Subject: [PATCH 56/67] Remove NORELEASE --- .../org/elasticsearch/upgrades/FullClusterRestartIT.java | 2 -- .../verify_version_constants/VerifyVersionConstantsIT.java | 6 ------ 2 files changed, 8 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 7493a2ae08213..d6dcd01a3e053 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -657,8 +657,6 @@ public void testEmptyShard() throws IOException { * Tests recovery of an index with or without a translog and the * statistics we gather about that. */ - // NORELEASE - @AwaitsFix(bugUrl = "Lucene version on 7.x is 8.10 instead of 8.9") public void testRecovery() throws Exception { int count; boolean shouldHaveTranslog; diff --git a/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java b/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java index 78979caf1466d..ee9e922e66871 100644 --- a/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java +++ b/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java @@ -24,12 +24,6 @@ public class VerifyVersionConstantsIT extends ESRestTestCase { - public void testEmpty() throws Exception { - - } - - // NORELEASE - @AwaitsFix(bugUrl = "Lucene version on 7.x is 8.10 instead of 8.9 - Remove the testEmpty above") public void testLuceneVersionConstant() throws IOException, ParseException { final Response response = client().performRequest(new Request("GET", "/")); assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); From a4e6b3288fbd6090562025c1173c4b76df014dea Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Mon, 20 Sep 2021 12:30:20 -0400 Subject: [PATCH 57/67] Do not remove max_merge_at_once_explicit --- .../src/main/java/org/elasticsearch/index/IndexSettings.java | 2 ++ .../main/java/org/elasticsearch/index/MergePolicyConfig.java | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index aa8b27e9cb08b..bd8f180368c7f 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -568,6 +568,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti mergePolicyConfig::setFloorSegmentSetting); scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING, mergePolicyConfig::setMaxMergesAtOnce); + scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING, + mergePolicyConfig::setMaxMergesAtOnceExplicit); scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING, mergePolicyConfig::setMaxMergedSegment); scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING, diff --git a/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java b/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java index 96043538ac8ae..446b8dd6ef7a4 100644 --- a/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java +++ b/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java @@ -188,6 +188,10 @@ void setMaxMergedSegment(ByteSizeValue maxMergedSegment) { mergePolicy.setMaxMergedSegmentMB(maxMergedSegment.getMbFrac()); } + void setMaxMergesAtOnceExplicit(Integer maxMergeAtOnceExplicit) { + + } + void setMaxMergesAtOnce(Integer maxMergeAtOnce) { mergePolicy.setMaxMergeAtOnce(maxMergeAtOnce); } From e53d3bef7e0469f8e81bef5fc9fc01e84b380e9a Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Mon, 20 Sep 2021 13:13:57 -0400 Subject: [PATCH 58/67] use existing filtered store fields visitor --- .../RecoverySourcePruneMergePolicy.java | 45 +------------------ 1 file changed, 1 insertion(+), 44 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicy.java b/server/src/main/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicy.java index 9409d23dd75ef..2c8738bf85a66 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicy.java +++ b/server/src/main/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicy.java @@ -31,6 +31,7 @@ import org.apache.lucene.search.Weight; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BitSetIterator; +import org.elasticsearch.search.internal.FilterStoredFieldVisitor; import java.io.IOException; import java.util.Arrays; @@ -246,49 +247,5 @@ public StoredFieldsReader clone() { } } - - private static class FilterStoredFieldVisitor extends StoredFieldVisitor { - private final StoredFieldVisitor visitor; - - FilterStoredFieldVisitor(StoredFieldVisitor visitor) { - this.visitor = visitor; - } - - @Override - public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { - visitor.binaryField(fieldInfo, value); - } - - @Override - public void stringField(FieldInfo fieldInfo, String value) throws IOException { - visitor.stringField(fieldInfo, value); - } - - @Override - public void intField(FieldInfo fieldInfo, int value) throws IOException { - visitor.intField(fieldInfo, value); - } - - @Override - public void longField(FieldInfo fieldInfo, long value) throws IOException { - visitor.longField(fieldInfo, value); - } - - @Override - public void floatField(FieldInfo fieldInfo, float value) throws IOException { - visitor.floatField(fieldInfo, value); - } - - @Override - public void doubleField(FieldInfo fieldInfo, double value) throws IOException { - visitor.doubleField(fieldInfo, value); - } - - @Override - public Status needsField(FieldInfo fieldInfo) throws IOException { - return visitor.needsField(fieldInfo); - } - } } - } From 7790565a38a84e2efbcf1572bcff4fb366d20014 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Mon, 20 Sep 2021 14:16:27 -0400 Subject: [PATCH 59/67] minimize changes --- .../main/java/org/elasticsearch/search/SearchService.java | 1 - .../org/elasticsearch/search/query/QuerySearchResult.java | 5 ----- .../java/org/elasticsearch/test/AbstractQueryTestCase.java | 1 + .../xpack/core/security/support/Automatons.java | 2 +- 4 files changed, 2 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 54dff33436244..6b3c456ef18bc 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -640,7 +640,6 @@ private ReaderContext findReaderContext(ShardSearchContextId id, TransportReques } final ReaderContext reader = activeReaders.get(id.getId()); if (reader == null) { - logger.info("Unknown reader context [{}], available contexts are [{}]", id.getId(), activeReaders.keySet()); throw new SearchContextMissingException(id); } try { diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java index 8b6ea203ca300..f35cb57314dba 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java @@ -8,8 +8,6 @@ package org.elasticsearch.search.query; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.TotalHits; import org.elasticsearch.Version; @@ -36,8 +34,6 @@ public final class QuerySearchResult extends SearchPhaseResult { - private static final Logger logger = LogManager.getLogger(QuerySearchResult.class); - private int from; private int size; private TopDocsAndMaxScore topDocsAndMaxScore; @@ -174,7 +170,6 @@ public void topDocs(TopDocsAndMaxScore topDocs, DocValueFormat[] sortValueFormat } private void setTopDocs(TopDocsAndMaxScore topDocsAndMaxScore) { - logger.info("Setting topdocs with {} score docs", topDocsAndMaxScore.topDocs.scoreDocs.length); this.topDocsAndMaxScore = topDocsAndMaxScore; this.totalHits = topDocsAndMaxScore.topDocs.totalHits; this.maxScore = topDocsAndMaxScore.maxScore; diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java index a9abf8d8c14cb..a6804f4f6d564 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java @@ -9,6 +9,7 @@ package org.elasticsearch.test; import com.fasterxml.jackson.core.io.JsonStringEncoder; + import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java index 821453639b386..a52235d51be2c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java @@ -16,8 +16,8 @@ import org.elasticsearch.common.cache.CacheBuilder; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.common.util.set.Sets; import java.util.ArrayList; import java.util.Arrays; From 1cbf2cdb4e924b280446e05d8904c1fc4a13ec71 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Mon, 20 Sep 2021 14:18:53 -0400 Subject: [PATCH 60/67] Remove ram tree --- .../segments/IndicesSegmentResponse.java | 7 ------ .../elasticsearch/index/engine/Segment.java | 25 ++++++------------- .../segments/IndicesSegmentsRequestTests.java | 14 ----------- .../index/engine/InternalEngineTests.java | 4 --- 4 files changed, 7 insertions(+), 43 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java index 17cfd38e93f7d..fffcc28aeb948 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java @@ -128,13 +128,6 @@ protected void addCustomXContentFields(XContentBuilder builder, Params params) t if (segment.getSegmentSort() != null) { toXContent(builder, segment.getSegmentSort()); } - if (segment.ramTree != null) { - builder.startArray(Fields.RAM_TREE); - for (Accountable child : segment.ramTree.getChildResources()) { - toXContent(builder, child); - } - builder.endArray(); - } if (segment.attributes != null && segment.attributes.isEmpty() == false) { builder.field("attributes", segment.attributes); } diff --git a/server/src/main/java/org/elasticsearch/index/engine/Segment.java b/server/src/main/java/org/elasticsearch/index/engine/Segment.java index d04bf41aa0971..cc4fb43dc5de8 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Segment.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Segment.java @@ -44,7 +44,6 @@ public class Segment implements Writeable { public Boolean compound = null; public String mergeId; public Sort segmentSort; - public Accountable ramTree = null; public Map attributes; public Segment(StreamInput in) throws IOException { @@ -62,8 +61,7 @@ public Segment(StreamInput in) throws IOException { in.readLong(); // memoryInBytes } if (in.readBoolean()) { - // verbose mode - ramTree = readRamTree(in); + readRamTree(in); } segmentSort = readSegmentSort(in); if (in.readBoolean()) { @@ -170,11 +168,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(0); // memoryInBytes } - boolean verbose = ramTree != null; - out.writeBoolean(verbose); - if (verbose) { - writeRamTree(out, ramTree); - } + out.writeBoolean(false); writeSegmentSort(out, segmentSort); boolean hasAttributes = attributes != null; out.writeBoolean(hasAttributes); @@ -274,18 +268,13 @@ private void writeSegmentSort(StreamOutput out, Sort sort) throws IOException { } } - private Accountable readRamTree(StreamInput in) throws IOException { - final String name = in.readString(); - final long bytes = in.readVLong(); + private static void readRamTree(StreamInput in) throws IOException { + in.readString(); + in.readVLong(); int numChildren = in.readVInt(); - if (numChildren == 0) { - return Accountables.namedAccountable(name, bytes); - } - List children = new ArrayList<>(numChildren); - while (numChildren-- > 0) { - children.add(readRamTree(in)); + for (int i = 0; i < numChildren; i++) { + readRamTree(in); } - return Accountables.namedAccountable(name, children, bytes); } // the ram tree is written recursively since the depth is fairly low (5 or 6) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java index ecddf51c49cad..7190b7ec216c1 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.engine.Segment; import org.elasticsearch.index.MergePolicyConfig; import org.elasticsearch.indices.IndexClosedException; import org.elasticsearch.plugins.Plugin; @@ -19,7 +18,6 @@ import org.junit.Before; import java.util.Collection; -import java.util.List; import static org.hamcrest.Matchers.is; @@ -47,18 +45,6 @@ public void setupIndex() { client().admin().indices().prepareRefresh().get(); } - public void testBasic() { - IndicesSegmentResponse rsp = client().admin().indices().prepareSegments("test").get(); - List segments = rsp.getIndices().get("test").iterator().next().getShards()[0].getSegments(); - assertNull(segments.get(0).toString(), segments.get(0).ramTree); - } - - public void testVerbose() { - IndicesSegmentResponse rsp = client().admin().indices().prepareSegments("test").setVerbose(true).get(); - List segments = rsp.getIndices().get("test").iterator().next().getShards()[0].getSegments(); - assertNull(segments.get(0).toString(), segments.get(0).ramTree); - } - /** * with the default IndicesOptions inherited from BroadcastOperationRequest this will raise an exception */ diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 1790cdde2c545..facd673a35752 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -284,7 +284,6 @@ public void testVerboseSegments() throws Exception { segments = engine.segments(true); assertThat(segments.size(), equalTo(1)); - assertThat(segments.get(0).ramTree, nullValue()); ParsedDocument doc2 = testParsedDocument("2", null, testDocumentWithTextField(), B_2, null); engine.index(indexForDoc(doc2)); @@ -295,9 +294,6 @@ public void testVerboseSegments() throws Exception { segments = engine.segments(true); assertThat(segments.size(), equalTo(3)); - assertThat(segments.get(0).ramTree, nullValue()); - assertThat(segments.get(1).ramTree, nullValue()); - assertThat(segments.get(2).ramTree, nullValue()); } } From 251cb8b66d59bd2a5c09ffc492cb73ae89eb8324 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Mon, 20 Sep 2021 16:01:16 -0400 Subject: [PATCH 61/67] delay expose analyzers --- .../analysis/common/CommonAnalysisPlugin.java | 4 --- .../NorwegianNormalizationFilterFactory.java | 32 ------------------- .../SwedishMinimalStemFilterFactory.java | 31 ------------------ .../TeluguNormalizationFilterFactory.java | 28 ---------------- .../common/TeluguStemFilterFactory.java | 28 ---------------- .../common/CommonAnalysisFactoryTests.java | 4 +-- 6 files changed, 2 insertions(+), 125 deletions(-) delete mode 100644 modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NorwegianNormalizationFilterFactory.java delete mode 100644 modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SwedishMinimalStemFilterFactory.java delete mode 100644 modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TeluguNormalizationFilterFactory.java delete mode 100644 modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TeluguStemFilterFactory.java diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java index 1cc1c300c468d..c1202a8ceed5b 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java @@ -289,7 +289,6 @@ public TokenStream create(TokenStream tokenStream) { }; }); - filters.put("norwegiannormalization", NorwegianNormalizationFilterFactory::new); filters.put("pattern_capture", requiresAnalysisSettings(PatternCaptureGroupTokenFilterFactory::new)); filters.put("pattern_replace", requiresAnalysisSettings(PatternReplaceTokenFilterFactory::new)); filters.put("persian_normalization", PersianNormalizationFilterFactory::new); @@ -306,11 +305,8 @@ public TokenStream create(TokenStream tokenStream) { filters.put("sorani_normalization", SoraniNormalizationFilterFactory::new); filters.put("stemmer_override", requiresAnalysisSettings(StemmerOverrideTokenFilterFactory::new)); filters.put("stemmer", StemmerTokenFilterFactory::new); - filters.put("swedishminimalstem", SwedishMinimalStemFilterFactory::new); filters.put("synonym", requiresAnalysisSettings(SynonymTokenFilterFactory::new)); filters.put("synonym_graph", requiresAnalysisSettings(SynonymGraphTokenFilterFactory::new)); - filters.put("telugunormalization", TeluguNormalizationFilterFactory::new); - filters.put("telugustem", TeluguStemFilterFactory::new); filters.put("trim", TrimTokenFilterFactory::new); filters.put("truncate", requiresAnalysisSettings(TruncateTokenFilterFactory::new)); filters.put("unique", UniqueTokenFilterFactory::new); diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NorwegianNormalizationFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NorwegianNormalizationFilterFactory.java deleted file mode 100644 index e25f6c73fbdfb..0000000000000 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NorwegianNormalizationFilterFactory.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.analysis.common; - -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.no.NorwegianNormalizationFilter; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; -import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory; - -/** - * Factory for {@link NorwegianNormalizationFilter} - */ -public class NorwegianNormalizationFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory { - - public NorwegianNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(indexSettings, name, settings); - } - - @Override - public TokenStream create(TokenStream tokenStream) { - return new NorwegianNormalizationFilter(tokenStream); - } -} diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SwedishMinimalStemFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SwedishMinimalStemFilterFactory.java deleted file mode 100644 index 352c0cdd57799..0000000000000 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SwedishMinimalStemFilterFactory.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.analysis.common; - -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.sv.SwedishMinimalStemFilter; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; - -/** - * Factory for {@link SwedishMinimalStemFilter} - */ -public class SwedishMinimalStemFilterFactory extends AbstractTokenFilterFactory { - - public SwedishMinimalStemFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(indexSettings, name, settings); - } - - @Override - public TokenStream create(TokenStream tokenStream) { - return new SwedishMinimalStemFilter(tokenStream); - } -} diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TeluguNormalizationFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TeluguNormalizationFilterFactory.java deleted file mode 100644 index 30dbece051836..0000000000000 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TeluguNormalizationFilterFactory.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.analysis.common; - -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.te.TeluguNormalizationFilter; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; - -public class TeluguNormalizationFilterFactory extends AbstractTokenFilterFactory { - - public TeluguNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(indexSettings, name, settings); - } - - @Override - public TokenStream create(TokenStream tokenStream) { - return new TeluguNormalizationFilter(tokenStream); - } -} diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TeluguStemFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TeluguStemFilterFactory.java deleted file mode 100644 index 4468d2f381c46..0000000000000 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TeluguStemFilterFactory.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.analysis.common; - -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.te.TeluguStemFilter; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; - -public class TeluguStemFilterFactory extends AbstractTokenFilterFactory { - - public TeluguStemFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(indexSettings, name, settings); - } - - @Override - public TokenStream create(TokenStream tokenStream) { - return new TeluguStemFilter(tokenStream); - } -} diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java index 5f047617ac6e1..de8c4d1d0f927 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java @@ -86,7 +86,7 @@ protected Map> getTokenFilters() { filters.put("latvianstem", StemmerTokenFilterFactory.class); filters.put("norwegianlightstem", StemmerTokenFilterFactory.class); filters.put("norwegianminimalstem", StemmerTokenFilterFactory.class); - filters.put("norwegiannormalization", NorwegianNormalizationFilterFactory.class); + filters.put("norwegiannormalization", Void.class); filters.put("portuguesestem", StemmerTokenFilterFactory.class); filters.put("portugueselightstem", StemmerTokenFilterFactory.class); filters.put("portugueseminimalstem", StemmerTokenFilterFactory.class); @@ -94,7 +94,7 @@ protected Map> getTokenFilters() { filters.put("soranistem", StemmerTokenFilterFactory.class); filters.put("spanishlightstem", StemmerTokenFilterFactory.class); filters.put("swedishlightstem", StemmerTokenFilterFactory.class); - filters.put("swedishminimalstem", SwedishMinimalStemFilterFactory.class); + filters.put("swedishminimalstem", Void.class); filters.put("stemmeroverride", StemmerOverrideTokenFilterFactory.class); filters.put("telugunormalization", TeluguNormalizationFilterFactory.class); filters.put("telugustem", TeluguStemFilterFactory.class); From e5609d09ae9f74eb2f11427276ebdb49303bea12 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Mon, 20 Sep 2021 16:01:21 -0400 Subject: [PATCH 62/67] stylecheck --- .../src/main/java/org/elasticsearch/index/engine/Segment.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/engine/Segment.java b/server/src/main/java/org/elasticsearch/index/engine/Segment.java index cc4fb43dc5de8..d1cac264611fc 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Segment.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Segment.java @@ -15,7 +15,6 @@ import org.apache.lucene.search.SortedSetSelector; import org.apache.lucene.search.SortedSetSortField; import org.apache.lucene.util.Accountable; -import org.apache.lucene.util.Accountables; import org.elasticsearch.core.Nullable; import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; @@ -25,9 +24,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import java.io.IOException; -import java.util.ArrayList; import java.util.Collection; -import java.util.List; import java.util.Map; import java.util.Objects; From 4fcb718fee86bae1bab41c2886d099d33de9bbc6 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Mon, 20 Sep 2021 16:20:44 -0400 Subject: [PATCH 63/67] remove japanese_completion --- .../JapaneseCompletionFilterFactory.java | 27 ------------------- .../kuromoji/AnalysisKuromojiPlugin.java | 2 -- .../AnalysisKuromojiFactoryTests.java | 2 -- 3 files changed, 31 deletions(-) delete mode 100644 plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/JapaneseCompletionFilterFactory.java diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/JapaneseCompletionFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/JapaneseCompletionFilterFactory.java deleted file mode 100644 index 8379bd61a2b6c..0000000000000 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/JapaneseCompletionFilterFactory.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.index.analysis; - -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.ja.JapaneseCompletionFilter; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.index.IndexSettings; - -public class JapaneseCompletionFilterFactory extends AbstractTokenFilterFactory { - - public JapaneseCompletionFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(indexSettings, name, settings); - } - - @Override - public TokenStream create(TokenStream tokenStream) { - return new JapaneseCompletionFilter(tokenStream); - } -} diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java index 9437f8c7e2f60..2c2592e91e5e7 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java @@ -11,7 +11,6 @@ import org.apache.lucene.analysis.Analyzer; import org.elasticsearch.index.analysis.AnalyzerProvider; import org.elasticsearch.index.analysis.CharFilterFactory; -import org.elasticsearch.index.analysis.JapaneseCompletionFilterFactory; import org.elasticsearch.index.analysis.JapaneseStopTokenFilterFactory; import org.elasticsearch.index.analysis.KuromojiAnalyzerProvider; import org.elasticsearch.index.analysis.KuromojiBaseFormFilterFactory; @@ -47,7 +46,6 @@ public Map> getTokenFilters() { extra.put("kuromoji_stemmer", KuromojiKatakanaStemmerFactory::new); extra.put("ja_stop", JapaneseStopTokenFilterFactory::new); extra.put("kuromoji_number", KuromojiNumberFilterFactory::new); - extra.put("japanese_completion", JapaneseCompletionFilterFactory::new); return extra; } diff --git a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/AnalysisKuromojiFactoryTests.java b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/AnalysisKuromojiFactoryTests.java index 56850e3b068c0..ec51c91dc1319 100644 --- a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/AnalysisKuromojiFactoryTests.java +++ b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/AnalysisKuromojiFactoryTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.index.analysis; -import org.apache.lucene.analysis.ja.JapaneseCompletionFilterFactory; import org.apache.lucene.analysis.ja.JapaneseTokenizerFactory; import org.elasticsearch.indices.analysis.AnalysisFactoryTestCase; import org.elasticsearch.plugin.analysis.kuromoji.AnalysisKuromojiPlugin; @@ -32,7 +31,6 @@ protected Map> getTokenizers() { protected Map> getTokenFilters() { Map> filters = new HashMap<>(super.getTokenFilters()); filters.put("japanesebaseform", KuromojiBaseFormFilterFactory.class); - filters.put("japanesecompletion", JapaneseCompletionFilterFactory.class); filters.put("japanesepartofspeechstop", KuromojiPartOfSpeechFilterFactory.class); filters.put("japanesereadingform", KuromojiReadingFormFilterFactory.class); filters.put("japanesekatakanastem", KuromojiKatakanaStemmerFactory.class); From 9d7e27f3eabbd5b6c47d1b1b13271ba2db655db5 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Mon, 20 Sep 2021 16:26:37 -0400 Subject: [PATCH 64/67] sort optimization --- .../lucene/search/grouping/CollapsingTopDocsCollector.java | 4 ---- .../org/elasticsearch/index/mapper/NumberFieldTypeTests.java | 3 +-- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/server/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java b/server/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java index 7f4b9059b5b0e..ff83f1ba8ec02 100644 --- a/server/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java +++ b/server/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java @@ -59,10 +59,6 @@ public final class CollapsingTopDocsCollector extends FirstPassGroupingCollec ((FieldComparator) comparator).setTopValue(after.fields[0]); } this.reversed = sortField.getReverse() ? -1 : 1; - - if (sortField.getOptimizeSortWithPoints() == false) { - comparator.disableSkipping(); - } } /** diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index 4024f9f80144e..fa8aa3f0e5fae 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -419,11 +419,10 @@ public void doTestDocValueRangeQueries(NumberType type, Supplier valueSu dir.close(); } - /* + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/74057") public void testIndexSortIntRange() throws Exception { doTestIndexSortRangeQueries(NumberType.INTEGER, random()::nextInt); } - */ public void testIndexSortLongRange() throws Exception { doTestIndexSortRangeQueries(NumberType.LONG, random()::nextLong); From b3b57dee200272bd0062442899bcbf6da1fd558e Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Mon, 20 Sep 2021 17:10:43 -0400 Subject: [PATCH 65/67] precommit --- plugins/analysis-kuromoji/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/analysis-kuromoji/build.gradle b/plugins/analysis-kuromoji/build.gradle index 2fd8515f4ca6a..214cf0592070c 100644 --- a/plugins/analysis-kuromoji/build.gradle +++ b/plugins/analysis-kuromoji/build.gradle @@ -29,7 +29,6 @@ tasks.named("dependencyLicenses").configure { tasks.named('splitPackagesAudit').configure { // o.e.i.analysis is owned by server, these should be moved to a kuromoji package ignoreClasses 'org.elasticsearch.index.analysis.JapaneseStopTokenFilterFactory', - 'org.elasticsearch.index.analysis.JapaneseCompletionFilterFactory', 'org.elasticsearch.index.analysis.KuromojiAnalyzerProvider', 'org.elasticsearch.index.analysis.KuromojiBaseFormFilterFactory', 'org.elasticsearch.index.analysis.KuromojiIterationMarkCharFilterFactory', From 16b26da73d09b24fb94597bbd361a7dcb757fea2 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Mon, 20 Sep 2021 18:06:02 -0400 Subject: [PATCH 66/67] fix unknown analyzer --- .../elasticsearch/indices/analysis/AnalysisFactoryTestCase.java | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java index f18b02ee1fc8f..168bf10a8591a 100644 --- a/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java @@ -190,6 +190,7 @@ public abstract class AnalysisFactoryTestCase extends ESTestCase { entry("delimitedboost", Void.class), // LUCENE-9574 entry("dropifflagged", Void.class), + entry("japanesecompletion", Void.class), // LUCENE-9575 entry("patterntyping", Void.class)); From b9d9c268bf0ee2f2dc4ac62042b2f1e844e8d7dc Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 21 Sep 2021 09:07:09 +0100 Subject: [PATCH 67/67] deef --- .../org/elasticsearch/index/mapper/DocumentLeafReader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java index 2ee2daa7cb3eb..a8cd07540ff44 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java @@ -191,12 +191,12 @@ public NumericDocValues getNormValues(String field) throws IOException { @Override public VectorValues getVectorValues(String field) throws IOException { - return null; + throw new UnsupportedOperationException(); } @Override public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs) throws IOException { - return null; + throw new UnsupportedOperationException(); } @Override