diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 146a081ac5355..326a92746d040 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,5 +1,5 @@ elasticsearch = 8.0.0 -lucene = 8.10.0-snapshot-bf2fcb53079 +lucene = 9.0.0-snapshot-32a0a16aff0 bundled_jdk_vendor = adoptium bundled_jdk = 16.0.2+7 @@ -11,7 +11,7 @@ spatial4j = 0.7 jts = 1.15.0 jackson = 2.10.4 snakeyaml = 1.26 -icu4j = 62.1 +icu4j = 68.2 supercsv = 2.4.0 # when updating log4j, please update also docs/java-api/index.asciidoc log4j = 2.11.1 diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java index 02533be7fdbcc..250aafa18cee6 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java @@ -8,7 +8,9 @@ package org.elasticsearch.common.settings; +import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.store.DataOutput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; @@ -345,13 +347,14 @@ private void possiblyAlterEncryptedBytes( byte[] encryptedBytes, int truncEncryptedDataLength ) throws Exception { - indexOutput.writeInt(4 + salt.length + 4 + iv.length + 4 + encryptedBytes.length); - indexOutput.writeInt(salt.length); - indexOutput.writeBytes(salt, salt.length); - indexOutput.writeInt(iv.length); - indexOutput.writeBytes(iv, iv.length); - indexOutput.writeInt(encryptedBytes.length - truncEncryptedDataLength); - indexOutput.writeBytes(encryptedBytes, encryptedBytes.length); + DataOutput out = EndiannessReverserUtil.wrapDataOutput(indexOutput); + out.writeInt(4 + salt.length + 4 + iv.length + 4 + encryptedBytes.length); + out.writeInt(salt.length); + out.writeBytes(salt, salt.length); + out.writeInt(iv.length); + out.writeBytes(iv, iv.length); + out.writeInt(encryptedBytes.length - truncEncryptedDataLength); + out.writeBytes(encryptedBytes, encryptedBytes.length); } public void testUpgradeAddsSeed() throws Exception { @@ -382,7 +385,7 @@ public void testBackcompatV1() throws Exception { Path configDir = env.configFile(); try ( Directory directory = newFSDirectory(configDir); - IndexOutput output = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT) + IndexOutput output = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT); ) { CodecUtil.writeHeader(output, "elasticsearch.keystore", 1); output.writeByte((byte) 0); // hasPassword = false @@ -417,9 +420,8 @@ public void testBackcompatV2() throws Exception { random().nextBytes(fileBytes); try ( Directory directory = newFSDirectory(configDir); - IndexOutput output = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT) + IndexOutput output = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT); ) { - CodecUtil.writeHeader(output, "elasticsearch.keystore", 2); output.writeByte((byte) 0); // hasPassword = false output.writeString("PKCS12"); diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc index 5ef5cac3ecf03..b95585ea13e43 100644 --- a/docs/Versions.asciidoc +++ b/docs/Versions.asciidoc @@ -1,8 +1,8 @@ include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[] -:lucene_version: 8.10.0 -:lucene_version_path: 8_10_0 +:lucene_version: 9.0.0 +:lucene_version_path: 9_0_0 :jdk: 11.0.2 :jdk_major: 11 :build_flavor: default diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicFilterFactory.java index 5ff0134f7807f..eb7658fed1274 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicFilterFactory.java @@ -8,7 +8,7 @@ package org.elasticsearch.analysis.common; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.standard.ClassicFilter; +import org.apache.lucene.analysis.classic.ClassicFilter; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicTokenizerFactory.java index c9b997f03cebf..580597bbc2442 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicTokenizerFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicTokenizerFactory.java @@ -9,7 +9,7 @@ package org.elasticsearch.analysis.common; import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.analysis.standard.ClassicTokenizer; +import org.apache.lucene.analysis.classic.ClassicTokenizer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java index 7edee0d149a67..c1202a8ceed5b 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java @@ -27,6 +27,8 @@ import org.apache.lucene.analysis.cjk.CJKWidthFilter; import org.apache.lucene.analysis.ckb.SoraniAnalyzer; import org.apache.lucene.analysis.ckb.SoraniNormalizationFilter; +import org.apache.lucene.analysis.classic.ClassicFilter; +import org.apache.lucene.analysis.classic.ClassicTokenizer; import org.apache.lucene.analysis.commongrams.CommonGramsFilter; import org.apache.lucene.analysis.core.DecimalDigitFilter; import org.apache.lucene.analysis.core.KeywordTokenizer; @@ -40,6 +42,7 @@ import org.apache.lucene.analysis.de.GermanNormalizationFilter; import org.apache.lucene.analysis.de.GermanStemFilter; import org.apache.lucene.analysis.el.GreekAnalyzer; +import org.apache.lucene.analysis.email.UAX29URLEmailTokenizer; import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.apache.lucene.analysis.en.KStemFilter; import org.apache.lucene.analysis.en.PorterStemFilter; @@ -89,10 +92,7 @@ import org.apache.lucene.analysis.ru.RussianAnalyzer; import org.apache.lucene.analysis.shingle.ShingleFilter; import org.apache.lucene.analysis.snowball.SnowballFilter; -import org.apache.lucene.analysis.standard.ClassicFilter; -import org.apache.lucene.analysis.standard.ClassicTokenizer; import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer; import org.apache.lucene.analysis.sv.SwedishAnalyzer; import org.apache.lucene.analysis.th.ThaiAnalyzer; import org.apache.lucene.analysis.th.ThaiTokenizer; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MinHashTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MinHashTokenFilterFactory.java index 37dc45d2a3017..76e1f6bab79b8 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MinHashTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MinHashTokenFilterFactory.java @@ -38,10 +38,18 @@ public TokenStream create(TokenStream tokenStream) { private Map convertSettings(Settings settings) { Map settingMap = new HashMap<>(); - settingMap.put("hashCount", settings.get("hash_count")); - settingMap.put("bucketCount", settings.get("bucket_count")); - settingMap.put("hashSetSize", settings.get("hash_set_size")); - settingMap.put("withRotation", settings.get("with_rotation")); + if (settings.hasValue("hash_count")) { + settingMap.put("hashCount", settings.get("hash_count")); + } + if (settings.hasValue("bucketCount")) { + settingMap.put("bucketCount", settings.get("bucket_count")); + } + if (settings.hasValue("hashSetSize")) { + settingMap.put("hashSetSize", settings.get("hash_set_size")); + } + if (settings.hasValue("with_rotation")) { + settingMap.put("withRotation", settings.get("with_rotation")); + } return settingMap; } } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UAX29URLEmailTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UAX29URLEmailTokenizerFactory.java index 3f5335da5906e..2391c36b90a29 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UAX29URLEmailTokenizerFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UAX29URLEmailTokenizerFactory.java @@ -9,8 +9,8 @@ package org.elasticsearch.analysis.common; import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.email.UAX29URLEmailTokenizer; import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java index 5e5bc79b2c145..de8c4d1d0f927 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java @@ -86,6 +86,7 @@ protected Map> getTokenFilters() { filters.put("latvianstem", StemmerTokenFilterFactory.class); filters.put("norwegianlightstem", StemmerTokenFilterFactory.class); filters.put("norwegianminimalstem", StemmerTokenFilterFactory.class); + filters.put("norwegiannormalization", Void.class); filters.put("portuguesestem", StemmerTokenFilterFactory.class); filters.put("portugueselightstem", StemmerTokenFilterFactory.class); filters.put("portugueseminimalstem", StemmerTokenFilterFactory.class); @@ -93,7 +94,10 @@ protected Map> getTokenFilters() { filters.put("soranistem", StemmerTokenFilterFactory.class); filters.put("spanishlightstem", StemmerTokenFilterFactory.class); filters.put("swedishlightstem", StemmerTokenFilterFactory.class); + filters.put("swedishminimalstem", Void.class); filters.put("stemmeroverride", StemmerOverrideTokenFilterFactory.class); + filters.put("telugunormalization", TeluguNormalizationFilterFactory.class); + filters.put("telugustem", TeluguStemFilterFactory.class); filters.put("kstem", KStemTokenFilterFactory.class); filters.put("synonym", SynonymTokenFilterFactory.class); filters.put("synonymgraph", SynonymGraphTokenFilterFactory.class); @@ -131,8 +135,6 @@ protected Map> getTokenFilters() { filters.put("brazilianstem", BrazilianStemTokenFilterFactory.class); filters.put("czechstem", CzechStemTokenFilterFactory.class); filters.put("germanstem", GermanStemTokenFilterFactory.class); - filters.put("telugunormalization", TeluguNormalizationFilterFactory.class); - filters.put("telugustem", TeluguStemFilterFactory.class); // this filter is not exposed and should only be used internally filters.put("fixedshingle", Void.class); return filters; @@ -216,7 +218,7 @@ protected Map> getPreConfiguredTokenizers() { tokenizers.put("keyword", null); tokenizers.put("lowercase", Void.class); tokenizers.put("classic", null); - tokenizers.put("uax_url_email", org.apache.lucene.analysis.standard.UAX29URLEmailTokenizerFactory.class); + tokenizers.put("uax_url_email", org.apache.lucene.analysis.email.UAX29URLEmailTokenizerFactory.class); tokenizers.put("path_hierarchy", null); tokenizers.put("letter", null); tokenizers.put("whitespace", null); diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java index 8efde7e6b1e03..776e53f4edee2 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java @@ -80,15 +80,17 @@ public void setup() { // that ignores position length attribute expectedQueryWithUnigram= new BooleanQuery.Builder() .add( - new SynonymQuery( - new Term("text_shingle_unigram", "foo"), - new Term("text_shingle_unigram", "foo bar") - ), BooleanClause.Occur.SHOULD) + new SynonymQuery.Builder("text_shingle_unigram") + .addTerm(new Term("text_shingle_unigram", "foo")) + .addTerm(new Term("text_shingle_unigram", "foo bar")) + .build(), + BooleanClause.Occur.SHOULD) .add( - new SynonymQuery( - new Term("text_shingle_unigram", "bar"), - new Term("text_shingle_unigram", "bar baz") - ), BooleanClause.Occur.SHOULD) + new SynonymQuery.Builder("text_shingle_unigram") + .addTerm(new Term("text_shingle_unigram", "bar")) + .addTerm(new Term("text_shingle_unigram", "bar baz")) + .build(), + BooleanClause.Occur.SHOULD) .add( new TermQuery( new Term("text_shingle_unigram", "baz") diff --git a/modules/lang-expression/licenses/lucene-expressions-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 0e051c6381995..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -97f27306f1817475b30e52f48b1407bc5d696d59 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..03931cd3c7e1e --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +561301335ba14e07ec89adff0e2321cb79146a48 \ No newline at end of file diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java index b708685487f9a..f352a32f2d27c 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java @@ -13,7 +13,6 @@ import org.apache.lucene.expressions.js.JavascriptCompiler; import org.apache.lucene.expressions.js.VariableContext; import org.apache.lucene.search.DoubleValuesSource; -import org.apache.lucene.search.SortField; import org.elasticsearch.SpecialPermission; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -232,7 +231,7 @@ private static NumberSortScript.LeafFactory newSortScript(Expression expr, Searc for (String variable : expr.variables) { try { if (variable.equals("_score")) { - bindings.add(new SortField("_score", SortField.Type.SCORE)); + bindings.add("_score", DoubleValuesSource.SCORES); needsScores = true; } else if (vars != null && vars.containsKey(variable)) { bindFromParams(vars, bindings, variable); @@ -283,7 +282,7 @@ private static AggregationScript.LeafFactory newAggregationScript(Expression exp for (String variable : expr.variables) { try { if (variable.equals("_score")) { - bindings.add(new SortField("_score", SortField.Type.SCORE)); + bindings.add("_score", DoubleValuesSource.SCORES); needsScores = true; } else if (variable.equals("_value")) { specialValue = new ReplaceableConstDoubleValueSource(); @@ -355,7 +354,7 @@ private static ScoreScript.LeafFactory newScoreScript(Expression expr, SearchLoo for (String variable : expr.variables) { try { if (variable.equals("_score")) { - bindings.add(new SortField("_score", SortField.Type.SCORE)); + bindings.add("_score", DoubleValuesSource.SCORES); needsScores = true; } else if (variable.equals("_value")) { specialValue = new ReplaceableConstDoubleValueSource(); diff --git a/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy b/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy index 763a06156828e..95b919b75bf85 100644 --- a/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy +++ b/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy @@ -18,4 +18,5 @@ grant { permission org.elasticsearch.script.ClassPermission "java.lang.Math"; permission org.elasticsearch.script.ClassPermission "org.apache.lucene.util.MathUtil"; permission org.elasticsearch.script.ClassPermission "org.apache.lucene.util.SloppyMath"; + permission org.elasticsearch.script.ClassPermission "org.apache.lucene.expressions.js.ExpressionMath"; }; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index 829585294a0ce..96f8c7740d10f 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -20,6 +20,10 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.spans.FieldMaskingSpanQuery; +import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.AutomatonQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; @@ -28,10 +32,6 @@ import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.spans.FieldMaskingSpanQuery; -import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.util.automaton.Automata; import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.Operations; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/SourceConfirmedTextQuery.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/SourceConfirmedTextQuery.java index baba8b0345f1f..180d5c7faf73a 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/SourceConfirmedTextQuery.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/SourceConfirmedTextQuery.java @@ -183,6 +183,11 @@ public int hashCode() { return 31 * Objects.hash(in, valueFetcherProvider, indexAnalyzer) + classHash(); } + @Override + public void visit(QueryVisitor visitor) { + in.visit(visitor.getSubVisitor(Occur.MUST, this)); + } + @Override public Query rewrite(IndexReader reader) throws IOException { Query inRewritten = in.rewrite(reader); @@ -244,11 +249,6 @@ public boolean isCacheable(LeafReaderContext ctx) { return false; } - @Override - public void extractTerms(Set termSet) { - termSet.addAll(terms); - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { RuntimePhraseScorer scorer = scorer(context); diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java index c48deb423c05a..289b659db7996 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java @@ -15,6 +15,9 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.spans.FieldMaskingSpanQuery; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; @@ -26,9 +29,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.spans.FieldMaskingSpanQuery; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.IndexSettings; diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/query/SourceConfirmedTextQueryTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/query/SourceConfirmedTextQueryTests.java index 249b02eed1614..c0d1b12f61f54 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/query/SourceConfirmedTextQueryTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/query/SourceConfirmedTextQueryTests.java @@ -16,6 +16,9 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.CheckHits; @@ -27,9 +30,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.store.Directory; import org.elasticsearch.common.CheckedIntFunction; import org.elasticsearch.common.lucene.Lucene; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java index f29fe0bfff2db..de870462ea135 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java @@ -13,6 +13,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.join.JoinUtil; import org.apache.lucene.search.join.ScoreMode; import org.apache.lucene.search.similarities.Similarity; @@ -411,6 +412,11 @@ public String toString(String s) { return "LateParsingQuery {joinField=" + joinField + "}"; } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + public int getMinChildren() { return minChildren; } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java index ecd70cf2d2664..95667ff1240d9 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java @@ -10,19 +10,19 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.ScorerSupplier; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; -import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Bits; import org.elasticsearch.core.CheckedFunction; @@ -32,7 +32,6 @@ import java.io.IOException; import java.util.List; import java.util.Objects; -import java.util.Set; final class PercolateQuery extends Query implements Accountable { @@ -75,10 +74,6 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo final Weight verifiedMatchesWeight = verifiedMatchesQuery.createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, boost); final Weight candidateMatchesWeight = candidateMatchesQuery.createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, boost); return new Weight(this) { - @Override - public void extractTerms(Set set) { - } - @Override public Explanation explain(LeafReaderContext leafReaderContext, int docId) throws IOException { Scorer scorer = scorer(leafReaderContext); @@ -239,6 +234,12 @@ public String toString(String s) { candidateMatchesQuery.toString(s) + "}}"; } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + + @Override public long ramBytesUsed() { long ramUsed = 0L; diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index 5140d3e165bdf..8142485570e89 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -19,9 +19,9 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.sandbox.search.CoveringQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.CoveringQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LongValuesSource; import org.apache.lucene.search.MatchNoDocsQuery; diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java index 687ff2082c6f9..98cf837b3b7d7 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java @@ -11,6 +11,8 @@ import org.apache.lucene.index.PrefixCodedTerms; import org.apache.lucene.index.Term; import org.apache.lucene.queries.BlendedTermQuery; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; @@ -24,8 +26,6 @@ import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.automaton.ByteRunAutomaton; diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java index 7f9c0ef0e62ec..aa84a9a46caba 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java @@ -13,7 +13,6 @@ import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; import org.apache.lucene.document.FloatPoint; -import org.apache.lucene.document.HalfFloatPoint; import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; @@ -35,10 +34,15 @@ import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.memory.MemoryIndex; import org.apache.lucene.queries.BlendedTermQuery; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanNotQuery; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; +import org.apache.lucene.sandbox.document.HalfFloatPoint; +import org.apache.lucene.sandbox.search.CoveringQuery; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.CoveringQuery; import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; @@ -49,6 +53,7 @@ import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Sort; @@ -58,10 +63,6 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.Weight; import org.apache.lucene.search.WildcardQuery; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanNotQuery; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; @@ -97,7 +98,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -1152,6 +1152,11 @@ public boolean equals(Object obj) { public int hashCode() { return classHash(); } + + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } } private static final class ControlQuery extends Query { @@ -1169,9 +1174,6 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo final IndexSearcher percolatorIndexSearcher = memoryIndex.createSearcher(); return new Weight(this) { - @Override - public void extractTerms(Set terms) {} - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { Scorer scorer = scorer(context); @@ -1260,6 +1262,11 @@ public int hashCode() { return classHash(); } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + } } diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java index 59f7f01a30ec1..a358e3d11e564 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java @@ -29,8 +29,8 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanTermQuery; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.store.Directory; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.test.ESTestCase; diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java index 3b2210676cc27..46a393bdda676 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java @@ -11,7 +11,6 @@ import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.FloatPoint; -import org.apache.lucene.document.HalfFloatPoint; import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; @@ -19,9 +18,10 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; import org.apache.lucene.index.memory.MemoryIndex; +import org.apache.lucene.sandbox.document.HalfFloatPoint; +import org.apache.lucene.sandbox.search.CoveringQuery; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.CoveringQuery; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermInSetQuery; diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java index 3160c8eaf9a83..acbd6d9df76bd 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java @@ -9,7 +9,6 @@ import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.FloatPoint; -import org.apache.lucene.document.HalfFloatPoint; import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LatLonPoint; @@ -20,6 +19,12 @@ import org.apache.lucene.queries.intervals.IntervalQuery; import org.apache.lucene.queries.intervals.Intervals; import org.apache.lucene.queries.intervals.IntervalsSource; +import org.apache.lucene.queries.spans.SpanFirstQuery; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanNotQuery; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; +import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; @@ -38,11 +43,6 @@ import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.search.join.QueryBitSetProducer; import org.apache.lucene.search.join.ScoreMode; -import org.apache.lucene.search.spans.SpanFirstQuery; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanNotQuery; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.lucene.search.function.CombineFunction; import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery; @@ -799,13 +799,16 @@ public void testExtractQueryMetadata_disjunctionMaxQuery() { } public void testSynonymQuery() { - SynonymQuery query = new SynonymQuery(); + SynonymQuery query = new SynonymQuery.Builder("field").build(); Result result = analyze(query); assertThat(result.verified, is(true)); assertThat(result.minimumShouldMatch, equalTo(0)); assertThat(result.extractions.isEmpty(), is(true)); - query = new SynonymQuery(new Term("_field", "_value1"), new Term("_field", "_value2")); + query = new SynonymQuery.Builder("_field") + .addTerm(new Term("_field", "_value1")) + .addTerm(new Term("_field", "_value2")) + .build(); result = analyze(query); assertThat(result.verified, is(true)); assertThat(result.minimumShouldMatch, equalTo(1)); diff --git a/plugins/analysis-icu/build.gradle b/plugins/analysis-icu/build.gradle index 148a08a002c33..7e07a9adc2fc0 100644 --- a/plugins/analysis-icu/build.gradle +++ b/plugins/analysis-icu/build.gradle @@ -1,5 +1,3 @@ -import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis - /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License @@ -23,7 +21,7 @@ tasks.named("forbiddenApisMain").configure { } dependencies { - api "org.apache.lucene:lucene-analyzers-icu:${versions.lucene}" + api "org.apache.lucene:lucene-analysis-icu:${versions.lucene}" api "com.ibm.icu:icu4j:${versions.icu4j}" } diff --git a/plugins/analysis-icu/licenses/icu4j-62.1.jar.sha1 b/plugins/analysis-icu/licenses/icu4j-62.1.jar.sha1 deleted file mode 100644 index c24c69cf4b90f..0000000000000 --- a/plugins/analysis-icu/licenses/icu4j-62.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7a4d00d5ec5febd252a6182e8b6e87a0a9821f81 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/icu4j-68.2.jar.sha1 b/plugins/analysis-icu/licenses/icu4j-68.2.jar.sha1 new file mode 100644 index 0000000000000..fcb3d79075099 --- /dev/null +++ b/plugins/analysis-icu/licenses/icu4j-68.2.jar.sha1 @@ -0,0 +1 @@ +76893e6000401ace133a65262254be0ebe556d46 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..4f21d907c34b8 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +5b64983cf184ddd4018d56114b086f3cebd734ad \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index dad0264f8c937..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -97560884369643ba928b7e683e092699330c4f5b \ No newline at end of file diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/ICUCollationKeyFilter.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/ICUCollationKeyFilter.java index 4c83862edba3b..ab89157b923b0 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/ICUCollationKeyFilter.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/ICUCollationKeyFilter.java @@ -22,7 +22,7 @@ import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.collation.ICUCollationDocValuesField; +import org.apache.lucene.analysis.icu.ICUCollationDocValuesField; import java.io.IOException; diff --git a/plugins/analysis-kuromoji/build.gradle b/plugins/analysis-kuromoji/build.gradle index 0b3beb8ab5dbe..214cf0592070c 100644 --- a/plugins/analysis-kuromoji/build.gradle +++ b/plugins/analysis-kuromoji/build.gradle @@ -14,7 +14,7 @@ esplugin { } dependencies { - api "org.apache.lucene:lucene-analyzers-kuromoji:${versions.lucene}" + api "org.apache.lucene:lucene-analysis-kuromoji:${versions.lucene}" } restResources { diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..dbe66f8da442d --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +3e27dd79ed9fa24bfb998446526baf4f16cc7a05 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index df913d91535b4..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3bc69fe29001399cfc98ddbd9f1d27864fa6a4a1 \ No newline at end of file diff --git a/plugins/analysis-nori/build.gradle b/plugins/analysis-nori/build.gradle index 099e620795148..74960bb331137 100644 --- a/plugins/analysis-nori/build.gradle +++ b/plugins/analysis-nori/build.gradle @@ -14,7 +14,7 @@ esplugin { } dependencies { - api "org.apache.lucene:lucene-analyzers-nori:${versions.lucene}" + api "org.apache.lucene:lucene-analysis-nori:${versions.lucene}" } restResources { diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..a3fc882d0c959 --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +48f2c1dec0aea566c3ac7c62322b52887a054bf6 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index ec341f6b95ed8..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -497cdb3e2b5dd63d735cd60140f7c18533f3c401 \ No newline at end of file diff --git a/plugins/analysis-phonetic/build.gradle b/plugins/analysis-phonetic/build.gradle index 89ff6f900c802..319f422d657ce 100644 --- a/plugins/analysis-phonetic/build.gradle +++ b/plugins/analysis-phonetic/build.gradle @@ -14,7 +14,7 @@ esplugin { } dependencies { - api "org.apache.lucene:lucene-analyzers-phonetic:${versions.lucene}" + api "org.apache.lucene:lucene-analysis-phonetic:${versions.lucene}" api "commons-codec:commons-codec:${versions.commonscodec}" } diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..81fd94a0b42bd --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +ee44e4909d2c2b8686606fd4d11b2b817cbd5b3f \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 90e9a1b66cfc0..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4fa14c42bf5864d28a7abfe197e79cf89b195c1b \ No newline at end of file diff --git a/plugins/analysis-smartcn/build.gradle b/plugins/analysis-smartcn/build.gradle index 3a3bd3012fd86..2c8de143db8a6 100644 --- a/plugins/analysis-smartcn/build.gradle +++ b/plugins/analysis-smartcn/build.gradle @@ -14,7 +14,7 @@ esplugin { } dependencies { - api "org.apache.lucene:lucene-analyzers-smartcn:${versions.lucene}" + api "org.apache.lucene:lucene-analysis-smartcn:${versions.lucene}" } restResources { diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..a1218843586a9 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +bf0d9a8984578032378342891c4d1e6d3147b780 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 4ebf1040e67b1..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -18971c579487a7c55a65b055fec07d9286ed49ce \ No newline at end of file diff --git a/plugins/analysis-stempel/build.gradle b/plugins/analysis-stempel/build.gradle index 3b1eb7623f947..363d923492e07 100644 --- a/plugins/analysis-stempel/build.gradle +++ b/plugins/analysis-stempel/build.gradle @@ -14,7 +14,7 @@ esplugin { } dependencies { - api "org.apache.lucene:lucene-analyzers-stempel:${versions.lucene}" + api "org.apache.lucene:lucene-analysis-stempel:${versions.lucene}" } restResources { diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..2f8752fc5480d --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +4a88bb19fa4035d9841f76de0572826ce6c43915 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 1476b72761e58..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6fcf11017c3bce6e33d5cda04e3ef49560baa2a2 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/build.gradle b/plugins/analysis-ukrainian/build.gradle index 2a059bd746c93..650ab20e59602 100644 --- a/plugins/analysis-ukrainian/build.gradle +++ b/plugins/analysis-ukrainian/build.gradle @@ -14,7 +14,7 @@ esplugin { } dependencies { - api "org.apache.lucene:lucene-analyzers-morfologik:${versions.lucene}" + api "org.apache.lucene:lucene-analysis-morfologik:${versions.lucene}" api "org.carrot2:morfologik-stemming:2.1.1" api "org.carrot2:morfologik-fsa:2.1.1" api "ua.net.nlp:morfologik-ukrainian-search:3.7.5" diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..437b6567b291b --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +88837c81d62b322d83669cf16f85b7bc66cfe3e8 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index f5ac5de21f228..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -af5750b21d1d38352b49561c63b039723911a0d4 \ No newline at end of file diff --git a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/smb/SmbNIOFSDirectoryTests.java b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/smb/SmbNIOFSDirectoryTests.java index 13e90747faea3..77613a8e0b515 100644 --- a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/smb/SmbNIOFSDirectoryTests.java +++ b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/smb/SmbNIOFSDirectoryTests.java @@ -8,12 +8,13 @@ package org.elasticsearch.index.store.smb; -import java.io.IOException; -import java.nio.file.Path; import org.apache.lucene.store.Directory; import org.apache.lucene.store.NIOFSDirectory; import org.elasticsearch.index.store.EsBaseDirectoryTestCase; +import java.io.IOException; +import java.nio.file.Path; + public class SmbNIOFSDirectoryTests extends EsBaseDirectoryTestCase { @Override diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index a208a103b0726..1ede5d2c87412 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -1,5 +1,4 @@ import org.elasticsearch.gradle.internal.info.BuildParams -import org.elasticsearch.gradle.OS apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.publish' @@ -85,6 +84,9 @@ tasks.named("yamlRestTestV7CompatTest").configure { 'indices.stats/20_translog/Translog retention without soft_deletes', 'indices.stats/20_translog/Translog stats on closed indices without soft-deletes', + // field usage results will be different between lucene versions + 'indices.stats/60_field_usage/*', + // upgrade api will only get a dummy endpoint returning an exception suggesting to use _reindex 'indices.upgrade/*/*', @@ -220,6 +222,8 @@ tasks.named("yamlRestTestV7CompatTransform").configure({ task -> "_all.primaries.indexing.types._doc.index_total" ) task.replaceValueInMatch("_all.primaries.indexing.types._doc.index_total", 2) + // points get touched by sorting in ES 8 + task.replaceValueInMatch("testindex.shards.0.stats.fields.price.points", 1) //override for "indices.open/10_basic/?wait_for_active_shards default is deprecated" and "indices.open/10_basic/?wait_for_active_shards=index-setting" task.addAllowedWarningRegexForTest("\\?wait_for_active_shards=index-setting is now the default behaviour.*", "?wait_for_active_shards=index-setting") diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/60_field_usage.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/60_field_usage.yml index e9c8fc2ad8ae8..47d9f73bd4023 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/60_field_usage.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/60_field_usage.yml @@ -1,9 +1,3 @@ ---- -setup: - - skip: - version: " - 7.14.99" - reason: field usage stats API is introduced in 7.15 - --- "Field usage stats": - do: @@ -115,7 +109,8 @@ setup: - match: { testindex.shards.0.stats.fields.price.inverted_index.proximity: 0 } - match: { testindex.shards.0.stats.fields.price.stored_fields: 0 } - gt: { testindex.shards.0.stats.fields.price.doc_values: 0 } - - match: { testindex.shards.0.stats.fields.price.points: 0 } + # can be 0 on pre-lucene 9 nodes or 1 on post-upgrade nodes + - gte: { testindex.shards.0.stats.fields.price.points: 0 } - match: { testindex.shards.0.stats.fields.price.norms: 0 } - match: { testindex.shards.0.stats.fields.price.term_vectors: 0 } - match: { testindex.shards.0.stats.fields.price.inverted_index.term_frequencies: 0 } diff --git a/server/build.gradle b/server/build.gradle index b4fb3f5a02ac7..d7d53e88db969 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -33,7 +33,7 @@ dependencies { // lucene api "org.apache.lucene:lucene-core:${versions.lucene}" - api "org.apache.lucene:lucene-analyzers-common:${versions.lucene}" + api "org.apache.lucene:lucene-analysis-common:${versions.lucene}" api "org.apache.lucene:lucene-backward-codecs:${versions.lucene}" api "org.apache.lucene:lucene-grouping:${versions.lucene}" api "org.apache.lucene:lucene-highlighter:${versions.lucene}" @@ -279,8 +279,6 @@ tasks.named('splitPackagesAudit').configure { 'org.apache.lucene.queryparser.classic.XQueryParser', 'org.apache.lucene.queries.BinaryDocValuesRangeQuery', 'org.apache.lucene.queries.BlendedTermQuery', - 'org.apache.lucene.queries.MinDocQuery', - 'org.apache.lucene.queries.SearchAfterSortedDocQuery', 'org.apache.lucene.queries.SpanMatchNoDocsQuery', 'org.apache.lucene.search.grouping.CollapseTopFieldDocs', 'org.apache.lucene.search.grouping.CollapsingDocValuesSource', diff --git a/server/licenses/lucene-analysis-common-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-analysis-common-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..afff795051ecd --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +bce23b7816187b44700ee646ce1afbdac4423dcc \ No newline at end of file diff --git a/server/licenses/lucene-analyzers-common-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-analyzers-common-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index fdede7b9c4cdc..0000000000000 --- a/server/licenses/lucene-analyzers-common-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -eb63f6ecd58a7e27a02b533b9c1e6cdb68f506fc \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-backward-codecs-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index d7b5d232b1d5b..0000000000000 --- a/server/licenses/lucene-backward-codecs-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2482a84e5e26a3eaf0bd7c5a77efc60435c7f688 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..be9eb98abf9e4 --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +63ef3d7a09cc122a77d4d84af9b5c03efa480d01 \ No newline at end of file diff --git a/server/licenses/lucene-core-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-core-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 2a546f6674e06..0000000000000 --- a/server/licenses/lucene-core-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f33b45dbbce59e727e5a82dddab48c8c7681e25b \ No newline at end of file diff --git a/server/licenses/lucene-core-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-core-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..9c473fb8da725 --- /dev/null +++ b/server/licenses/lucene-core-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +49c6fdf8fdca15e4bc97973056e883e34de82b7a \ No newline at end of file diff --git a/server/licenses/lucene-grouping-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-grouping-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index c39c757ef5945..0000000000000 --- a/server/licenses/lucene-grouping-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4020eb4e53d759fa11819b8d6b6a49422f51abe8 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-grouping-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..38aee36eb1ff5 --- /dev/null +++ b/server/licenses/lucene-grouping-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +0d2bf347c51f37e2f659bb8a015189e8743ec8e2 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-highlighter-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index eda61949a36fd..0000000000000 --- a/server/licenses/lucene-highlighter-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b2c9682be68699860aeeb9b048665ab6cf259c7c \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-highlighter-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..996fb57e4c01e --- /dev/null +++ b/server/licenses/lucene-highlighter-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +d0159455b0ca8930d9caf95a7275ba192f4bd9ab \ No newline at end of file diff --git a/server/licenses/lucene-join-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-join-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 60e4cb4bed007..0000000000000 --- a/server/licenses/lucene-join-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -be4f561159763de6a28d47b463438331a96c31c9 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-join-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..3d0703d61dd55 --- /dev/null +++ b/server/licenses/lucene-join-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +342f80a704dd37012253a6b19bfad88e08287791 \ No newline at end of file diff --git a/server/licenses/lucene-memory-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-memory-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 0cb9423697434..0000000000000 --- a/server/licenses/lucene-memory-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8fc0745c4a589cdfbd56d156236fd91dbab0dacb \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-memory-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..7525a6064b0a8 --- /dev/null +++ b/server/licenses/lucene-memory-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +5944a54b0d46ee4554f53c40c0b92ac7f1d70c1b \ No newline at end of file diff --git a/server/licenses/lucene-misc-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-misc-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 4860f9a1e54ea..0000000000000 --- a/server/licenses/lucene-misc-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7089f903268271acf6eb668918ac51f0cba15213 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-misc-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..47757720517e2 --- /dev/null +++ b/server/licenses/lucene-misc-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +1553866522ffdd4aa2123fa14b1a3ed2d1d64713 \ No newline at end of file diff --git a/server/licenses/lucene-queries-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-queries-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 0f6f657b0747b..0000000000000 --- a/server/licenses/lucene-queries-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -35a4945ac05c2aeb0c9e244098827fd7aeea1858 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-queries-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..aa3d74f718093 --- /dev/null +++ b/server/licenses/lucene-queries-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +0e695c67e7de44914d6b9a8f95d06c2948ef8b7e \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-queryparser-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 48ccf9f681ce5..0000000000000 --- a/server/licenses/lucene-queryparser-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3d0929b7a5a2ba7f83d0357553f240f6d8362446 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-queryparser-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..f24a672cafa29 --- /dev/null +++ b/server/licenses/lucene-queryparser-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +21687ed2a47b1b78eee80c2bd6de3f11eb63e473 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-sandbox-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 062d28281a90a..0000000000000 --- a/server/licenses/lucene-sandbox-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d1d6696a4857bb580f6fc4a93bd3307effddd736 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-sandbox-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..ef971205715f0 --- /dev/null +++ b/server/licenses/lucene-sandbox-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +52acae7ec3773b136019613a4dd897cb915da856 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-spatial-extras-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index f7c69cd9622e3..0000000000000 --- a/server/licenses/lucene-spatial-extras-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bc52ac3d5fed41fde8b7ad95c7d5ce703b90377f \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..dca5dc13ce1cb --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +26b1aebb92871f79b6f2c8dab8b3f47371ec30da \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-spatial3d-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index bacaf78b1aec2..0000000000000 --- a/server/licenses/lucene-spatial3d-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -57d8cc36815cf45eb16d43648c8d2a5b251b4e62 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-spatial3d-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..459ace5f18578 --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +ca9b60241861bfa17401bd28287f09af61fa1c5c \ No newline at end of file diff --git a/server/licenses/lucene-suggest-8.10.0-snapshot-bf2fcb53079.jar.sha1 b/server/licenses/lucene-suggest-8.10.0-snapshot-bf2fcb53079.jar.sha1 deleted file mode 100644 index 90ade912a8369..0000000000000 --- a/server/licenses/lucene-suggest-8.10.0-snapshot-bf2fcb53079.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -17e5a74d57ecb861e93c3cfbf4feb356a0571bbf \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.0.0-snapshot-32a0a16aff0.jar.sha1 b/server/licenses/lucene-suggest-9.0.0-snapshot-32a0a16aff0.jar.sha1 new file mode 100644 index 0000000000000..f1acea0dd5681 --- /dev/null +++ b/server/licenses/lucene-suggest-9.0.0-snapshot-32a0a16aff0.jar.sha1 @@ -0,0 +1 @@ +e2449a6766c2e1c78628ce1dfbd00b172eb45b37 \ No newline at end of file diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java index bd71dd47a1aef..d97500db8b722 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java @@ -312,8 +312,6 @@ public void testLimitOnExpandedFields() throws Exception { doAssertOneHitForQueryString("field_A0:foo"); // expanding to the limit should work doAssertOneHitForQueryString("field_A\\*:foo"); - // expanding two blocks to the limit still works - doAssertOneHitForQueryString("field_A\\*:foo field_B\\*:bar"); // adding a non-existing field on top shouldn't overshoot the limit doAssertOneHitForQueryString("field_A\\*:foo unmapped:something"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java index df9e8216b4086..85e7c9634a2bc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java @@ -111,8 +111,11 @@ public void testExplainValidateQueryTwoNodes() throws IOException { assertThat(response.isValid(), equalTo(true)); assertThat(response.getQueryExplanation().size(), equalTo(1)); assertThat(response.getQueryExplanation().get(0).getExplanation(), - equalTo("(MatchNoDocsQuery(\"failed [bar] query, caused by number_format_exception:[For input string: \"foo\"]\") " + - "| foo:foo | baz:foo)")); + containsString("MatchNoDocsQuery(\"failed [bar] query, caused by number_format_exception:[For input string: \"foo\"]\")")); + assertThat(response.getQueryExplanation().get(0).getExplanation(), + containsString("foo:foo")); + assertThat(response.getQueryExplanation().get(0).getExplanation(), + containsString("baz:foo")); assertThat(response.getQueryExplanation().get(0).getError(), nullValue()); } } diff --git a/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java b/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java index 194d13d4dc970..eebe045aaddc7 100644 --- a/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java +++ b/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java @@ -15,6 +15,7 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TwoPhaseIterator; @@ -108,6 +109,13 @@ public boolean isCacheable(LeafReaderContext ctx) { }; } + @Override + public void visit(QueryVisitor visitor) { + if (visitor.acceptField(fieldName)) { + visitor.visitLeaf(this); + } + } + @Override public String toString(String field) { return "BinaryDocValuesRangeQuery(fieldName=" + field + ",from=" + originalFrom + ",to=" + originalTo + ")"; diff --git a/server/src/main/java/org/apache/lucene/queries/SpanMatchNoDocsQuery.java b/server/src/main/java/org/apache/lucene/queries/SpanMatchNoDocsQuery.java index acae5073afa39..a609ec3d96479 100644 --- a/server/src/main/java/org/apache/lucene/queries/SpanMatchNoDocsQuery.java +++ b/server/src/main/java/org/apache/lucene/queries/SpanMatchNoDocsQuery.java @@ -10,16 +10,16 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermStates; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanWeight; +import org.apache.lucene.queries.spans.Spans; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanWeight; -import org.apache.lucene.search.spans.Spans; import java.io.IOException; import java.util.Collections; import java.util.Map; -import java.util.Set; /** * A {@link SpanQuery} that matches no documents. @@ -53,6 +53,11 @@ public int hashCode() { return classHash(); } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + @Override public SpanWeight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new SpanWeight(this, searcher, Collections.emptyMap(), boost) { @@ -64,9 +69,6 @@ public Spans getSpans(LeafReaderContext ctx, Postings requiredPostings) { return null; } - @Override - public void extractTerms(Set terms) {} - @Override public boolean isCacheable(LeafReaderContext ctx) { return true; diff --git a/server/src/main/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighter.java b/server/src/main/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighter.java index 7493eecac8dda..0837a7ef1394a 100644 --- a/server/src/main/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighter.java +++ b/server/src/main/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighter.java @@ -11,18 +11,18 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.CheckedSupplier; -import org.elasticsearch.core.Nullable; import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.search.ESToParentBlockJoinQuery; diff --git a/server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java b/server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java index e80158a97b83a..732de53e92932 100644 --- a/server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java +++ b/server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java @@ -11,15 +11,15 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.queries.BlendedTermQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; +import org.apache.lucene.sandbox.search.CombinedFieldQuery; import org.apache.lucene.search.BoostQuery; -import org.apache.lucene.search.CombinedFieldQuery; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.MultiPhraseQuery; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery; import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery; import org.elasticsearch.index.search.ESToParentBlockJoinQuery; diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 7579025b99dd7..5d0afb150a1ac 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -90,7 +90,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_7_14_2 = new Version(7140299, org.apache.lucene.util.Version.LUCENE_8_9_0); public static final Version V_7_15_0 = new Version(7150099, org.apache.lucene.util.Version.LUCENE_8_9_0); public static final Version V_7_16_0 = new Version(7160099, org.apache.lucene.util.Version.LUCENE_8_10_0); - public static final Version V_8_0_0 = new Version(8000099, org.apache.lucene.util.Version.LUCENE_8_10_0); + public static final Version V_8_0_0 = new Version(8000099, org.apache.lucene.util.Version.LUCENE_9_0_0); public static final Version CURRENT = V_8_0_0; private static final ImmutableOpenIntMap idToVersion; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java index 972459d97a831..c2a4f32c08324 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java @@ -9,14 +9,15 @@ package org.elasticsearch.action.admin.indices.diskusage; import org.apache.logging.log4j.Logger; +import org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat; +import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.codecs.FieldsProducer; import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PointsReader; import org.apache.lucene.codecs.StoredFieldsReader; import org.apache.lucene.codecs.TermVectorsReader; -import org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat; -import org.apache.lucene.codecs.lucene84.Lucene84PostingsFormat; +import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValuesType; @@ -42,7 +43,6 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.FutureArrays; import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.lucene.FilterIndexCommit; @@ -53,6 +53,8 @@ import org.elasticsearch.index.store.LuceneFilesExtensions; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -169,8 +171,8 @@ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { } @Override - public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { - trackField(fieldInfo, Integer.BYTES + value.length); + public void stringField(FieldInfo fieldInfo, String value) throws IOException { + trackField(fieldInfo, Integer.BYTES + value.getBytes(StandardCharsets.UTF_8).length); } @Override @@ -290,6 +292,10 @@ private void readProximity(Terms terms, PostingsEnum postings) throws IOExceptio private BlockTermState getBlockTermState(TermsEnum termsEnum, BytesRef term) throws IOException { if (term != null && termsEnum.seekExact(term)) { final TermState termState = termsEnum.termState(); + if (termState instanceof Lucene90PostingsFormat.IntBlockTermState) { + final Lucene90PostingsFormat.IntBlockTermState blockTermState = (Lucene90PostingsFormat.IntBlockTermState) termState; + return new BlockTermState(blockTermState.docStartFP, blockTermState.posStartFP, blockTermState.payStartFP); + } if (termState instanceof Lucene84PostingsFormat.IntBlockTermState) { final Lucene84PostingsFormat.IntBlockTermState blockTermState = (Lucene84PostingsFormat.IntBlockTermState) termState; return new BlockTermState(blockTermState.docStartFP, blockTermState.posStartFP, blockTermState.payStartFP); @@ -425,8 +431,8 @@ public void visit(int docID, byte[] packedValue) throws IOException { public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { for (int dim = 0; dim < numDims; dim++) { int offset = dim * bytesPerDim; - if (FutureArrays.compareUnsigned(minPackedValue, offset, offset + bytesPerDim, point, offset, offset + bytesPerDim) > 0 || - FutureArrays.compareUnsigned(maxPackedValue, offset, offset + bytesPerDim, point, offset, offset + bytesPerDim) < 0) { + if (Arrays.compareUnsigned(minPackedValue, offset, offset + bytesPerDim, point, offset, offset + bytesPerDim) > 0 || + Arrays.compareUnsigned(maxPackedValue, offset, offset + bytesPerDim, point, offset, offset + bytesPerDim) < 0) { return PointValues.Relation.CELL_OUTSIDE_QUERY; } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java index 17cfd38e93f7d..fffcc28aeb948 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java @@ -128,13 +128,6 @@ protected void addCustomXContentFields(XContentBuilder builder, Params params) t if (segment.getSegmentSort() != null) { toXContent(builder, segment.getSegmentSort()); } - if (segment.ramTree != null) { - builder.startArray(Fields.RAM_TREE); - for (Accountable child : segment.ramTree.getChildResources()) { - toXContent(builder, child); - } - builder.endArray(); - } if (segment.attributes != null && segment.attributes.isEmpty() == false) { builder.field("attributes", segment.attributes); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index 909549dc1de17..b6ad9f2ee57b3 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -184,7 +184,6 @@ static TopDocs mergeTopDocs(Collection results, int topN, int from) { if (results.isEmpty()) { return null; } - final boolean setShardIndex = false; final TopDocs topDocs = results.stream().findFirst().get(); final TopDocs mergedTopDocs; final int numShards = results.size(); @@ -194,15 +193,15 @@ static TopDocs mergeTopDocs(Collection results, int topN, int from) { CollapseTopFieldDocs firstTopDocs = (CollapseTopFieldDocs) topDocs; final Sort sort = new Sort(firstTopDocs.fields); final CollapseTopFieldDocs[] shardTopDocs = results.toArray(new CollapseTopFieldDocs[numShards]); - mergedTopDocs = CollapseTopFieldDocs.merge(sort, from, topN, shardTopDocs, setShardIndex); + mergedTopDocs = CollapseTopFieldDocs.merge(sort, from, topN, shardTopDocs, false); } else if (topDocs instanceof TopFieldDocs) { TopFieldDocs firstTopDocs = (TopFieldDocs) topDocs; final Sort sort = new Sort(firstTopDocs.fields); final TopFieldDocs[] shardTopDocs = results.toArray(new TopFieldDocs[numShards]); - mergedTopDocs = TopDocs.merge(sort, from, topN, shardTopDocs, setShardIndex); + mergedTopDocs = TopDocs.merge(sort, from, topN, shardTopDocs); } else { final TopDocs[] shardTopDocs = results.toArray(new TopDocs[numShards]); - mergedTopDocs = TopDocs.merge(from, topN, shardTopDocs, setShardIndex); + mergedTopDocs = TopDocs.merge(from, topN, shardTopDocs); } return mergedTopDocs; } diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java index 6d084b4fa89f9..3cdd25c6e56a9 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java @@ -8,8 +8,8 @@ package org.elasticsearch.action.search; -import org.apache.lucene.store.RAMOutputStream; import org.elasticsearch.common.io.stream.ByteArrayStreamInput; +import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; @@ -30,7 +30,8 @@ static InternalScrollSearchRequest internalScrollSearchRequest(ShardSearchContex } static String buildScrollId(AtomicArray searchPhaseResults) { - try (RAMOutputStream out = new RAMOutputStream()) { + try { + BytesStreamOutput out = new BytesStreamOutput(); out.writeString(INCLUDE_CONTEXT_UUID); out.writeString(searchPhaseResults.length() == 1 ? ParsedScrollId.QUERY_AND_FETCH_TYPE : ParsedScrollId.QUERY_THEN_FETCH_TYPE); out.writeVInt(searchPhaseResults.asList().size()); @@ -45,9 +46,7 @@ static String buildScrollId(AtomicArray searchPhase out.writeString(searchShardTarget.getNodeId()); } } - byte[] bytes = new byte[(int) out.getFilePointer()]; - out.writeTo(bytes, 0); - return Base64.getUrlEncoder().encodeToString(bytes); + return Base64.getUrlEncoder().encodeToString(out.copyBytes().array()); } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java b/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java index 6709134407336..661e78bf682ff 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java @@ -599,8 +599,8 @@ public static double arcDistance(double lat1, double lon1, double lat2, double l * 4 decimal degrees */ public static double planeDistance(double lat1, double lon1, double lat2, double lon2) { - double x = (lon2 - lon1) * SloppyMath.TO_RADIANS * Math.cos((lat2 + lat1) / 2.0 * SloppyMath.TO_RADIANS); - double y = (lat2 - lat1) * SloppyMath.TO_RADIANS; + double x = Math.toRadians(lon2 - lon1) * Math.cos(Math.toRadians((lat2 + lat1) / 2.0)); + double y = Math.toRadians(lat2 - lat1); return Math.sqrt(x * x + y * y) * EARTH_MEAN_RADIUS; } diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java index 206f9706ddcaf..43acc68198114 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -49,6 +49,7 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.Terms; +import org.apache.lucene.index.VectorValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.FieldDoc; @@ -74,17 +75,16 @@ import org.apache.lucene.store.Lock; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.Version; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.core.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader; import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore; import org.elasticsearch.common.util.iterable.Iterables; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -101,7 +101,7 @@ import java.util.Map; public class Lucene { - public static final String LATEST_CODEC = "Lucene87"; + public static final String LATEST_CODEC = "Lucene90"; public static final String SOFT_DELETES_FIELD = "__soft_deletes"; @@ -197,7 +197,7 @@ public static SegmentInfos pruneUnreferencedFiles(String segmentsFileName, Direc * since checksums don's match anymore. that's why we prune the name here directly. * We also want the caller to know if we were not able to remove a segments_N file. */ - if (file.startsWith(IndexFileNames.SEGMENTS) || file.equals(IndexFileNames.OLD_SEGMENTS_GEN)) { + if (file.startsWith(IndexFileNames.SEGMENTS)) { foundSegmentFiles++; if (file.equals(si.getSegmentsFileName()) == false) { directory.deleteFile(file); // remove all segment_N files except of the one we wanna keep @@ -235,7 +235,7 @@ public static IndexCommit getIndexCommit(SegmentInfos si, Directory directory) t public static void cleanLuceneIndex(Directory directory) throws IOException { try (Lock writeLock = directory.obtainLock(IndexWriter.WRITE_LOCK_NAME)) { for (final String file : directory.listAll()) { - if (file.startsWith(IndexFileNames.SEGMENTS) || file.equals(IndexFileNames.OLD_SEGMENTS_GEN)) { + if (file.startsWith(IndexFileNames.SEGMENTS)) { directory.deleteFile(file); // remove all segment_N files } } @@ -1024,6 +1024,16 @@ public NumericDocValues getNormValues(String field) { return null; } + @Override + public VectorValues getVectorValues(String field) throws IOException { + return null; + } + + @Override + public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs) throws IOException { + return null; + } + public FieldInfos getFieldInfos() { return new FieldInfos(new FieldInfo[0]); } @@ -1039,7 +1049,8 @@ public PointValues getPointValues(String fieldName) { public void checkIntegrity() { } - public Fields getTermVectors(int docID) { + @Override + public Fields getTermVectors(int docID) throws IOException { return null; } @@ -1074,7 +1085,7 @@ public CacheHelper getReaderCacheHelper() { /** * Prepares a new {@link IndexWriterConfig} that does not do any merges, by setting both the merge policy and the merge scheduler. * Setting just the merge policy means that constructing the index writer will create a {@link ConcurrentMergeScheduler} by default, - * which is quite heavyweight and in particular it can unnecessarily block on {@link IOUtils#spins}. + * which is quite heavyweight. */ @SuppressForbidden(reason = "NoMergePolicy#INSTANCE is safe to use since we also set NoMergeScheduler#INSTANCE") public static IndexWriterConfig indexWriterConfigWithNoMerging(Analyzer analyzer) { diff --git a/server/src/main/java/org/elasticsearch/common/lucene/MinimumScoreCollector.java b/server/src/main/java/org/elasticsearch/common/lucene/MinimumScoreCollector.java index 3dcc869d53d39..eeff832c36d70 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/MinimumScoreCollector.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/MinimumScoreCollector.java @@ -34,7 +34,7 @@ public MinimumScoreCollector(Collector collector, float minimumScore) { @Override public void setScorer(Scorable scorer) throws IOException { if ((scorer instanceof ScoreCachingWrappingScorer) == false) { - scorer = new ScoreCachingWrappingScorer(scorer); + scorer = ScoreCachingWrappingScorer.wrap(scorer); } this.scorer = scorer; leafCollector.setScorer(scorer); diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java b/server/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java index 1f0674f059e0c..14bdccc4d512c 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java @@ -19,6 +19,7 @@ import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.similarities.ClassicSimilarity; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.search.similarities.TFIDFSimilarity; @@ -148,6 +149,11 @@ public Query rewrite(IndexReader reader) throws IOException { return createQuery(mlt); } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + private Query createQuery(XMoreLikeThis mlt) throws IOException { BooleanQuery.Builder bqBuilder = new BooleanQuery.Builder(); if (this.likeFields != null) { diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java b/server/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java index 99d705eece950..1a2e693f1910d 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java @@ -19,6 +19,7 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiPhraseQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.StringHelper; @@ -303,4 +304,11 @@ private boolean termArraysEquals(List termArrays1, List termArra public String getField() { return field; } + + @Override + public void visit(QueryVisitor visitor) { + if (visitor.acceptField(field)) { + visitor.visitLeaf(this); // TODO implement term visiting + } + } } diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/SpanBooleanQueryRewriteWithMaxClause.java b/server/src/main/java/org/elasticsearch/common/lucene/search/SpanBooleanQueryRewriteWithMaxClause.java index 6ef9ecc74cd7e..06287611e1825 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/search/SpanBooleanQueryRewriteWithMaxClause.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/search/SpanBooleanQueryRewriteWithMaxClause.java @@ -15,13 +15,13 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.queries.SpanMatchNoDocsQuery; +import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java b/server/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java index 6f733c9f0a5f1..9365a96e09552 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java @@ -10,7 +10,6 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.FilterScorer; @@ -34,7 +33,6 @@ import java.util.List; import java.util.Locale; import java.util.Objects; -import java.util.Set; /** * A query that allows for a pluggable boost function / filter. If it matches @@ -248,11 +246,6 @@ class CustomBoostFactorWeight extends Weight { this.needsScores = needsScores; } - @Override - public void extractTerms(Set terms) { - subQueryWeight.extractTerms(terms); - } - private FunctionFactorScorer functionScorer(LeafReaderContext context) throws IOException { Scorer subQueryScorer = subQueryWeight.scorer(context); if (subQueryScorer == null) { diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreQuery.java b/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreQuery.java index 71dfe2690215b..3432c8b899512 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreQuery.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreQuery.java @@ -10,20 +10,19 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.FilterLeafCollector; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.Query; -import org.apache.lucene.search.Scorable; -import org.apache.lucene.search.Weight; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.Scorable; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.BulkScorer; +import org.apache.lucene.search.Weight; import org.apache.lucene.util.Bits; import org.elasticsearch.Version; import org.elasticsearch.script.DocValuesDocReader; @@ -34,7 +33,6 @@ import java.io.IOException; import java.util.Objects; -import java.util.Set; /** * A query that uses a script to compute documents' scores. @@ -93,11 +91,6 @@ public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { } } - @Override - public void extractTerms(Set terms) { - subQueryWeight.extractTerms(terms); - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { Scorer subQueryScorer = subQueryWeight.scorer(context); diff --git a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java index 83a959ce97733..49e3a458e386e 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java +++ b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java @@ -8,14 +8,13 @@ package org.elasticsearch.common.settings; +import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.store.BufferedChecksumIndexInput; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.util.SetOnce; @@ -25,15 +24,6 @@ import org.elasticsearch.common.Randomness; import org.elasticsearch.common.hash.MessageDigests; -import javax.crypto.AEADBadTagException; -import javax.crypto.Cipher; -import javax.crypto.CipherInputStream; -import javax.crypto.CipherOutputStream; -import javax.crypto.SecretKey; -import javax.crypto.SecretKeyFactory; -import javax.crypto.spec.GCMParameterSpec; -import javax.crypto.spec.PBEKeySpec; -import javax.crypto.spec.SecretKeySpec; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; @@ -63,6 +53,15 @@ import java.util.Map; import java.util.Set; import java.util.regex.Pattern; +import javax.crypto.AEADBadTagException; +import javax.crypto.Cipher; +import javax.crypto.CipherInputStream; +import javax.crypto.CipherOutputStream; +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.GCMParameterSpec; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.SecretKeySpec; /** * A disk based container for sensitive settings in Elasticsearch. @@ -235,8 +234,7 @@ public static KeyStoreWrapper load(Path configDir) throws IOException { } Directory directory = new NIOFSDirectory(configDir); - try (IndexInput indexInput = directory.openInput(KEYSTORE_FILENAME, IOContext.READONCE)) { - ChecksumIndexInput input = new BufferedChecksumIndexInput(indexInput); + try (ChecksumIndexInput input = EndiannessReverserUtil.openChecksumInput(directory, KEYSTORE_FILENAME, IOContext.READONCE)) { final int formatVersion; try { formatVersion = CodecUtil.checkHeader(input, KEYSTORE_FILENAME, MIN_FORMAT_VERSION, FORMAT_VERSION); @@ -515,7 +513,7 @@ public synchronized void save(Path configDir, char[] password, boolean preserveP // write to tmp file first, then overwrite String tmpFile = KEYSTORE_FILENAME + ".tmp"; Path keystoreTempFile = configDir.resolve(tmpFile); - try (IndexOutput output = directory.createOutput(tmpFile, IOContext.DEFAULT)) { + try (IndexOutput output = EndiannessReverserUtil.createOutput(directory, tmpFile, IOContext.DEFAULT)) { CodecUtil.writeHeader(output, KEYSTORE_FILENAME, FORMAT_VERSION); output.writeByte(password.length == 0 ? (byte)0 : (byte)1); diff --git a/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java b/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java index e4133a15b5b4d..62815c03f47df 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; @@ -91,7 +92,7 @@ private void writeStateToFirstLocation(final T state, Path stateLocation, Direct throws WriteStateException { try { deleteFileIfExists(stateLocation, stateDir, tmpFileName); - try (IndexOutput out = stateDir.createOutput(tmpFileName, IOContext.DEFAULT)) { + try (IndexOutput out = EndiannessReverserUtil.createOutput(stateDir, tmpFileName, IOContext.DEFAULT)) { CodecUtil.writeHeader(out, STATE_FILE_CODEC, STATE_FILE_VERSION); out.writeInt(FORMAT.index()); try (XContentBuilder builder = newXContentBuilder(FORMAT, new IndexOutputOutputStream(out) { @@ -267,7 +268,7 @@ protected XContentBuilder newXContentBuilder(XContentType type, OutputStream str */ public final T read(NamedXContentRegistry namedXContentRegistry, Path file) throws IOException { try (Directory dir = newDirectory(file.getParent())) { - try (IndexInput indexInput = dir.openInput(file.getFileName().toString(), IOContext.DEFAULT)) { + try (IndexInput indexInput = EndiannessReverserUtil.openInput(dir, file.getFileName().toString(), IOContext.DEFAULT)) { // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here. CodecUtil.checksumEntireFile(indexInput); CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, MIN_COMPATIBLE_STATE_FILE_VERSION, STATE_FILE_VERSION); diff --git a/server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java b/server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java index 605660aeade76..7e651ff64158f 100644 --- a/server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java +++ b/server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java @@ -73,15 +73,6 @@ public int getMaxMergeAtOnce() { return regularMergePolicy.getMaxMergeAtOnce(); } - public void setMaxMergeAtOnceExplicit(int maxMergeAtOnceExplicit) { - regularMergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit); - forcedMergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit); - } - - public int getMaxMergeAtOnceExplicit() { - return forcedMergePolicy.getMaxMergeAtOnceExplicit(); - } - // only setter that must NOT delegate to the forced merge policy public void setMaxMergedSegmentMB(double mbFrac) { regularMergePolicy.setMaxMergedSegmentMB(mbFrac); diff --git a/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java b/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java index d1717a41d8847..446b8dd6ef7a4 100644 --- a/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java +++ b/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java @@ -130,6 +130,7 @@ public final class MergePolicyConfig { public static final Setting INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING = Setting.intSetting("index.merge.policy.max_merge_at_once", DEFAULT_MAX_MERGE_AT_ONCE, 2, Property.Dynamic, Property.IndexScope); + // TODO deprecate this as it no longer has any effect public static final Setting INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING = Setting.intSetting("index.merge.policy.max_merge_at_once_explicit", DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT, 2, Property.Dynamic, Property.IndexScope); @@ -150,6 +151,7 @@ public final class MergePolicyConfig { double forceMergeDeletesPctAllowed = indexSettings.getValue(INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING); // percentage ByteSizeValue floorSegment = indexSettings.getValue(INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING); int maxMergeAtOnce = indexSettings.getValue(INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING); + // TODO this is removed from lucene 9 - do we need to emit a warning if it is set? int maxMergeAtOnceExplicit = indexSettings.getValue(INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING); // TODO is this really a good default number for max_merge_segment, what happens for large indices, // won't they end up with many segments? @@ -166,7 +168,6 @@ public final class MergePolicyConfig { mergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed); mergePolicy.setFloorSegmentMB(floorSegment.getMbFrac()); mergePolicy.setMaxMergeAtOnce(maxMergeAtOnce); - mergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit); mergePolicy.setMaxMergedSegmentMB(maxMergedSegment.getMbFrac()); mergePolicy.setSegmentsPerTier(segmentsPerTier); mergePolicy.setDeletesPctAllowed(deletesPctAllowed); @@ -188,7 +189,7 @@ void setMaxMergedSegment(ByteSizeValue maxMergedSegment) { } void setMaxMergesAtOnceExplicit(Integer maxMergeAtOnceExplicit) { - mergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit); + } void setMaxMergesAtOnce(Integer maxMergeAtOnce) { diff --git a/server/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java b/server/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java index 6dde1e9280102..d757461b1471c 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java @@ -37,7 +37,6 @@ public class PreBuiltAnalyzerProviderFactory extends PreConfiguredAnalysisCompon super(name, new PreBuiltAnalyzersDelegateCache(name, preBuiltAnalyzer)); this.create = preBuiltAnalyzer::getAnalyzer; Analyzer analyzer = preBuiltAnalyzer.getAnalyzer(Version.CURRENT); - analyzer.setVersion(Version.CURRENT.luceneVersion); current = new PreBuiltAnalyzerProvider(name, AnalyzerScope.INDICES, analyzer); } @@ -45,7 +44,6 @@ public PreBuiltAnalyzerProviderFactory(String name, PreBuiltCacheFactory.Caching super(name, cache); this.create = version -> create.get(); Analyzer analyzer = create.get(); - analyzer.setVersion(Version.CURRENT.luceneVersion); this.current = new PreBuiltAnalyzerProvider(name, AnalyzerScope.INDICES, analyzer); } @@ -66,7 +64,6 @@ public AnalyzerProvider get(IndexSettings indexSettings, protected AnalyzerProvider create(Version version) { assert Version.CURRENT.equals(version) == false; Analyzer analyzer = create.apply(version); - analyzer.setVersion(version.luceneVersion); return new PreBuiltAnalyzerProvider(getName(), AnalyzerScope.INDICES, analyzer); } diff --git a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java index d55f2d41e064f..1e623ccbb0456 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java +++ b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java @@ -9,8 +9,7 @@ package org.elasticsearch.index.codec; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene87.Lucene87Codec; -import org.apache.lucene.codecs.lucene87.Lucene87Codec.Mode; +import org.apache.lucene.codecs.lucene90.Lucene90Codec; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.mapper.MapperService; @@ -35,13 +34,13 @@ public class CodecService { public CodecService(@Nullable MapperService mapperService) { final var codecs = new HashMap(); if (mapperService == null) { - codecs.put(DEFAULT_CODEC, new Lucene87Codec()); - codecs.put(BEST_COMPRESSION_CODEC, new Lucene87Codec(Mode.BEST_COMPRESSION)); + codecs.put(DEFAULT_CODEC, new Lucene90Codec()); + codecs.put(BEST_COMPRESSION_CODEC, new Lucene90Codec(Lucene90Codec.Mode.BEST_COMPRESSION)); } else { codecs.put(DEFAULT_CODEC, - new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService)); + new PerFieldMappingPostingFormatCodec(Lucene90Codec.Mode.BEST_SPEED, mapperService)); codecs.put(BEST_COMPRESSION_CODEC, - new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService)); + new PerFieldMappingPostingFormatCodec(Lucene90Codec.Mode.BEST_COMPRESSION, mapperService)); } codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault()); for (String codec : Codec.availableCodecs()) { diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java index d9a6590bdc80e..240728b3eec2f 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java +++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java @@ -11,8 +11,8 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat; -import org.apache.lucene.codecs.lucene87.Lucene87Codec; +import org.apache.lucene.codecs.lucene90.Lucene90Codec; +import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.index.mapper.MapperService; @@ -24,11 +24,10 @@ * per index in real time via the mapping API. If no specific postings format is * configured for a specific field the default postings format is used. */ -public class PerFieldMappingPostingFormatCodec extends Lucene87Codec { - +public class PerFieldMappingPostingFormatCodec extends Lucene90Codec { private final MapperService mapperService; - // Always enable compression on binary doc values - private final DocValuesFormat docValuesFormat = new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION); + + private final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat(); static { assert Codec.forName(Lucene.LATEST_CODEC).getClass().isAssignableFrom(PerFieldMappingPostingFormatCodec.class) : diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index 0c5725b6fb691..e85153245a471 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -25,7 +25,6 @@ import org.apache.lucene.search.ReferenceManager; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.AlreadyClosedException; -import org.apache.lucene.util.Accountables; import org.apache.lucene.util.SetOnce; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.index.IndexRequest; @@ -895,9 +894,6 @@ private void fillSegmentInfo(SegmentReader segmentReader, boolean verbose, boole logger.trace(() -> new ParameterizedMessage("failed to get size for [{}]", info.info.name), e); } segment.segmentSort = info.info.getIndexSort(); - if (verbose) { - segment.ramTree = Accountables.namedAccountable("root", segmentReader); - } segment.attributes = info.info.getAttributes(); // TODO: add more fine grained mem stats values to per segment info here segments.put(info.info.name, segment); diff --git a/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java b/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java index 44294e111eab3..6c1939eabfa4b 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java +++ b/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java @@ -266,7 +266,6 @@ private TopDocs searchOperations(FieldDoc after, boolean accurateTotalHits) thro final Query rangeQuery = rangeQuery(Math.max(fromSeqNo, lastSeenSeqNo), toSeqNo); assert accurateTotalHits == false || after == null : "accurate total hits is required by the first batch only"; final SortField sortBySeqNo = new SortField(SeqNoFieldMapper.NAME, SortField.Type.LONG); - sortBySeqNo.setCanUsePoints(); final TopFieldCollector collector = TopFieldCollector.create(new Sort(sortBySeqNo), searchBatchSize, after, accurateTotalHits ? Integer.MAX_VALUE : 0); indexSearcher.search(rangeQuery, collector); diff --git a/server/src/main/java/org/elasticsearch/index/engine/PrunePostingsMergePolicy.java b/server/src/main/java/org/elasticsearch/index/engine/PrunePostingsMergePolicy.java index 52978412f8c74..fb578e91c3f3a 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/PrunePostingsMergePolicy.java +++ b/server/src/main/java/org/elasticsearch/index/engine/PrunePostingsMergePolicy.java @@ -127,11 +127,6 @@ public ImpactsEnum impacts(int flags) throws IOException { public int size() { return postingsReader.size(); } - - @Override - public long ramBytesUsed() { - return postingsReader.ramBytesUsed(); - } }; } diff --git a/server/src/main/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicy.java b/server/src/main/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicy.java index d4efb5eacb1ab..2c8738bf85a66 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicy.java +++ b/server/src/main/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicy.java @@ -22,7 +22,7 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; -import org.apache.lucene.search.ConjunctionDISI; +import org.apache.lucene.search.ConjunctionUtils; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -96,7 +96,7 @@ public NumericDocValues getNumeric(FieldInfo field) throws IOException { // we can't return null here lucenes DocIdMerger expects an instance intersection = DocIdSetIterator.empty(); } else { - intersection = ConjunctionDISI.intersectIterators(Arrays.asList(numeric, + intersection = ConjunctionUtils.intersectIterators(Arrays.asList(numeric, new BitSetIterator(recoverySourceToKeep, recoverySourceToKeep.length()))); } return new FilterNumericDocValues(numeric) { @@ -179,11 +179,6 @@ public void checkIntegrity() throws IOException { public void close() throws IOException { in.close(); } - - @Override - public long ramBytesUsed() { - return in.ramBytesUsed(); - } } private abstract static class FilterStoredFieldsReader extends StoredFieldsReader { @@ -194,11 +189,6 @@ private abstract static class FilterStoredFieldsReader extends StoredFieldsReade this.in = fieldsReader; } - @Override - public long ramBytesUsed() { - return in.ramBytesUsed(); - } - @Override public void close() throws IOException { in.close(); @@ -257,7 +247,5 @@ public StoredFieldsReader clone() { } } - } - } diff --git a/server/src/main/java/org/elasticsearch/index/engine/Segment.java b/server/src/main/java/org/elasticsearch/index/engine/Segment.java index d04bf41aa0971..d1cac264611fc 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Segment.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Segment.java @@ -15,7 +15,6 @@ import org.apache.lucene.search.SortedSetSelector; import org.apache.lucene.search.SortedSetSortField; import org.apache.lucene.util.Accountable; -import org.apache.lucene.util.Accountables; import org.elasticsearch.core.Nullable; import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; @@ -25,9 +24,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import java.io.IOException; -import java.util.ArrayList; import java.util.Collection; -import java.util.List; import java.util.Map; import java.util.Objects; @@ -44,7 +41,6 @@ public class Segment implements Writeable { public Boolean compound = null; public String mergeId; public Sort segmentSort; - public Accountable ramTree = null; public Map attributes; public Segment(StreamInput in) throws IOException { @@ -62,8 +58,7 @@ public Segment(StreamInput in) throws IOException { in.readLong(); // memoryInBytes } if (in.readBoolean()) { - // verbose mode - ramTree = readRamTree(in); + readRamTree(in); } segmentSort = readSegmentSort(in); if (in.readBoolean()) { @@ -170,11 +165,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(0); // memoryInBytes } - boolean verbose = ramTree != null; - out.writeBoolean(verbose); - if (verbose) { - writeRamTree(out, ramTree); - } + out.writeBoolean(false); writeSegmentSort(out, segmentSort); boolean hasAttributes = attributes != null; out.writeBoolean(hasAttributes); @@ -274,18 +265,13 @@ private void writeSegmentSort(StreamOutput out, Sort sort) throws IOException { } } - private Accountable readRamTree(StreamInput in) throws IOException { - final String name = in.readString(); - final long bytes = in.readVLong(); + private static void readRamTree(StreamInput in) throws IOException { + in.readString(); + in.readVLong(); int numChildren = in.readVInt(); - if (numChildren == 0) { - return Accountables.namedAccountable(name, bytes); - } - List children = new ArrayList<>(numChildren); - while (numChildren-- > 0) { - children.add(readRamTree(in)); + for (int i = 0; i < numChildren; i++) { + readRamTree(in); } - return Accountables.namedAccountable(name, children, bytes); } // the ram tree is written recursively since the depth is fairly low (5 or 6) diff --git a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java index d14af5a264d37..7d5fb293e60f1 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java +++ b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java @@ -32,7 +32,10 @@ import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.index.VectorValues; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; @@ -56,7 +59,6 @@ import org.elasticsearch.index.translog.Translog; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.Map; import java.util.Set; @@ -136,14 +138,14 @@ private static class TranslogLeafReader extends LeafReader { private static final FieldInfo FAKE_SOURCE_FIELD = new FieldInfo(SourceFieldMapper.NAME, 1, false, false, false, IndexOptions.NONE, - DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, false); + DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, 0, VectorSimilarityFunction.EUCLIDEAN, false); private static final FieldInfo FAKE_ROUTING_FIELD = new FieldInfo(RoutingFieldMapper.NAME, 2, false, false, false, IndexOptions.NONE, - DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, false); + DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, 0, VectorSimilarityFunction.EUCLIDEAN, false); private static final FieldInfo FAKE_ID_FIELD = new FieldInfo(IdFieldMapper.NAME, 3, false, false, false, IndexOptions.DOCS, - DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, false); - private static Set TRANSLOG_FIELD_NAMES = + DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, 0, VectorSimilarityFunction.EUCLIDEAN, false); + private static final Set TRANSLOG_FIELD_NAMES = Sets.newHashSet(SourceFieldMapper.NAME, RoutingFieldMapper.NAME, IdFieldMapper.NAME); @@ -273,6 +275,16 @@ public NumericDocValues getNormValues(String field) throws IOException { return getDelegate().getNormValues(field); } + @Override + public VectorValues getVectorValues(String field) throws IOException { + return getDelegate().getVectorValues(field); + } + + @Override + public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs) throws IOException { + return getDelegate().searchNearestVectors(field, target, k, acceptDocs); + } + @Override public FieldInfos getFieldInfos() { return getDelegate().getFieldInfos(); @@ -348,7 +360,7 @@ private void readStoredFieldsDirectly(StoredFieldVisitor visitor) throws IOExcep } } if (operation.routing() != null && visitor.needsField(FAKE_ROUTING_FIELD) == StoredFieldVisitor.Status.YES) { - visitor.stringField(FAKE_ROUTING_FIELD, operation.routing().getBytes(StandardCharsets.UTF_8)); + visitor.stringField(FAKE_ROUTING_FIELD, operation.routing()); } if (visitor.needsField(FAKE_ID_FIELD) == StoredFieldVisitor.Status.YES) { final byte[] id = new byte[uid.length]; diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java index 8dfa8a1e14502..b757a2daaf967 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java @@ -33,6 +33,7 @@ * Base class for numeric field data. */ public abstract class IndexNumericFieldData implements IndexFieldData { + /** * The type of number. */ @@ -103,6 +104,8 @@ public final SortField sortField( SortedNumericSelector.Type.MAX : SortedNumericSelector.Type.MIN; SortField sortField = new SortedNumericSortField(getFieldName(), getNumericType().sortFieldType, reverse, selectorType); sortField.setMissingValue(source.missingObject(missingValue, reverse)); + // TODO fix this! + sortField.setOptimizeSortWithPoints(false); return sortField; } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java index a90be903ea2a0..07350942d51cd 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java @@ -7,10 +7,6 @@ */ package org.elasticsearch.index.fielddata.plain; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.lucene.codecs.blocktree.FieldReader; -import org.apache.lucene.codecs.blocktree.Stats; import org.apache.lucene.index.FilteredTermsEnum; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; @@ -23,9 +19,9 @@ import org.apache.lucene.util.PagedBytes; import org.apache.lucene.util.packed.PackedInts; import org.apache.lucene.util.packed.PackedLongValues; -import org.elasticsearch.core.Nullable; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.IndexFieldDataCache; @@ -45,7 +41,6 @@ import java.io.IOException; public class PagedBytesIndexFieldData extends AbstractIndexOrdinalsFieldData { - private static final Logger logger = LogManager.getLogger(PagedBytesIndexFieldData.class); private final double minFrequency, maxFrequency; private final int minSegmentSize; @@ -188,32 +183,6 @@ public long bytesPerValue(BytesRef term) { return bytes; } - /** - * @return the estimate for loading the entire term set into field data, or 0 if unavailable - */ - public long estimateStringFieldData() { - try { - LeafReader reader = context.reader(); - Terms terms = reader.terms(getFieldName()); - - final Terms fieldTerms = reader.terms(getFieldName()); - - if (fieldTerms instanceof FieldReader) { - final Stats stats = ((FieldReader) fieldTerms).getStats(); - long totalTermBytes = stats.totalTermBytes; - if (logger.isTraceEnabled()) { - logger.trace("totalTermBytes: {}, terms.size(): {}, terms.getSumDocFreq(): {}", - totalTermBytes, terms.size(), terms.getSumDocFreq()); - } - long totalBytes = totalTermBytes + (2 * terms.size()) + (4 * terms.getSumDocFreq()); - return totalBytes; - } - } catch (Exception e) { - logger.warn("Unable to estimate memory overhead", e); - } - return 0; - } - /** * Determine whether the BlockTreeTermsReader.FieldReader can be used * for estimating the field data, adding the estimate to the circuit @@ -229,25 +198,7 @@ public TermsEnum beforeLoad(Terms terms) throws IOException { TermsEnum iterator = terms.iterator(); TermsEnum filteredIterator = filter(terms, iterator, reader); - final boolean filtered = iterator != filteredIterator; - iterator = filteredIterator; - - if (filtered) { - if (logger.isTraceEnabled()) { - logger.trace("Filter exists, can't circuit break normally, using RamAccountingTermsEnum"); - } - return new RamAccountingTermsEnum(iterator, breaker, this, this.fieldName); - } else { - estimatedBytes = this.estimateStringFieldData(); - // If we weren't able to estimate, wrap in the RamAccountingTermsEnum - if (estimatedBytes == 0) { - iterator = new RamAccountingTermsEnum(iterator, breaker, this, this.fieldName); - } else { - breaker.addEstimateBytesAndMaybeBreak(estimatedBytes, fieldName); - } - - return iterator; - } + return new RamAccountingTermsEnum(filteredIterator, breaker, this, this.fieldName); } private TermsEnum filter(Terms terms, TermsEnum iterator, LeafReader reader) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java index a3c7e649ad100..b3d105816f7d3 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java @@ -8,13 +8,13 @@ package org.elasticsearch.index.fielddata.plain; -import org.apache.lucene.document.HalfFloatPoint; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.time.DateUtils; diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java index 1b3551bbddab3..1c55a8feb085b 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java @@ -19,7 +19,6 @@ import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.Uid; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -107,10 +106,9 @@ public void binaryField(FieldInfo fieldInfo, BytesRef value) { } @Override - public void stringField(FieldInfo fieldInfo, byte[] bytes) { + public void stringField(FieldInfo fieldInfo, String value) { assert IdFieldMapper.NAME.equals(fieldInfo.name) == false : "_id field must go through binaryField"; assert sourceFieldName.equals(fieldInfo.name) == false : "source field must go through binaryField"; - final String value = new String(bytes, StandardCharsets.UTF_8); addValue(fieldInfo.name, value); } diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java index 0b2bcc134a1e2..febb0decd97bb 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java @@ -9,12 +9,11 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Uid; -import org.apache.lucene.util.BytesRef; -import java.nio.charset.StandardCharsets; import java.util.List; /** @@ -60,8 +59,8 @@ public void binaryField(FieldInfo fieldInfo, byte[] value) { } @Override - public void stringField(FieldInfo fieldInfo, byte[] bytes) { - addValue(new String(bytes, StandardCharsets.UTF_8)); + public void stringField(FieldInfo fieldInfo, String value) { + addValue(value); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java index 35146ebca63e2..f57bbb3fc2c2a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java @@ -9,10 +9,10 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper; +import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; -import org.apache.lucene.search.spans.SpanQuery; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.time.DateMathParser; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index 1bb75e9d4eb79..0154a06703b98 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -301,7 +301,7 @@ public CompletionFieldType fieldType() { } static PostingsFormat postingsFormat() { - return PostingsFormat.forName("Completion84"); + return PostingsFormat.forName("Completion90"); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index bbf95d6df9e73..1a0a22e8824cb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -14,8 +14,8 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PointValues; +import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; -import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.Query; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java index 985a2b9e2cf47..a8cd07540ff44 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java @@ -25,13 +25,15 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.Terms; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.index.VectorValues; import org.apache.lucene.index.memory.MemoryIndex; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; @@ -161,7 +163,7 @@ public void document(int docID, StoredFieldVisitor visitor) throws IOException { visitor.doubleField(fieldInfo, v.doubleValue()); } } else if (field.stringValue() != null) { - visitor.stringField(fieldInfo, field.stringValue().getBytes(StandardCharsets.UTF_8)); + visitor.stringField(fieldInfo, field.stringValue()); } else if (field.binaryValue() != null) { // We can't just pass field.binaryValue().bytes here as there may be offset/length // considerations @@ -187,6 +189,16 @@ public NumericDocValues getNormValues(String field) throws IOException { throw new UnsupportedOperationException(); } + @Override + public VectorValues getVectorValues(String field) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs) throws IOException { + throw new UnsupportedOperationException(); + } + @Override public Bits getLiveDocs() { throw new UnsupportedOperationException(); @@ -248,6 +260,8 @@ private static FieldInfo fieldInfo(String name) { 0, 0, 0, + 0, + VectorSimilarityFunction.EUCLIDEAN, false ); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index e7f2349984121..5e13e3cc3f10a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -25,8 +25,8 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; -import org.apache.lucene.search.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper; +import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.core.Nullable; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 8489ce8a3c401..5648d796a6507 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -14,14 +14,14 @@ import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; import org.apache.lucene.document.FloatPoint; -import org.apache.lucene.document.HalfFloatPoint; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.sandbox.document.HalfFloatPoint; +import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; -import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 66b09ba8ffbde..61bca319741c7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -24,6 +24,12 @@ import org.apache.lucene.index.Term; import org.apache.lucene.queries.intervals.Intervals; import org.apache.lucene.queries.intervals.IntervalsSource; +import org.apache.lucene.queries.spans.FieldMaskingSpanQuery; +import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.AutomatonQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; @@ -36,12 +42,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.spans.FieldMaskingSpanQuery; -import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.Automata; import org.apache.lucene.util.automaton.Automaton; diff --git a/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java index bfb8955f24bd0..8a1ed211d03b9 100644 --- a/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java @@ -12,15 +12,14 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute; import org.apache.lucene.index.Term; +import org.apache.lucene.sandbox.search.CombinedFieldQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostAttribute; import org.apache.lucene.search.BoostQuery; -import org.apache.lucene.search.CombinedFieldQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.similarities.BM25Similarity; import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.search.similarity.LegacyBM25Similarity; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.QueryBuilder; import org.elasticsearch.common.xcontent.ParseField; @@ -36,6 +35,7 @@ import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.search.QueryParserHelper; +import org.elasticsearch.lucene.similarity.LegacyBM25Similarity; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java index 23b9b66f39b28..27b060f94744a 100644 --- a/server/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java @@ -8,9 +8,9 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.FieldMaskingSpanQuery; +import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.FieldMaskingSpanQuery; -import org.apache.lucene.search.spans.SpanQuery; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java index e5b949ab75f53..846e49170b601 100644 --- a/server/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java @@ -14,6 +14,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TwoPhaseIterator; @@ -170,6 +171,11 @@ public int hashCode() { return h; } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java index 23f4f43003b75..19855fed66777 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java @@ -8,13 +8,13 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanContainingQuery; +import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanContainingQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -24,7 +24,7 @@ import static org.elasticsearch.index.query.SpanQueryBuilder.SpanQueryBuilderUtil.checkNoBoost; /** - * Builder for {@link org.apache.lucene.search.spans.SpanContainingQuery}. + * Builder for {@link org.apache.lucene.queries.spans.SpanContainingQuery}. */ public class SpanContainingQueryBuilder extends AbstractQueryBuilder implements SpanQueryBuilder { public static final String NAME = "span_containing"; diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java index d5d6477241dba..506dd80b639a6 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java @@ -8,13 +8,13 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanFirstQuery; +import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanFirstQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java index 5f92133edd5e7..c2a65cf5e68ca 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java @@ -8,19 +8,19 @@ package org.elasticsearch.index.query; import org.apache.lucene.queries.SpanMatchNoDocsQuery; +import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TopTermsRewrite; -import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; -import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.search.SpanBooleanQueryRewriteWithMaxClause; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.MappedFieldType; diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java index 4b0abbe4b2b2f..2880d563dbc9b 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java @@ -8,14 +8,14 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java index a1af3b254e306..7b31272261f98 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java @@ -8,13 +8,13 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanNotQuery; +import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanNotQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java index 6f25623f6510e..78695bf650df1 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java @@ -8,9 +8,9 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanQuery; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java index 2606f7b29bfae..1eceb5b017e3b 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java @@ -9,13 +9,13 @@ package org.elasticsearch.index.query; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; -import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.lucene.BytesRefs; +import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.MappedFieldType; diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java index 93ca8ac1c52f1..6b8526705657f 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java @@ -8,13 +8,13 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanWithinQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanWithinQuery; -import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -24,7 +24,7 @@ import static org.elasticsearch.index.query.SpanQueryBuilder.SpanQueryBuilderUtil.checkNoBoost; /** - * Builder for {@link org.apache.lucene.search.spans.SpanWithinQuery}. + * Builder for {@link org.apache.lucene.queries.spans.SpanWithinQuery}. */ public class SpanWithinQueryBuilder extends AbstractQueryBuilder implements SpanQueryBuilder { public static final String NAME = "span_within"; diff --git a/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java index d7a745c408d84..6241fe614d97e 100644 --- a/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java @@ -10,8 +10,8 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.Term; +import org.apache.lucene.sandbox.search.CoveringQuery; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.CoveringQuery; import org.apache.lucene.search.DoubleValues; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LongValues; diff --git a/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java b/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java index abe470563271b..04322eedeb679 100644 --- a/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java +++ b/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java @@ -17,17 +17,17 @@ import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute; import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostAttribute; import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.util.QueryBuilder; import org.apache.lucene.util.graph.GraphTokenStreamFiniteStrings; import org.elasticsearch.ElasticsearchException; @@ -425,11 +425,6 @@ private SpanQuery newSpanQuery(Term[] terms, boolean isPrefix) { return new SpanOrQuery(spanQueries); } - @Override - protected SpanQuery createSpanQuery(TokenStream in, String field) throws IOException { - return createSpanQuery(in, field, false); - } - private SpanQuery createSpanQuery(TokenStream in, String field, boolean isPrefix) throws IOException { TermToBytesRefAttribute termAtt = in.getAttribute(TermToBytesRefAttribute.class); PositionIncrementAttribute posIncAtt = in.getAttribute(PositionIncrementAttribute.class); diff --git a/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java b/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java index 50f72f0c19337..e4d1d52bd9939 100644 --- a/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java +++ b/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java @@ -13,6 +13,9 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.queryparser.classic.Token; import org.apache.lucene.queryparser.classic.XQueryParser; @@ -28,9 +31,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.WildcardQuery; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.common.lucene.search.Queries; @@ -602,11 +602,12 @@ private Query getPossiblyAnalyzedPrefixQuery(String field, String termStr, Mappe } } else if (isLastPos == false) { // build a synonym query for terms in the same position. - Term[] terms = new Term[plist.size()]; - for (int i = 0; i < plist.size(); i++) { - terms[i] = new Term(field, plist.get(i)); + SynonymQuery.Builder sb = new SynonymQuery.Builder(field); + for (String synonym : plist) { + sb.addTerm(new Term(field, synonym)); + } - posQuery = new SynonymQuery(terms); + posQuery = sb.build(); } else { List innerClauses = new ArrayList<>(); for (String token : plist) { diff --git a/server/src/main/java/org/elasticsearch/index/search/SimpleQueryStringQueryParser.java b/server/src/main/java/org/elasticsearch/index/search/SimpleQueryStringQueryParser.java index da844a4e63a3c..4ecaf22938211 100644 --- a/server/src/main/java/org/elasticsearch/index/search/SimpleQueryStringQueryParser.java +++ b/server/src/main/java/org/elasticsearch/index/search/SimpleQueryStringQueryParser.java @@ -254,11 +254,12 @@ private Query newPossiblyAnalyzedQuery(String field, String termStr, Analyzer an } } else if (isLastPos == false) { // build a synonym query for terms in the same position. - Term[] terms = new Term[plist.size()]; - for (int i = 0; i < plist.size(); i++) { - terms[i] = new Term(field, plist.get(i)); + SynonymQuery.Builder sb = new SynonymQuery.Builder(field); + for (BytesRef bytesRef : plist) { + sb.addTerm(new Term(field, bytesRef)); + } - posQuery = new SynonymQuery(terms); + posQuery = sb.build(); } else { BooleanQuery.Builder innerBuilder = new BooleanQuery.Builder(); for (BytesRef token : plist) { diff --git a/server/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java b/server/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java index d59a755993e9e..cea9b5884f704 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java +++ b/server/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java @@ -19,6 +19,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TwoPhaseIterator; @@ -186,6 +187,11 @@ public int hashCode() { return classHash() ^ result; } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + private static void findSplitDocs(String idField, Predicate includeInShard, LeafReader leafReader, IntConsumer consumer) throws IOException { Terms terms = leafReader.terms(idField); @@ -228,13 +234,10 @@ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { } @Override - public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { - spare.bytes = value; - spare.offset = 0; - spare.length = value.length; + public void stringField(FieldInfo fieldInfo, String value) throws IOException { switch (fieldInfo.name) { case RoutingFieldMapper.NAME: - routing = spare.utf8ToString(); + routing = value; break; default: throw new IllegalStateException("Unexpected field: " + fieldInfo.name); diff --git a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java index 5c6079b684c21..74e5b28768519 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.misc.store.HardlinkCopyDirectoryWrapper; import org.apache.lucene.search.Sort; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FilterDirectory; @@ -142,7 +143,7 @@ void addIndices(final RecoveryState.Index indexRecoveryStats, final Directory ta assert sources.length > 0; final int luceneIndexCreatedVersionMajor = Lucene.readSegmentInfos(sources[0]).getIndexCreatedVersionMajor(); - final Directory hardLinkOrCopyTarget = new org.apache.lucene.store.HardlinkCopyDirectoryWrapper(target); + final Directory hardLinkOrCopyTarget = new HardlinkCopyDirectoryWrapper(target); IndexWriterConfig iwc = indexWriterConfigWithNoMerging(null) .setSoftDeletesField(Lucene.SOFT_DELETES_FIELD) diff --git a/server/src/main/java/org/elasticsearch/index/similarity/SimilarityProviders.java b/server/src/main/java/org/elasticsearch/index/similarity/SimilarityProviders.java index 352e04953f71b..df7de9ff6515c 100644 --- a/server/src/main/java/org/elasticsearch/index/similarity/SimilarityProviders.java +++ b/server/src/main/java/org/elasticsearch/index/similarity/SimilarityProviders.java @@ -37,11 +37,11 @@ import org.apache.lucene.search.similarities.NormalizationH2; import org.apache.lucene.search.similarities.NormalizationH3; import org.apache.lucene.search.similarities.NormalizationZ; -import org.apache.lucene.search.similarity.LegacyBM25Similarity; import org.elasticsearch.Version; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.lucene.similarity.LegacyBM25Similarity; import java.util.Arrays; import java.util.HashSet; @@ -238,9 +238,7 @@ public static LegacyBM25Similarity createBM25Similarity(Settings settings, Versi float b = settings.getAsFloat("b", 0.75f); boolean discountOverlaps = settings.getAsBoolean(DISCOUNT_OVERLAPS, true); - LegacyBM25Similarity similarity = new LegacyBM25Similarity(k1, b); - similarity.setDiscountOverlaps(discountOverlaps); - return similarity; + return new LegacyBM25Similarity(k1, b, discountOverlaps); } public static BooleanSimilarity createBooleanSimilarity(Settings settings, Version indexCreatedVersion) { diff --git a/server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java b/server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java index fd585ae0bbb85..8838e88408446 100644 --- a/server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java +++ b/server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java @@ -17,17 +17,17 @@ import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.search.similarities.Similarity.SimScorer; -import org.apache.lucene.search.similarity.LegacyBM25Similarity; import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; -import org.elasticsearch.core.Nullable; import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.lucene.similarity.LegacyBM25Similarity; import org.elasticsearch.script.ScriptService; import java.util.Collections; diff --git a/server/src/main/java/org/elasticsearch/index/store/Store.java b/server/src/main/java/org/elasticsearch/index/store/Store.java index 8c202fcbe88ce..cf62b50a17756 100644 --- a/server/src/main/java/org/elasticsearch/index/store/Store.java +++ b/server/src/main/java/org/elasticsearch/index/store/Store.java @@ -52,10 +52,10 @@ import org.elasticsearch.common.lucene.store.InputStreamIndexInput; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; -import org.elasticsearch.core.TimeValue; +import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.RefCounted; -import org.elasticsearch.common.util.iterable.Iterables; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.ShardLock; @@ -75,6 +75,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; +import java.io.UncheckedIOException; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.util.ArrayList; @@ -623,7 +624,6 @@ public void cleanupAndVerify(String reason, MetadataSnapshot sourceMetadata) thr // FNF should not happen since we hold a write lock? } catch (IOException ex) { if (existingFile.startsWith(IndexFileNames.SEGMENTS) - || existingFile.equals(IndexFileNames.OLD_SEGMENTS_GEN) || existingFile.startsWith(CORRUPTED_MARKER_NAME_PREFIX)) { // TODO do we need to also fail this if we can't delete the pending commit file? // if one of those files can't be deleted we better fail the cleanup otherwise we might leave an old commit @@ -968,9 +968,6 @@ public RecoveryDiff recoveryDiff(MetadataSnapshot recoveryTargetSnapshot) { final List perCommitStoreFiles = new ArrayList<>(); for (StoreFileMetadata meta : this) { - if (IndexFileNames.OLD_SEGMENTS_GEN.equals(meta.name())) { // legacy - continue; // we don't need that file at all - } final String segmentId = IndexFileNames.parseSegmentName(meta.name()); final String extension = IndexFileNames.getExtension(meta.name()); if (IndexFileNames.SEGMENTS.equals(segmentId) || @@ -1006,9 +1003,9 @@ public RecoveryDiff recoveryDiff(MetadataSnapshot recoveryTargetSnapshot) { } RecoveryDiff recoveryDiff = new RecoveryDiff(Collections.unmodifiableList(identical), Collections.unmodifiableList(different), Collections.unmodifiableList(missing)); - assert recoveryDiff.size() == this.metadata.size() - (metadata.containsKey(IndexFileNames.OLD_SEGMENTS_GEN) ? 1 : 0) + assert recoveryDiff.size() == this.metadata.size() : "some files are missing recoveryDiff size: [" + recoveryDiff.size() + "] metadata size: [" + - this.metadata.size() + "] contains segments.gen: [" + metadata.containsKey(IndexFileNames.OLD_SEGMENTS_GEN) + "]"; + this.metadata.size() + "]"; return recoveryDiff; } @@ -1148,7 +1145,7 @@ public void verify() throws IOException { String footerDigest = null; if (metadata.checksum().equals(actualChecksum) && writtenBytes == metadata.length()) { ByteArrayIndexInput indexInput = new ByteArrayIndexInput("checksum", this.footerChecksum); - footerDigest = digestToString(indexInput.readLong()); + footerDigest = digestToString(CodecUtil.readBELong(indexInput)); if (metadata.checksum().equals(footerDigest)) { return; } @@ -1287,9 +1284,9 @@ public void seek(long pos) throws IOException { // skipping the verified portion input.seek(verifiedPosition); // and checking unverified - skipBytes(pos - verifiedPosition); + super.seek(pos); } else { - skipBytes(pos - getFilePointer()); + super.seek(pos); } } } @@ -1320,7 +1317,12 @@ public IndexInput slice(String sliceDescription, long offset, long length) throw } public long getStoredChecksum() { - return new ByteArrayDataInput(checksum).readLong(); + try { + return CodecUtil.readBELong(new ByteArrayDataInput(checksum)); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } } public long verify() throws CorruptIndexException { diff --git a/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java b/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java index 4d6865976492b..f88f9b83f34ba 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java +++ b/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.translog; +import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; @@ -40,11 +41,12 @@ final class Checkpoint { final long minTranslogGeneration; final long trimmedAboveSeqNo; - private static final int CURRENT_VERSION = 3; + private static final int VERSION_LUCENE_8 = 3; // int values written in BE format + private static final int CURRENT_VERSION = 4; private static final String CHECKPOINT_CODEC = "ckp"; - static final int V3_FILE_SIZE = CodecUtil.headerLength(CHECKPOINT_CODEC) + static final int V4_FILE_SIZE = CodecUtil.headerLength(CHECKPOINT_CODEC) + Integer.BYTES // ops + Long.BYTES // offset + Long.BYTES // generation @@ -115,6 +117,10 @@ static Checkpoint emptyTranslogCheckpoint(final long offset, final long generati } static Checkpoint readCheckpointV3(final DataInput in) throws IOException { + return readCheckpointV4(EndiannessReverserUtil.wrapDataInput(in)); + } + + static Checkpoint readCheckpointV4(final DataInput in) throws IOException { final long offset = in.readLong(); final int numOps = in.readInt(); final long generation = in.readLong(); @@ -145,10 +151,13 @@ public static Checkpoint read(Path path) throws IOException { try (IndexInput indexInput = dir.openInput(path.getFileName().toString(), IOContext.DEFAULT)) { // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here. CodecUtil.checksumEntireFile(indexInput); - final int fileVersion = CodecUtil.checkHeader(indexInput, CHECKPOINT_CODEC, CURRENT_VERSION, CURRENT_VERSION); - assert fileVersion == CURRENT_VERSION : fileVersion; - assert indexInput.length() == V3_FILE_SIZE : indexInput.length(); - return Checkpoint.readCheckpointV3(indexInput); + final int fileVersion = CodecUtil.checkHeader(indexInput, CHECKPOINT_CODEC, VERSION_LUCENE_8, CURRENT_VERSION); + assert fileVersion == VERSION_LUCENE_8 || fileVersion == CURRENT_VERSION; + assert indexInput.length() == V4_FILE_SIZE : indexInput.length(); + if (fileVersion == CURRENT_VERSION) { + return Checkpoint.readCheckpointV4(indexInput); + } + return readCheckpointV3(indexInput); } catch (CorruptIndexException | NoSuchFileException | IndexFormatTooOldException | IndexFormatTooNewException e) { throw new TranslogCorruptedException(path.toString(), e); } @@ -175,7 +184,7 @@ public static void write(FileChannel fileChannel, Path checkpointFile, Checkpoin } private static byte[] createCheckpointBytes(Path checkpointFile, Checkpoint checkpoint) throws IOException { - final ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(V3_FILE_SIZE) { + final ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(V4_FILE_SIZE) { @Override public synchronized byte[] toByteArray() { // don't clone @@ -184,13 +193,13 @@ public synchronized byte[] toByteArray() { }; final String resourceDesc = "checkpoint(path=\"" + checkpointFile + "\", gen=" + checkpoint + ")"; try (OutputStreamIndexOutput indexOutput = - new OutputStreamIndexOutput(resourceDesc, checkpointFile.toString(), byteOutputStream, V3_FILE_SIZE)) { + new OutputStreamIndexOutput(resourceDesc, checkpointFile.toString(), byteOutputStream, V4_FILE_SIZE)) { CodecUtil.writeHeader(indexOutput, CHECKPOINT_CODEC, CURRENT_VERSION); checkpoint.write(indexOutput); CodecUtil.writeFooter(indexOutput); - assert indexOutput.getFilePointer() == V3_FILE_SIZE : - "get you numbers straight; bytes written: " + indexOutput.getFilePointer() + ", buffer size: " + V3_FILE_SIZE; + assert indexOutput.getFilePointer() == V4_FILE_SIZE : + "get you numbers straight; bytes written: " + indexOutput.getFilePointer() + ", buffer size: " + V4_FILE_SIZE; assert indexOutput.getFilePointer() < 512 : "checkpoint files have to be smaller than 512 bytes for atomic writes; size: " + indexOutput.getFilePointer(); } diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index d4f1c3ffbfab0..fd7689d6edf2d 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -11,7 +11,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.LRUQueryCache; @@ -128,11 +127,6 @@ protected CachingWeightWrapper(Weight in) { this.in = in; } - @Override - public void extractTerms(Set terms) { - in.extractTerms(terms); - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { shardKeyMap.add(context.reader()); diff --git a/server/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java b/server/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java index 4b1afbe54968d..db44de570391f 100644 --- a/server/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java +++ b/server/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java @@ -13,12 +13,12 @@ import org.apache.lucene.analysis.hunspell.Dictionary; import org.apache.lucene.store.Directory; import org.apache.lucene.store.NIOFSDirectory; -import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.Environment; import java.io.IOException; diff --git a/server/src/main/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzers.java b/server/src/main/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzers.java index 652ac3a789cb6..a55d8de400466 100644 --- a/server/src/main/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzers.java +++ b/server/src/main/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzers.java @@ -9,12 +9,12 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.CharArraySet; +import org.apache.lucene.analysis.classic.ClassicAnalyzer; import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.analysis.core.SimpleAnalyzer; import org.apache.lucene.analysis.core.StopAnalyzer; import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.analysis.en.EnglishAnalyzer; -import org.apache.lucene.analysis.standard.ClassicAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.elasticsearch.Version; import org.elasticsearch.indices.analysis.PreBuiltCacheFactory.CachingStrategy; @@ -26,9 +26,7 @@ public enum PreBuiltAnalyzers { STANDARD(CachingStrategy.ELASTICSEARCH) { @Override protected Analyzer create(Version version) { - final Analyzer a = new StandardAnalyzer(CharArraySet.EMPTY_SET); - a.setVersion(version.luceneVersion); - return a; + return new StandardAnalyzer(CharArraySet.EMPTY_SET); } }, @@ -51,36 +49,28 @@ protected Analyzer create(Version version) { STOP { @Override protected Analyzer create(Version version) { - Analyzer a = new StopAnalyzer(EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); - a.setVersion(version.luceneVersion); - return a; + return new StopAnalyzer(EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); } }, WHITESPACE { @Override protected Analyzer create(Version version) { - Analyzer a = new WhitespaceAnalyzer(); - a.setVersion(version.luceneVersion); - return a; + return new WhitespaceAnalyzer(); } }, SIMPLE { @Override protected Analyzer create(Version version) { - Analyzer a = new SimpleAnalyzer(); - a.setVersion(version.luceneVersion); - return a; + return new SimpleAnalyzer(); } }, CLASSIC { @Override protected Analyzer create(Version version) { - Analyzer a = new ClassicAnalyzer(); - a.setVersion(version.luceneVersion); - return a; + return new ClassicAnalyzer(); } }; diff --git a/server/src/main/java/org/apache/lucene/queries/MinDocQuery.java b/server/src/main/java/org/elasticsearch/lucene/queries/MinDocQuery.java similarity index 96% rename from server/src/main/java/org/apache/lucene/queries/MinDocQuery.java rename to server/src/main/java/org/elasticsearch/lucene/queries/MinDocQuery.java index b2c1100fb73d0..14102e8dad7df 100644 --- a/server/src/main/java/org/apache/lucene/queries/MinDocQuery.java +++ b/server/src/main/java/org/elasticsearch/lucene/queries/MinDocQuery.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; @@ -15,6 +15,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; @@ -136,6 +137,10 @@ public long cost() { } } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } @Override public String toString(String field) { diff --git a/server/src/main/java/org/apache/lucene/queries/SearchAfterSortedDocQuery.java b/server/src/main/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQuery.java similarity index 97% rename from server/src/main/java/org/apache/lucene/queries/SearchAfterSortedDocQuery.java rename to server/src/main/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQuery.java index ea2d98370d4b5..0c0a0ffb99065 100644 --- a/server/src/main/java/org/apache/lucene/queries/SearchAfterSortedDocQuery.java +++ b/server/src/main/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQuery.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.ConstantScoreScorer; @@ -17,6 +17,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafFieldComparator; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Sort; @@ -112,6 +113,11 @@ public int hashCode() { return Objects.hash(classHash(), sort, after.doc, after.score, Arrays.hashCode(after.fields)); } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + interface TopComparator { boolean lessThanTop(int doc) throws IOException; } diff --git a/server/src/main/java/org/elasticsearch/lucene/similarity/LegacyBM25Similarity.java b/server/src/main/java/org/elasticsearch/lucene/similarity/LegacyBM25Similarity.java new file mode 100644 index 0000000000000..ccd19b3f984a6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/lucene/similarity/LegacyBM25Similarity.java @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/* @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.elasticsearch.lucene.similarity; + +import org.apache.lucene.index.FieldInvertState; +import org.apache.lucene.search.CollectionStatistics; +import org.apache.lucene.search.TermStatistics; +import org.apache.lucene.search.similarities.BM25Similarity; +import org.apache.lucene.search.similarities.Similarity; + +/** + * Similarity that behaves like {@link BM25Similarity} while also applying + * the k1+1 factor to the numerator of the scoring formula + * + * @see BM25Similarity + * + * @deprecated {@link BM25Similarity} should be used instead + */ +@Deprecated +public final class LegacyBM25Similarity extends Similarity { + + private final BM25Similarity bm25Similarity; + + /** BM25 with these default values: + *
    + *
  • {@code k1 = 1.2}
    • + *
  • {@code b = 0.75}
    • + *
+ */ + public LegacyBM25Similarity() { + this.bm25Similarity = new BM25Similarity(); + } + + /** + * BM25 with the supplied parameter values. + * @param k1 Controls non-linear term frequency normalization (saturation). + * @param b Controls to what degree document length normalizes tf values. + * @throws IllegalArgumentException if {@code k1} is infinite or negative, or if {@code b} is + * not within the range {@code [0..1]} + */ + public LegacyBM25Similarity(float k1, float b, boolean discountOverlaps) { + this.bm25Similarity = new BM25Similarity(k1, b, discountOverlaps); + } + + @Override + public long computeNorm(FieldInvertState state) { + return bm25Similarity.computeNorm(state); + } + + @Override + public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) { + return bm25Similarity.scorer(boost * (1 + bm25Similarity.getK1()), collectionStats, termStats); + } + + /** + * Returns the k1 parameter + */ + public float getK1() { + return bm25Similarity.getK1(); + } + + /** + * Returns the b parameter + */ + public float getB() { + return bm25Similarity.getB(); + } + + /** + * Returns true if overlap tokens are discounted from the document's length. + */ + public boolean getDiscountOverlaps() { + return bm25Similarity.getDiscountOverlaps(); + } + + @Override + public String toString() { + return bm25Similarity.toString(); + } +} diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java index 385151cb74286..e6eecc427ee99 100644 --- a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -13,7 +13,6 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.util.SPIClassIterator; import org.elasticsearch.Build; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; @@ -27,6 +26,7 @@ import org.elasticsearch.index.IndexModule; import org.elasticsearch.jdk.JarHell; import org.elasticsearch.node.ReportingService; +import org.elasticsearch.plugins.spi.SPIClassIterator; import org.elasticsearch.threadpool.ExecutorBuilder; import java.io.IOException; diff --git a/server/src/main/java/org/elasticsearch/plugins/spi/SPIClassIterator.java b/server/src/main/java/org/elasticsearch/plugins/spi/SPIClassIterator.java new file mode 100644 index 0000000000000..b8b70a8e122e9 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/plugins/spi/SPIClassIterator.java @@ -0,0 +1,170 @@ +/* @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.elasticsearch.plugins.spi; + +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.SuppressForbidden; + +import java.io.IOException; +import java.io.InputStream; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Enumeration; +import java.util.Iterator; +import java.util.Locale; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.ServiceConfigurationError; + +/** + * Helper class for loading SPI classes from classpath (META-INF files). + * This is a light impl of {@link java.util.ServiceLoader} but is guaranteed to + * be bug-free regarding classpath order and does not instantiate or initialize + * the classes found. + */ +@SuppressForbidden(reason="Copied from Lucene") +public final class SPIClassIterator implements Iterator> { + private static final String META_INF_SERVICES = "META-INF/services/"; + + private final Class clazz; + private final ClassLoader loader; + private final Enumeration profilesEnum; + private Iterator linesIterator; + + /** Creates a new SPI iterator to lookup services of type {@code clazz} using + * the same {@link ClassLoader} as the argument. */ + public static SPIClassIterator get(Class clazz) { + return new SPIClassIterator<>(clazz, + Objects.requireNonNull(clazz.getClassLoader(), () -> clazz + " has no classloader.")); + } + + /** Creates a new SPI iterator to lookup services of type {@code clazz} using the given classloader. */ + public static SPIClassIterator get(Class clazz, ClassLoader loader) { + return new SPIClassIterator<>(clazz, loader); + } + + /** + * Utility method to check if some class loader is a (grand-)parent of or the same as another one. + * This means the child will be able to load all classes from the parent, too. + *

+ * If caller's codesource doesn't have enough permissions to do the check, {@code false} is returned + * (this is fine, because if we get a {@code SecurityException} it is for sure no parent). + */ + public static boolean isParentClassLoader(final ClassLoader parent, final ClassLoader child) { + try { + ClassLoader cl = child; + while (cl != null) { + if (cl == parent) { + return true; + } + cl = cl.getParent(); + } + return false; + } catch (SecurityException se) { + return false; + } + } + + private SPIClassIterator(Class clazz, ClassLoader loader) { + this.clazz = Objects.requireNonNull(clazz, "clazz"); + this.loader = Objects.requireNonNull(loader, "loader"); + try { + final String fullName = META_INF_SERVICES + clazz.getName(); + this.profilesEnum = loader.getResources(fullName); + } catch (IOException ioe) { + throw new ServiceConfigurationError("Error loading SPI profiles for type " + clazz.getName() + " from classpath", ioe); + } + this.linesIterator = Collections.emptySet().iterator(); + } + + private boolean loadNextProfile() { + ArrayList lines = null; + while (profilesEnum.hasMoreElements()) { + if (lines != null) { + lines.clear(); + } else { + lines = new ArrayList<>(); + } + final URL url = profilesEnum.nextElement(); + try { + final InputStream in = url.openStream(); + boolean success = false; + try { + final BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)); + String line; + while ((line = reader.readLine()) != null) { + final int pos = line.indexOf('#'); + if (pos >= 0) { + line = line.substring(0, pos); + } + line = line.trim(); + if (line.length() > 0) { + lines.add(line); + } + } + success = true; + } finally { + if (success) { + IOUtils.close(in); + } else { + IOUtils.closeWhileHandlingException(in); + } + } + } catch (IOException ioe) { + throw new ServiceConfigurationError("Error loading SPI class list from URL: " + url, ioe); + } + if (lines.isEmpty() == false) { + this.linesIterator = lines.iterator(); + return true; + } + } + return false; + } + + @Override + public boolean hasNext() { + return linesIterator.hasNext() || loadNextProfile(); + } + + @Override + public Class next() { + // hasNext() implicitely loads the next profile, so it is essential to call this here! + if (hasNext() == false) { + throw new NoSuchElementException(); + } + assert linesIterator.hasNext(); + final String c = linesIterator.next(); + try { + // don't initialize the class (pass false as 2nd parameter): + return Class.forName(c, false, loader).asSubclass(clazz); + } catch (ClassNotFoundException cnfe) { + throw new ServiceConfigurationError(String.format(Locale.ROOT, "An SPI class of type %s with classname %s does not exist, "+ + "please fix the file '%s%1$s' in your classpath.", clazz.getName(), c, META_INF_SERVICES)); + } + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/MultiBucketCollector.java b/server/src/main/java/org/elasticsearch/search/aggregations/MultiBucketCollector.java index 28094c513acf2..7fe7b09b8f806 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/MultiBucketCollector.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/MultiBucketCollector.java @@ -213,7 +213,7 @@ private MultiLeafBucketCollector(List collectors, boolean c @Override public void setScorer(Scorable scorer) throws IOException { if (cacheScores) { - scorer = new ScoreCachingWrappingScorer(scorer); + scorer = ScoreCachingWrappingScorer.wrap(scorer); } for (int i = 0; i < numCollectors; ++i) { final LeafCollector c = collectors[i]; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java index 4ac07241921bf..da4f3669c09eb 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java @@ -13,7 +13,6 @@ import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.queries.SearchAfterSortedDocQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.CollectionTerminatedException; @@ -36,6 +35,7 @@ import org.elasticsearch.common.Rounding; import org.elasticsearch.core.Releasables; import org.elasticsearch.index.IndexSortConfig; +import org.elasticsearch.lucene.queries.SearchAfterSortedDocQuery; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/PointsSortedDocsProducer.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/PointsSortedDocsProducer.java index c9c1aaf4fd438..0a37ae3be6f05 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/PointsSortedDocsProducer.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/PointsSortedDocsProducer.java @@ -14,9 +14,9 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.Query; import org.apache.lucene.util.DocIdSetBuilder; -import org.apache.lucene.util.FutureArrays; import java.io.IOException; +import java.util.Arrays; import java.util.function.ToLongFunction; /** @@ -142,10 +142,9 @@ public void visit(int docID, byte[] packedValue) throws IOException { @Override public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { - if ((upperPointQuery != null - && FutureArrays.compareUnsigned(minPackedValue, 0, bytesPerDim, upperPointQuery, 0, bytesPerDim) > 0) + if ((upperPointQuery != null && Arrays.compareUnsigned(minPackedValue, 0, bytesPerDim, upperPointQuery, 0, bytesPerDim) > 0) || (lowerPointQuery != null - && FutureArrays.compareUnsigned(maxPackedValue, 0, bytesPerDim, lowerPointQuery, 0, bytesPerDim) < 0)) { + && Arrays.compareUnsigned(maxPackedValue, 0, bytesPerDim, lowerPointQuery, 0, bytesPerDim) < 0)) { // does not match the query return PointValues.Relation.CELL_OUTSIDE_QUERY; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQuery.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQuery.java index 226ea637457b1..157e5bc68d8b8 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQuery.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQuery.java @@ -18,6 +18,7 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.ScorerSupplier; @@ -180,6 +181,13 @@ public boolean equals(Object obj) { && delegateForSingleValuedSegments.equals(other.delegateForSingleValuedSegments); } + @Override + public void visit(QueryVisitor visitor) { + if (visitor.acceptField(field)) { + this.delegateForMultiValuedSegments.visit(visitor); + } + } + @Override public int hashCode() { return Objects.hash(classHash(), delegateForMultiValuedSegments, delegateForSingleValuedSegments); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java index 4d26ad1d1d70f..6375ed4268be5 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java @@ -10,6 +10,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BulkScorer; @@ -17,7 +18,6 @@ import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java index 7f0814f7bb1bb..6562fba3726e7 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java @@ -10,8 +10,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.search.DiversifiedTopDocsCollector; -import org.apache.lucene.search.DiversifiedTopDocsCollector.ScoreDocKey; +import org.apache.lucene.misc.search.DiversifiedTopDocsCollector; import org.apache.lucene.search.TopDocsCollector; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; @@ -28,6 +27,8 @@ import java.util.Map; import java.util.function.Consumer; +import static org.apache.lucene.misc.search.DiversifiedTopDocsCollector.ScoreDocKey; + /** * Alternative, faster implementation for converting String keys to longs but * with the potential for hash collisions. diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java index a69fb6357ba39..650617cb370e9 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java @@ -10,8 +10,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.search.DiversifiedTopDocsCollector; -import org.apache.lucene.search.DiversifiedTopDocsCollector.ScoreDocKey; +import org.apache.lucene.misc.search.DiversifiedTopDocsCollector; import org.apache.lucene.search.TopDocsCollector; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; @@ -30,6 +29,8 @@ import java.util.Map; import java.util.function.Consumer; +import static org.apache.lucene.misc.search.DiversifiedTopDocsCollector.ScoreDocKey; + public class DiversifiedMapSamplerAggregator extends SamplerAggregator { private ValuesSource valuesSource; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java index d4589df6e4efd..29545e4a88e67 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java @@ -11,8 +11,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.search.DiversifiedTopDocsCollector; -import org.apache.lucene.search.DiversifiedTopDocsCollector.ScoreDocKey; +import org.apache.lucene.misc.search.DiversifiedTopDocsCollector; import org.apache.lucene.search.TopDocsCollector; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.index.fielddata.AbstractNumericDocValues; @@ -27,6 +26,8 @@ import java.util.Map; import java.util.function.Consumer; +import static org.apache.lucene.misc.search.DiversifiedTopDocsCollector.ScoreDocKey; + public class DiversifiedNumericSamplerAggregator extends SamplerAggregator { private ValuesSource.Numeric valuesSource; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java index a10abe6944438..e74cc91a03556 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java @@ -13,8 +13,7 @@ import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.search.DiversifiedTopDocsCollector; -import org.apache.lucene.search.DiversifiedTopDocsCollector.ScoreDocKey; +import org.apache.lucene.misc.search.DiversifiedTopDocsCollector; import org.apache.lucene.search.TopDocsCollector; import org.elasticsearch.index.fielddata.AbstractNumericDocValues; import org.elasticsearch.search.aggregations.Aggregator; @@ -28,6 +27,8 @@ import java.util.Map; import java.util.function.Consumer; +import static org.apache.lucene.misc.search.DiversifiedTopDocsCollector.ScoreDocKey; + public class DiversifiedOrdinalsSamplerAggregator extends SamplerAggregator { private ValuesSource.Bytes.WithOrdinals.FieldData valuesSource; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregator.java index 4eb29ae86798b..077f2297bd048 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregator.java @@ -8,7 +8,7 @@ package org.elasticsearch.search.aggregations.bucket.sampler; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.DiversifiedTopDocsCollector; +import org.apache.lucene.misc.search.DiversifiedTopDocsCollector; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java index 1e571c4db5dda..6b94e603a0f86 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java @@ -117,16 +117,22 @@ public InternalAggregation reduce(List aggregations, Reduce InternalTopHits topHitsAgg = (InternalTopHits) aggregations.get(i); shardDocs[i] = topHitsAgg.topDocs.topDocs; shardHits[i] = topHitsAgg.searchHits; + for (ScoreDoc doc : shardDocs[i].scoreDocs) { + doc.shardIndex = i; + } } - reducedTopDocs = TopDocs.merge(sort, from, size, (TopFieldDocs[]) shardDocs, true); + reducedTopDocs = TopDocs.merge(sort, from, size, (TopFieldDocs[]) shardDocs); } else { shardDocs = new TopDocs[aggregations.size()]; for (int i = 0; i < shardDocs.length; i++) { InternalTopHits topHitsAgg = (InternalTopHits) aggregations.get(i); shardDocs[i] = topHitsAgg.topDocs.topDocs; shardHits[i] = topHitsAgg.searchHits; + for (ScoreDoc doc : shardDocs[i].scoreDocs) { + doc.shardIndex = i; + } } - reducedTopDocs = TopDocs.merge(from, size, shardDocs, true); + reducedTopDocs = TopDocs.merge(from, size, shardDocs); } float maxScore = Float.NaN; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregator.java index b3a60f379aa84..11b805cbc3f6b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregator.java @@ -12,7 +12,6 @@ import org.apache.lucene.index.PointValues; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.FutureArrays; import org.elasticsearch.common.util.DoubleArray; import org.elasticsearch.core.Releasables; import org.elasticsearch.index.fielddata.NumericDoubleValues; @@ -28,6 +27,7 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import java.io.IOException; +import java.util.Arrays; import java.util.Map; import java.util.function.Function; @@ -168,7 +168,7 @@ public void visit(int docID, byte[] packedValue) { @Override public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { - if (FutureArrays.equals(maxValue, 0, numBytes, maxPackedValue, 0, numBytes)) { + if (Arrays.equals(maxValue, 0, numBytes, maxPackedValue, 0, numBytes)) { // we only check leaves that contain the max value for the segment. return PointValues.Relation.CELL_CROSSES_QUERY; } else { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java index dd8bd9732d118..cf454d7b1b73a 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java @@ -11,7 +11,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.CollectionTerminatedException; import org.apache.lucene.search.Collector; -import org.apache.lucene.search.ConjunctionDISI; +import org.apache.lucene.search.ConjunctionUtils; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.ScoreMode; @@ -160,7 +160,7 @@ public static void intersect(Weight weight, Weight innerHitQueryWeight, Collecto try { Bits acceptDocs = ctx.reader().getLiveDocs(); - DocIdSetIterator iterator = ConjunctionDISI.intersectIterators(Arrays.asList(innerHitQueryScorer.iterator(), + DocIdSetIterator iterator = ConjunctionUtils.intersectIterators(Arrays.asList(innerHitQueryScorer.iterator(), scorer.iterator())); for (int docId = iterator.nextDoc(); docId < DocIdSetIterator.NO_MORE_DOCS; docId = iterator.nextDoc()) { if (acceptDocs == null || acceptDocs.get(docId)) { diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index e0897f55cace0..14d6f4afcc209 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -16,7 +16,7 @@ import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.CollectionTerminatedException; import org.apache.lucene.search.Collector; -import org.apache.lucene.search.ConjunctionDISI; +import org.apache.lucene.search.ConjunctionUtils; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; @@ -207,11 +207,6 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto private Weight wrapWeight(Weight weight) { if (cancellable.isEnabled()) { return new Weight(weight.getQuery()) { - @Override - public void extractTerms(Set terms) { - throw new UnsupportedOperationException(); - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { throw new UnsupportedOperationException(); @@ -261,7 +256,7 @@ static void intersectScorerAndBitSet(Scorer scorer, BitSet acceptDocs, collector.setScorer(scorer); // ConjunctionDISI uses the DocIdSetIterator#cost() to order the iterators, so if roleBits has the lowest cardinality it should // be used first: - DocIdSetIterator iterator = ConjunctionDISI.intersectIterators(Arrays.asList(new BitSetIterator(acceptDocs, + DocIdSetIterator iterator = ConjunctionUtils.intersectIterators(Arrays.asList(new BitSetIterator(acceptDocs, acceptDocs.approximateCardinality()), scorer.iterator())); int seen = 0; checkCancelled.run(); diff --git a/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java b/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java index 34004f7a3ebc3..a51cf8aa77141 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java +++ b/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java @@ -213,11 +213,6 @@ public void checkIntegrity() throws IOException { public void close() throws IOException { reader.close(); } - - @Override - public long ramBytesUsed() { - return reader.ramBytesUsed(); - } } private class FieldUsageTrackingTerms extends FilterTerms { diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilterFieldNamesProvidingStoredFieldsVisitor.java b/server/src/main/java/org/elasticsearch/search/internal/FilterFieldNamesProvidingStoredFieldsVisitor.java index 329a746f075a8..6d3c7395a62a0 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/FilterFieldNamesProvidingStoredFieldsVisitor.java +++ b/server/src/main/java/org/elasticsearch/search/internal/FilterFieldNamesProvidingStoredFieldsVisitor.java @@ -28,7 +28,7 @@ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { } @Override - public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { + public void stringField(FieldInfo fieldInfo, String value) throws IOException { visitor.stringField(fieldInfo, value); } diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilterStoredFieldVisitor.java b/server/src/main/java/org/elasticsearch/search/internal/FilterStoredFieldVisitor.java index a695fae27346d..1051eee5d54f1 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/FilterStoredFieldVisitor.java +++ b/server/src/main/java/org/elasticsearch/search/internal/FilterStoredFieldVisitor.java @@ -26,7 +26,7 @@ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { } @Override - public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { + public void stringField(FieldInfo fieldInfo, String value) throws IOException { visitor.stringField(fieldInfo, value); } diff --git a/server/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java b/server/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java index 52a520b01fab4..2114f7321ad79 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java +++ b/server/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java @@ -9,7 +9,6 @@ package org.elasticsearch.search.profile.query; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.Query; @@ -19,7 +18,6 @@ import org.elasticsearch.search.profile.Timer; import java.io.IOException; -import java.util.Set; /** * Weight wrapper that will compute how much time it takes to build the @@ -103,8 +101,8 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio } @Override - public void extractTerms(Set set) { - subQueryWeight.extractTerms(set); + public int count(LeafReaderContext context) throws IOException { + return subQueryWeight.count(context); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 997d420cb8e5d..52f94bd8e5af1 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -12,8 +12,6 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.queries.MinDocQuery; -import org.apache.lucene.queries.SearchAfterSortedDocQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Collector; @@ -23,7 +21,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.search.SearchShardTask; @@ -32,10 +29,8 @@ import org.elasticsearch.common.util.concurrent.EWMATrackingEsThreadPoolExecutor; import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor; import org.elasticsearch.core.Booleans; -import org.elasticsearch.index.IndexSortConfig; -import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.lucene.queries.MinDocQuery; +import org.elasticsearch.lucene.queries.SearchAfterSortedDocQuery; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchContextSourcePrinter; import org.elasticsearch.search.SearchService; @@ -208,11 +203,6 @@ static boolean executeInternal(SearchContext searchContext) throws QueryPhaseExe hasFilterCollector = true; } - if (SYS_PROP_REWRITE_SORT) { - optimizeNumericSort(searchContext, searcher.getIndexReader()); - // TODO: sort leaves according to search sort when Lucene supports sharing bottom sort value between collectors - } - boolean timeoutSet = scrollContext == null && searchContext.timeout() != null && searchContext.timeout().equals(SearchService.NO_TIMEOUT) == false; @@ -301,35 +291,6 @@ private static boolean searchWithCollector(SearchContext searchContext, ContextI return topDocsFactory.shouldRescore(); } - - // TODO: remove this after Lucene 9.0, as sort optimization is enabled by default there - private static void optimizeNumericSort(SearchContext searchContext, IndexReader reader) { - if (searchContext.sort() == null) return; - // disable this optimization if index sorting matches the query sort since it's already optimized by index searcher - if (canEarlyTerminate(reader, searchContext.sort())) return; - - SortField sortField = searchContext.sort().sort.getSort()[0]; - SortField.Type sortType = IndexSortConfig.getSortFieldType(sortField); - //if (SortField.Type.LONG.equals(IndexSortConfig.getSortFieldType(sortField)) == false) return null; - - if (sortType != SortField.Type.LONG) return; // for now restrict sort optimization only to long sort - String fieldName = sortField.getField(); - if (fieldName == null) return; // happens when _score or _doc is the 1st sort field - SearchExecutionContext searchExecutionContext = searchContext.getSearchExecutionContext(); - final MappedFieldType fieldType = searchExecutionContext.getFieldType(fieldName); - if (fieldType == null) return; // for unmapped fields, default behaviour depending on "unmapped_type" flag - if (fieldType.isSearchable() == false) return; - if (fieldType.hasDocValues() == false) return; - - // For now restrict sort optimization only to long and date fields - // For sort optimization SortField.Type must match with the type of indexed points (Type.LONG and LongPoint) - // Some fields there is no match (e.g. integer field uses SortField.Type.LONG, but indexed as IntegerPoint) - if ((fieldType.typeName().equals("long") == false) && (fieldType instanceof DateFieldMapper.DateFieldType == false)) return; - - sortField.setCanUsePoints(); - return; - } - /** * Returns true if the provided query returns docs in index order (internal doc ids). * @param query The query to execute diff --git a/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java b/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java index 1d0000a2c579d..c36863d2f30de 100644 --- a/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java +++ b/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java @@ -16,6 +16,7 @@ import org.apache.lucene.index.PointValues; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; +import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.Collector; @@ -39,14 +40,13 @@ import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.grouping.CollapseTopFieldDocs; import org.apache.lucene.search.grouping.CollapsingTopDocsCollector; -import org.apache.lucene.search.spans.SpanQuery; import org.elasticsearch.action.search.MaxScoreCollector; -import org.elasticsearch.core.Nullable; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore; import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery; import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery; import org.elasticsearch.common.util.CachedSupplier; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.search.ESToParentBlockJoinQuery; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.collapse.CollapseContext; diff --git a/server/src/main/java/org/elasticsearch/search/runtime/AbstractScriptFieldQuery.java b/server/src/main/java/org/elasticsearch/search/runtime/AbstractScriptFieldQuery.java index c73891b799958..8d81ce4e6dc5c 100644 --- a/server/src/main/java/org/elasticsearch/search/runtime/AbstractScriptFieldQuery.java +++ b/server/src/main/java/org/elasticsearch/search/runtime/AbstractScriptFieldQuery.java @@ -15,6 +15,7 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TwoPhaseIterator; @@ -111,6 +112,13 @@ public boolean equals(Object obj) { return script.equals(other.script) && fieldName.equals(other.fieldName); } + @Override + public void visit(QueryVisitor visitor) { + if (visitor.acceptField(fieldName)) { + visitor.visitLeaf(this); + } + } + final Explanation explainMatch(float boost, String description) { return Explanation.match( boost, diff --git a/server/src/main/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQuery.java b/server/src/main/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQuery.java index 9d8fd3768ba53..a7977c18d338c 100644 --- a/server/src/main/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQuery.java +++ b/server/src/main/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQuery.java @@ -11,7 +11,6 @@ import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.geo.GeoUtils; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; @@ -26,7 +25,6 @@ import java.io.IOException; import java.util.Objects; -import java.util.Set; import java.util.function.Function; public final class GeoPointScriptFieldDistanceFeatureQuery extends AbstractScriptFieldQuery { @@ -79,9 +77,6 @@ public boolean isCacheable(LeafReaderContext ctx) { return false; } - @Override - public void extractTerms(Set terms) {} - @Override public Scorer scorer(LeafReaderContext context) { return new DistanceScorer(this, scriptContextFunction().apply(context), context.reader().maxDoc(), boost); diff --git a/server/src/main/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQuery.java b/server/src/main/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQuery.java index f0b9a76bb65da..c5a751e4ba0e8 100644 --- a/server/src/main/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQuery.java +++ b/server/src/main/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQuery.java @@ -9,7 +9,6 @@ package org.elasticsearch.search.runtime; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; @@ -23,7 +22,6 @@ import java.io.IOException; import java.util.Objects; -import java.util.Set; import java.util.function.Function; public final class LongScriptFieldDistanceFeatureQuery extends AbstractScriptFieldQuery { @@ -56,9 +54,6 @@ public boolean isCacheable(LeafReaderContext ctx) { return false; } - @Override - public void extractTerms(Set terms) {} - @Override public Scorer scorer(LeafReaderContext context) { return new DistanceScorer(this, scriptContextFunction().apply(context), context.reader().maxDoc(), boost); diff --git a/server/src/main/java/org/elasticsearch/search/slice/SliceQuery.java b/server/src/main/java/org/elasticsearch/search/slice/SliceQuery.java index 920f0a4691f86..3a9f8cc449561 100644 --- a/server/src/main/java/org/elasticsearch/search/slice/SliceQuery.java +++ b/server/src/main/java/org/elasticsearch/search/slice/SliceQuery.java @@ -9,6 +9,7 @@ package org.elasticsearch.search.slice; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import java.util.Objects; @@ -67,4 +68,10 @@ public String toString(String f) { return getClass().getSimpleName() + "[field=" + field + ", id=" + id + ", max=" + max + "]"; } + @Override + public void visit(QueryVisitor visitor) { + if (visitor.acceptField(field)) { + visitor.visitLeaf(this); + } + } } diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy index 5ae15e74ec2d4..b081ff31cb2ef 100644 --- a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -22,6 +22,8 @@ grant codeBase "${codebase.elasticsearch-secure-sm}" { grant codeBase "${codebase.elasticsearch}" { // needed for loading plugins which may expect the context class loader to be set permission java.lang.RuntimePermission "setContextClassLoader"; + // needed for SPI class loading + permission java.lang.RuntimePermission "accessDeclaredMembers"; }; //// Very special jar permissions: diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java index e6f70bf1d1a04..3e65892738c3c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java @@ -10,9 +10,9 @@ import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat; -import org.apache.lucene.codecs.lucene84.Lucene84PostingsFormat; -import org.apache.lucene.codecs.lucene87.Lucene87Codec; +import org.apache.lucene.codecs.lucene90.Lucene90Codec; +import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; +import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.apache.lucene.document.BinaryDocValuesField; @@ -38,7 +38,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentReader; -import org.apache.lucene.search.suggest.document.Completion84PostingsFormat; +import org.apache.lucene.search.suggest.document.Completion90PostingsFormat; import org.apache.lucene.search.suggest.document.CompletionPostingsFormat; import org.apache.lucene.search.suggest.document.SuggestField; import org.apache.lucene.store.Directory; @@ -153,11 +153,11 @@ public void testCompletionField() throws Exception { IndexWriterConfig config = new IndexWriterConfig() .setCommitOnClose(true) .setUseCompoundFile(false) - .setCodec(new Lucene87Codec(Lucene87Codec.Mode.BEST_SPEED) { + .setCodec(new Lucene90Codec(Lucene90Codec.Mode.BEST_SPEED) { @Override public PostingsFormat getPostingsFormatForField(String field) { if (field.startsWith("suggest_")) { - return new Completion84PostingsFormat(randomFrom(CompletionPostingsFormat.FSTLoadMode.values())); + return new Completion90PostingsFormat(randomFrom(CompletionPostingsFormat.FSTLoadMode.values())); } else { return super.postingsFormat(); } @@ -214,38 +214,26 @@ public void testMixedFields() throws Exception { enum CodecMode { BEST_SPEED { @Override - Lucene87Codec.Mode mode() { - return Lucene87Codec.Mode.BEST_SPEED; - } - - @Override - DocValuesFormat dvFormat() { - return new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_SPEED); + Lucene90Codec.Mode mode() { + return Lucene90Codec.Mode.BEST_SPEED; } }, BEST_COMPRESSION { @Override - Lucene87Codec.Mode mode() { - return Lucene87Codec.Mode.BEST_COMPRESSION; - } - - @Override - DocValuesFormat dvFormat() { - return new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION); + Lucene90Codec.Mode mode() { + return Lucene90Codec.Mode.BEST_COMPRESSION; } }; - abstract Lucene87Codec.Mode mode(); - - abstract DocValuesFormat dvFormat(); + abstract Lucene90Codec.Mode mode(); } static void indexRandomly(Directory directory, CodecMode codecMode, int numDocs, Consumer addFields) throws IOException { IndexWriterConfig config = new IndexWriterConfig() .setCommitOnClose(true) .setUseCompoundFile(randomBoolean()) - .setCodec(new Lucene87Codec(codecMode.mode())); + .setCodec(new Lucene90Codec(codecMode.mode())); try (IndexWriter writer = new IndexWriter(directory, config)) { for (int i = 0; i < numDocs; i++) { final Document doc = new Document(); @@ -406,15 +394,15 @@ static void rewriteIndexWithPerFieldCodec(Directory source, CodecMode mode, Dire IndexWriterConfig config = new IndexWriterConfig() .setSoftDeletesField(Lucene.SOFT_DELETES_FIELD) .setUseCompoundFile(randomBoolean()) - .setCodec(new Lucene87Codec(mode.mode()) { + .setCodec(new Lucene90Codec(mode.mode()) { @Override public PostingsFormat getPostingsFormatForField(String field) { - return new Lucene84PostingsFormat(); + return new Lucene90PostingsFormat(); } @Override public DocValuesFormat getDocValuesFormatForField(String field) { - return mode.dvFormat(); + return new Lucene90DocValuesFormat(); } @Override diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java index c9a32f55c25d4..7190b7ec216c1 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.engine.Segment; import org.elasticsearch.index.MergePolicyConfig; import org.elasticsearch.indices.IndexClosedException; import org.elasticsearch.plugins.Plugin; @@ -19,7 +18,6 @@ import org.junit.Before; import java.util.Collection; -import java.util.List; import static org.hamcrest.Matchers.is; @@ -47,18 +45,6 @@ public void setupIndex() { client().admin().indices().prepareRefresh().get(); } - public void testBasic() { - IndicesSegmentResponse rsp = client().admin().indices().prepareSegments("test").get(); - List segments = rsp.getIndices().get("test").iterator().next().getShards()[0].getSegments(); - assertNull(segments.get(0).toString(), segments.get(0).ramTree); - } - - public void testVerbose() { - IndicesSegmentResponse rsp = client().admin().indices().prepareSegments("test").setVerbose(true).get(); - List segments = rsp.getIndices().get("test").iterator().next().getShards()[0].getSegments(); - assertNotNull(segments.get(0).toString(), segments.get(0).ramTree); - } - /** * with the default IndicesOptions inherited from BroadcastOperationRequest this will raise an exception */ diff --git a/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java b/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java index 7f5fe7a09b1c9..acbac4aa51a1c 100644 --- a/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java +++ b/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java @@ -32,6 +32,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -434,6 +435,11 @@ public int hashCode() { return 42; } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new Weight(this) { @@ -443,11 +449,6 @@ public boolean isCacheable(LeafReaderContext ctx) { return true; } - @Override - public void extractTerms(Set terms) { - throw new UnsupportedOperationException(); - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { throw new UnsupportedOperationException(); diff --git a/server/src/test/java/org/elasticsearch/common/lucene/search/function/MinScoreScorerTests.java b/server/src/test/java/org/elasticsearch/common/lucene/search/function/MinScoreScorerTests.java index 0adbf9ad33e66..f21b528725808 100644 --- a/server/src/test/java/org/elasticsearch/common/lucene/search/function/MinScoreScorerTests.java +++ b/server/src/test/java/org/elasticsearch/common/lucene/search/function/MinScoreScorerTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.common.lucene.search.function; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.MatchAllDocsQuery; @@ -59,11 +58,6 @@ public int advance(int target) throws IOException { private static Weight fakeWeight() { return new Weight(new MatchAllDocsQuery()) { - @Override - public void extractTerms(Set terms) { - - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return null; diff --git a/server/src/test/java/org/elasticsearch/gateway/MetadataStateFormatTests.java b/server/src/test/java/org/elasticsearch/gateway/MetadataStateFormatTests.java index 8663050dda91f..8dc61cc03e039 100644 --- a/server/src/test/java/org/elasticsearch/gateway/MetadataStateFormatTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/MetadataStateFormatTests.java @@ -179,7 +179,7 @@ public static void corruptFile(Path fileToCorrupt, Logger logger) throws IOExcep assertThat(input.getFilePointer(), is(0L)); input.seek(input.length() - 8); // one long is the checksum... 8 bytes checksumAfterCorruption = input.getChecksum(); - actualChecksumAfterCorruption = input.readLong(); + actualChecksumAfterCorruption = CodecUtil.readBELong(input); } StringBuilder msg = new StringBuilder(); msg.append("Checksum before: [").append(checksumBeforeCorruption).append("]"); diff --git a/server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java b/server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java index afe7af9c61a3e..c78a19b77e963 100644 --- a/server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java +++ b/server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java @@ -48,12 +48,6 @@ public void testSetMaxMergeAtOnce() { assertEquals(42, policy.regularMergePolicy.getMaxMergeAtOnce()); } - public void testSetMaxMergeAtOnceExplicit() { - EsTieredMergePolicy policy = new EsTieredMergePolicy(); - policy.setMaxMergeAtOnceExplicit(42); - assertEquals(42, policy.forcedMergePolicy.getMaxMergeAtOnceExplicit()); - } - public void testSetSegmentsPerTier() { EsTieredMergePolicy policy = new EsTieredMergePolicy(); policy.setSegmentsPerTier(42); diff --git a/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java b/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java index c81dcf0dfffbb..5396b0b880b49 100644 --- a/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java @@ -96,14 +96,6 @@ public void testTieredMergePolicySettingsUpdate() throws IOException { assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1); - assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), - MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT); - indexSettings.updateIndexMetadata(newIndexMeta("index", - Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING.getKey(), - MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT - 1).build())); - assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), - MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT-1); - assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getMbFrac(), 0.0001); indexSettings.updateIndexMetadata(newIndexMeta("index", @@ -138,8 +130,6 @@ public void testTieredMergePolicySettingsUpdate() throws IOException { new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb(), ByteSizeUnit.MB).getMbFrac(), 0.00); assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE); - assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), - MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT); assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001); assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index 48d99a7f97cad..ec1a49dafe75c 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -9,17 +9,14 @@ package org.elasticsearch.index.codec; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat; -import org.apache.lucene.codecs.lucene87.Lucene87Codec; -import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat; -import org.apache.lucene.document.BinaryDocValuesField; +import org.apache.lucene.codecs.lucene90.Lucene90Codec; +import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.SegmentReader; import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; @@ -44,24 +41,21 @@ public class CodecTests extends ESTestCase { public void testResolveDefaultCodecs() throws Exception { CodecService codecService = createCodecService(); assertThat(codecService.codec("default"), instanceOf(PerFieldMappingPostingFormatCodec.class)); - assertThat(codecService.codec("default"), instanceOf(Lucene87Codec.class)); - assertThat(codecService.codec("Lucene87"), instanceOf(Lucene87Codec.class)); + assertThat(codecService.codec("default"), instanceOf(Lucene90Codec.class)); } public void testDefault() throws Exception { Codec codec = createCodecService().codec("default"); - assertStoredFieldsCompressionEquals(Lucene87StoredFieldsFormat.Mode.BEST_SPEED, codec); - assertDocValuesCompressionEquals(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsCompressionEquals(Lucene90Codec.Mode.BEST_SPEED, codec); } public void testBestCompression() throws Exception { Codec codec = createCodecService().codec("best_compression"); - assertStoredFieldsCompressionEquals(Lucene87StoredFieldsFormat.Mode.BEST_COMPRESSION, codec); - assertDocValuesCompressionEquals(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsCompressionEquals(Lucene90Codec.Mode.BEST_COMPRESSION, codec); } // write some docs with it, inspect .si to see this was the used compression - private void assertStoredFieldsCompressionEquals(Lucene87StoredFieldsFormat.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene90Codec.Mode expected, Codec actual) throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(null); iwc.setCodec(actual); @@ -71,29 +65,9 @@ private void assertStoredFieldsCompressionEquals(Lucene87StoredFieldsFormat.Mode iw.close(); DirectoryReader ir = DirectoryReader.open(dir); SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader(); - String v = sr.getSegmentInfo().info.getAttribute(Lucene87StoredFieldsFormat.MODE_KEY); + String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene87StoredFieldsFormat.Mode.valueOf(v)); - ir.close(); - dir.close(); - } - - // write some docs with it, inspect .si to see this was the used compression - private void assertDocValuesCompressionEquals(Lucene80DocValuesFormat.Mode expected, Codec actual) throws Exception { - Directory dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(null); - iwc.setCodec(actual); - IndexWriter iw = new IndexWriter(dir, iwc); - Document doc = new Document(); - doc.add(new BinaryDocValuesField("foo", new BytesRef("aaa"))); - iw.addDocument(doc); - iw.commit(); - iw.close(); - DirectoryReader ir = DirectoryReader.open(dir); - SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader(); - String v = sr.getFieldInfos().fieldInfo("foo").getAttribute(Lucene80DocValuesFormat.MODE_KEY); - assertNotNull(v); - assertEquals(expected, Lucene80DocValuesFormat.Mode.valueOf(v)); + assertEquals(expected, Lucene90Codec.Mode.valueOf(v)); ir.close(); dir.close(); } diff --git a/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java b/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java index d981062b678a1..502fca04063d5 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java @@ -8,14 +8,14 @@ package org.elasticsearch.index.engine; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene87.Lucene87Codec; +import org.apache.lucene.codecs.lucene90.Lucene90Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryCachingPolicy; -import org.apache.lucene.search.suggest.document.Completion84PostingsFormat; +import org.apache.lucene.search.suggest.document.Completion90PostingsFormat; import org.apache.lucene.search.suggest.document.SuggestField; import org.apache.lucene.store.Directory; import org.elasticsearch.ElasticsearchException; @@ -45,8 +45,8 @@ public void testExceptionsAreNotCached() { public void testCompletionStatsCache() throws IOException, InterruptedException { final IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); - final PostingsFormat postingsFormat = new Completion84PostingsFormat(); - indexWriterConfig.setCodec(new Lucene87Codec() { + final PostingsFormat postingsFormat = new Completion90PostingsFormat(); + indexWriterConfig.setCodec(new Lucene90Codec() { @Override public PostingsFormat getPostingsFormatForField(String field) { return postingsFormat; // all fields are suggest fields diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 122b2a80c3eb5..facd673a35752 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -284,7 +284,6 @@ public void testVerboseSegments() throws Exception { segments = engine.segments(true); assertThat(segments.size(), equalTo(1)); - assertThat(segments.get(0).ramTree, notNullValue()); ParsedDocument doc2 = testParsedDocument("2", null, testDocumentWithTextField(), B_2, null); engine.index(indexForDoc(doc2)); @@ -295,9 +294,6 @@ public void testVerboseSegments() throws Exception { segments = engine.segments(true); assertThat(segments.size(), equalTo(3)); - assertThat(segments.get(0).ramTree, notNullValue()); - assertThat(segments.get(1).ramTree, notNullValue()); - assertThat(segments.get(2).ramTree, notNullValue()); } } diff --git a/server/src/test/java/org/elasticsearch/index/engine/SegmentTests.java b/server/src/test/java/org/elasticsearch/index/engine/SegmentTests.java index 3aab467fdbabe..21c54f7ad08bc 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/SegmentTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/SegmentTests.java @@ -66,7 +66,7 @@ static Segment randomSegment() { segment.sizeInBytes = randomNonNegativeLong(); segment.docCount = randomIntBetween(1, Integer.MAX_VALUE); segment.delDocCount = randomIntBetween(0, segment.docCount); - segment.version = Version.LUCENE_7_0_0; + segment.version = Version.LUCENE_8_0_0; segment.compound = randomBoolean(); segment.mergeId = randomAlphaOfLengthBetween(1, 10); segment.segmentSort = randomIndexSort(); diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java index 2dc9c9258e8d1..bae54a1d82461 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java @@ -203,7 +203,7 @@ protected void fillExtendedMvSet() throws Exception { writer.addDocument(d); d = new Document(); - d.add(new StringField("_id", "6", Field.Store.NO)); + addField(d, "_id", "6"); writer.addDocument(d); d = new Document(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index 27ad439f16a57..2285087d5f517 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -14,7 +14,7 @@ import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; -import org.apache.lucene.search.suggest.document.Completion84PostingsFormat; +import org.apache.lucene.search.suggest.document.Completion90PostingsFormat; import org.apache.lucene.search.suggest.document.CompletionAnalyzer; import org.apache.lucene.search.suggest.document.ContextSuggestField; import org.apache.lucene.search.suggest.document.FuzzyCompletionQuery; @@ -124,7 +124,7 @@ public void testPostingsFormat() throws IOException { Codec codec = codecService.codec("default"); assertThat(codec, instanceOf(PerFieldMappingPostingFormatCodec.class)); PerFieldMappingPostingFormatCodec perFieldCodec = (PerFieldMappingPostingFormatCodec) codec; - assertThat(perFieldCodec.getPostingsFormatForField("field"), instanceOf(Completion84PostingsFormat.class)); + assertThat(perFieldCodec.getPostingsFormatForField("field"), instanceOf(Completion90PostingsFormat.class)); } public void testDefaultConfiguration() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index 75792b47dfcf2..72ae9f6976d84 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -16,9 +16,9 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexOrDocValuesQuery; -import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.elasticsearch.Version; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index 56de4f3a9034a..fa8aa3f0e5fae 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -12,16 +12,16 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.FloatPoint; -import org.apache.lucene.document.HalfFloatPoint; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.sandbox.document.HalfFloatPoint; +import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.Sort; @@ -419,6 +419,7 @@ public void doTestDocValueRangeQueries(NumberType type, Supplier valueSu dir.close(); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/74057") public void testIndexSortIntRange() throws Exception { doTestIndexSortRangeQueries(NumberType.INTEGER, random()::nextInt); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 1b41a5bea136c..4d220a0b09ef4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -26,6 +26,10 @@ import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.queries.spans.FieldMaskingSpanQuery; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; @@ -38,10 +42,6 @@ import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.spans.FieldMaskingSpanQuery; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Strings; import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery; diff --git a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java index ca0f7d6ae2316..7947ad35ef4b8 100644 --- a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java @@ -8,8 +8,8 @@ package org.elasticsearch.index.query; +import org.apache.lucene.sandbox.search.CombinedFieldQuery; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.CombinedFieldQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; diff --git a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java index e8d508a6e1ed0..995d87efc4711 100644 --- a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java @@ -13,10 +13,10 @@ import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.index.Term; +import org.apache.lucene.sandbox.search.CombinedFieldQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; -import org.apache.lucene.search.CombinedFieldQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; diff --git a/server/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java index 451a4c729c872..26e75ae63383a 100644 --- a/server/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java @@ -18,14 +18,9 @@ import java.io.IOException; import java.util.Collection; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; -import static org.hamcrest.Matchers.closeTo; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; - public class DisMaxQueryBuilderTests extends AbstractQueryTestCase { /** * @return a {@link DisMaxQueryBuilder} with random inner queries @@ -46,14 +41,8 @@ protected DisMaxQueryBuilder doCreateTestQueryBuilder() { @Override protected void doAssertLuceneQuery(DisMaxQueryBuilder queryBuilder, Query query, SearchExecutionContext context) throws IOException { Collection queries = AbstractQueryBuilder.toQueries(queryBuilder.innerQueries(), context); - assertThat(query, instanceOf(DisjunctionMaxQuery.class)); - DisjunctionMaxQuery disjunctionMaxQuery = (DisjunctionMaxQuery) query; - assertThat(disjunctionMaxQuery.getTieBreakerMultiplier(), equalTo(queryBuilder.tieBreaker())); - assertThat(disjunctionMaxQuery.getDisjuncts().size(), equalTo(queries.size())); - Iterator queryIterator = queries.iterator(); - for (int i = 0; i < disjunctionMaxQuery.getDisjuncts().size(); i++) { - assertThat(disjunctionMaxQuery.getDisjuncts().get(i), equalTo(queryIterator.next())); - } + Query expected = new DisjunctionMaxQuery(queries, queryBuilder.tieBreaker()); + assertEquals(expected, query); } @Override @@ -92,20 +81,10 @@ public void testToQueryInnerPrefixQuery() throws Exception { " }\n" + "}"; Query query = parseQuery(queryAsString).toQuery(createSearchExecutionContext()); - assertThat(query, instanceOf(DisjunctionMaxQuery.class)); - DisjunctionMaxQuery disjunctionMaxQuery = (DisjunctionMaxQuery) query; - - List disjuncts = disjunctionMaxQuery.getDisjuncts(); - assertThat(disjuncts.size(), equalTo(1)); - - assertThat(disjuncts.get(0), instanceOf(BoostQuery.class)); - BoostQuery boostQuery = (BoostQuery) disjuncts.get(0); - assertThat((double) boostQuery.getBoost(), closeTo(1.2, 0.00001)); - assertThat(boostQuery.getQuery(), instanceOf(PrefixQuery.class)); - PrefixQuery firstQ = (PrefixQuery) boostQuery.getQuery(); - // since age is automatically registered in data, we encode it as numeric - assertThat(firstQ.getPrefix(), equalTo(new Term(TEXT_FIELD_NAME, "sh"))); - + Query expected = new DisjunctionMaxQuery(List.of( + new BoostQuery(new PrefixQuery(new Term(TEXT_FIELD_NAME, "sh")), 1.2f) + ), 0); + assertEquals(expected, query); } public void testFromJson() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java index 7874867773f83..64621efe28636 100644 --- a/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java @@ -9,10 +9,10 @@ package org.elasticsearch.index.query; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.spans.FieldMaskingSpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.FieldMaskingSpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.elasticsearch.common.ParsingException; import org.elasticsearch.test.AbstractQueryTestCase; diff --git a/server/src/test/java/org/elasticsearch/index/query/MatchBoolPrefixQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MatchBoolPrefixQueryBuilderTests.java index 57288be5cdbbf..d1b2bf106aea0 100644 --- a/server/src/test/java/org/elasticsearch/index/query/MatchBoolPrefixQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/MatchBoolPrefixQueryBuilderTests.java @@ -249,7 +249,10 @@ public void testAnalysisSynonym() throws Exception { assertBooleanQuery(query, asList( new TermQuery(new Term(TEXT_FIELD_NAME, "fox")), - new SynonymQuery(new Term(TEXT_FIELD_NAME, "dogs"), new Term(TEXT_FIELD_NAME, "dog")), + new SynonymQuery.Builder(TEXT_FIELD_NAME) + .addTerm(new Term(TEXT_FIELD_NAME, "dogs")) + .addTerm(new Term(TEXT_FIELD_NAME, "dog")) + .build(), new PrefixQuery(new Term(TEXT_FIELD_NAME, "red")) )); } diff --git a/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java index 6fbfa88a61b35..5512341eca213 100644 --- a/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java @@ -12,6 +12,10 @@ import org.apache.lucene.analysis.CannedBinaryTokenStream; import org.apache.lucene.analysis.MockSynonymAnalyzer; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.FuzzyQuery; @@ -22,10 +26,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.common.ParsingException; diff --git a/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java index 9d15aeb3ec5c5..d528964ec01b5 100644 --- a/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java @@ -41,7 +41,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBooleanSubQuery; import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.either; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.hasItems; import static org.hamcrest.CoreMatchers.instanceOf; @@ -197,34 +196,29 @@ public void testToQueryMultipleTermsBooleanQuery() throws Exception { public void testToQueryMultipleFieldsDisableDismax() throws Exception { Query query = multiMatchQuery("test").field(TEXT_FIELD_NAME).field(KEYWORD_FIELD_NAME).tieBreaker(1.0f) .toQuery(createSearchExecutionContext()); - assertThat(query, instanceOf(DisjunctionMaxQuery.class)); - DisjunctionMaxQuery dQuery = (DisjunctionMaxQuery) query; - assertThat(dQuery.getTieBreakerMultiplier(), equalTo(1.0f)); - assertThat(dQuery.getDisjuncts().size(), equalTo(2)); - assertThat(dQuery.getDisjuncts(), - hasItems(new TermQuery(new Term(TEXT_FIELD_NAME, "test")), new TermQuery(new Term(KEYWORD_FIELD_NAME, "test")))); + Query expected = new DisjunctionMaxQuery( + List.of(new TermQuery(new Term(TEXT_FIELD_NAME, "test")), new TermQuery(new Term(KEYWORD_FIELD_NAME, "test"))), + 1 + ); + assertEquals(expected, query); } public void testToQueryMultipleFieldsDisMaxQuery() throws Exception { Query query = multiMatchQuery("test").field(TEXT_FIELD_NAME).field(KEYWORD_FIELD_NAME).toQuery(createSearchExecutionContext()); - assertThat(query, instanceOf(DisjunctionMaxQuery.class)); - DisjunctionMaxQuery disMaxQuery = (DisjunctionMaxQuery) query; - assertThat(disMaxQuery.getTieBreakerMultiplier(), equalTo(0.0f)); - List disjuncts = disMaxQuery.getDisjuncts(); - assertThat(disjuncts.get(0), instanceOf(TermQuery.class)); - assertThat(((TermQuery) disjuncts.get(0)).getTerm(), equalTo(new Term(TEXT_FIELD_NAME, "test"))); - assertThat(disjuncts.get(1), instanceOf(TermQuery.class)); - assertThat(((TermQuery) disjuncts.get(1)).getTerm(), equalTo(new Term(KEYWORD_FIELD_NAME, "test"))); + Query expected = new DisjunctionMaxQuery( + List.of(new TermQuery(new Term(TEXT_FIELD_NAME, "test")), new TermQuery(new Term(KEYWORD_FIELD_NAME, "test"))), + 0 + ); + assertEquals(expected, query); } public void testToQueryFieldsWildcard() throws Exception { Query query = multiMatchQuery("test").field("mapped_str*").tieBreaker(1.0f).toQuery(createSearchExecutionContext()); - assertThat(query, instanceOf(DisjunctionMaxQuery.class)); - DisjunctionMaxQuery dQuery = (DisjunctionMaxQuery) query; - assertThat(dQuery.getTieBreakerMultiplier(), equalTo(1.0f)); - assertThat(dQuery.getDisjuncts().size(), equalTo(2)); - assertThat(dQuery.getDisjuncts(), - hasItems(new TermQuery(new Term(TEXT_FIELD_NAME, "test")), new TermQuery(new Term(KEYWORD_FIELD_NAME, "test")))); + Query expected = new DisjunctionMaxQuery( + List.of(new TermQuery(new Term(TEXT_FIELD_NAME, "test")), new TermQuery(new Term(KEYWORD_FIELD_NAME, "test"))), + 1 + ); + assertEquals(expected, query); } public void testToQueryFieldMissing() throws Exception { @@ -251,23 +245,15 @@ public void testToQueryBooleanPrefixMultipleFields() throws IOException { final MultiMatchQueryBuilder queryBuilder = new MultiMatchQueryBuilder("foo bar", TEXT_FIELD_NAME, KEYWORD_FIELD_NAME); queryBuilder.type(Type.BOOL_PREFIX); final Query query = queryBuilder.toQuery(createSearchExecutionContext()); - assertThat(query, instanceOf(DisjunctionMaxQuery.class)); - final DisjunctionMaxQuery disMaxQuery = (DisjunctionMaxQuery) query; - assertThat(disMaxQuery.getDisjuncts(), hasSize(2)); - for (Query disjunct : disMaxQuery.getDisjuncts()) { - if (disjunct instanceof BooleanQuery) { - final BooleanQuery firstDisjunct = (BooleanQuery) disjunct; - assertThat(firstDisjunct.clauses(), hasSize(2)); - assertThat(assertBooleanSubQuery(firstDisjunct, TermQuery.class, 0).getTerm(), - equalTo(new Term(TEXT_FIELD_NAME, "foo"))); - } else if (disjunct instanceof PrefixQuery) { - final PrefixQuery secondDisjunct = (PrefixQuery) disjunct; - assertThat(secondDisjunct.getPrefix(), equalTo(new Term(KEYWORD_FIELD_NAME, "foo bar"))); - } else { - throw new AssertionError(); - } - assertThat(disjunct, either(instanceOf(BooleanQuery.class)).or(instanceOf(PrefixQuery.class))); - } + Query expected = new DisjunctionMaxQuery(List.of( + new BooleanQuery.Builder() + .add(new TermQuery(new Term(TEXT_FIELD_NAME, "foo")), BooleanClause.Occur.SHOULD) + .add(new PrefixQuery(new Term(TEXT_FIELD_NAME, "bar")), BooleanClause.Occur.SHOULD) + .build(), + new PrefixQuery(new Term(KEYWORD_FIELD_NAME, "foo bar"))), + 1 + ); + assertEquals(expected, query); } } diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java index 9a57d718a40c0..d6c106aa1a9e3 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java @@ -34,9 +34,9 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.search.WildcardQuery; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanTermQuery; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.Automata; import org.apache.lucene.util.automaton.Automaton; @@ -489,29 +489,33 @@ public void testToQueryMultipleFieldsBooleanQuery() throws Exception { Query query = queryStringQuery("test").field(TEXT_FIELD_NAME) .field(KEYWORD_FIELD_NAME) .toQuery(createSearchExecutionContext()); - assertThat(query, instanceOf(DisjunctionMaxQuery.class)); - DisjunctionMaxQuery dQuery = (DisjunctionMaxQuery) query; - assertThat(dQuery.getDisjuncts().size(), equalTo(2)); - assertThat(dQuery.getDisjuncts(), - hasItems(new TermQuery(new Term(TEXT_FIELD_NAME, "test")), new TermQuery(new Term(KEYWORD_FIELD_NAME, "test")))); + Query expected = new DisjunctionMaxQuery(List.of( + new TermQuery(new Term(TEXT_FIELD_NAME, "test")), + new TermQuery(new Term(KEYWORD_FIELD_NAME, "test"))), + 0 + ); + assertEquals(expected, query); } public void testToQueryMultipleFieldsDisMaxQuery() throws Exception { Query query = queryStringQuery("test").field(TEXT_FIELD_NAME).field(KEYWORD_FIELD_NAME) .toQuery(createSearchExecutionContext()); - assertThat(query, instanceOf(DisjunctionMaxQuery.class)); - DisjunctionMaxQuery disMaxQuery = (DisjunctionMaxQuery) query; - assertThat(disMaxQuery.getDisjuncts(), - hasItems(new TermQuery(new Term(TEXT_FIELD_NAME, "test")), new TermQuery(new Term(KEYWORD_FIELD_NAME, "test")))); + Query expected = new DisjunctionMaxQuery(List.of( + new TermQuery(new Term(TEXT_FIELD_NAME, "test")), + new TermQuery(new Term(KEYWORD_FIELD_NAME, "test"))), + 0 + ); + assertEquals(expected, query); } public void testToQueryFieldsWildcard() throws Exception { Query query = queryStringQuery("test").field("mapped_str*").toQuery(createSearchExecutionContext()); - assertThat(query, instanceOf(DisjunctionMaxQuery.class)); - DisjunctionMaxQuery dQuery = (DisjunctionMaxQuery) query; - assertThat(dQuery.getDisjuncts().size(), equalTo(2)); - assertThat(dQuery.getDisjuncts(), - hasItems(new TermQuery(new Term(TEXT_FIELD_NAME, "test")), new TermQuery(new Term(KEYWORD_FIELD_NAME, "test")))); + Query expected = new DisjunctionMaxQuery(List.of( + new TermQuery(new Term(TEXT_FIELD_NAME, "test")), + new TermQuery(new Term(KEYWORD_FIELD_NAME, "test"))), + 0 + ); + assertEquals(expected, query); } /** @@ -532,11 +536,12 @@ public void testToQueryDisMaxQuery() throws Exception { Query query = queryStringQuery("test").field(TEXT_FIELD_NAME, 2.2f) .field(KEYWORD_FIELD_NAME) .toQuery(createSearchExecutionContext()); - assertThat(query, instanceOf(DisjunctionMaxQuery.class)); - DisjunctionMaxQuery disMaxQuery = (DisjunctionMaxQuery) query; - List disjuncts = disMaxQuery.getDisjuncts(); - assertTermOrBoostQuery(disjuncts.get(0), TEXT_FIELD_NAME, "test", 2.2f); - assertTermOrBoostQuery(disjuncts.get(1), KEYWORD_FIELD_NAME, "test", 1.0f); + Query expected = new DisjunctionMaxQuery(List.of( + new BoostQuery(new TermQuery(new Term(TEXT_FIELD_NAME, "test")), 2.2f), + new TermQuery(new Term(KEYWORD_FIELD_NAME, "test"))), + 0 + ); + assertEquals(expected, query); } public void testToQueryWildcardQuery() throws Exception { @@ -587,13 +592,19 @@ public void testToQueryWilcardQueryWithSynonyms() throws Exception { Query query = queryParser.parse("first foo-bar-foobar* last"); Query expectedQuery = new BooleanQuery.Builder() - .add(new BooleanClause(new SynonymQuery(new Term(TEXT_FIELD_NAME, "first"), - new Term(TEXT_FIELD_NAME, "first")), defaultOp)) + .add(new BooleanClause(new SynonymQuery.Builder(TEXT_FIELD_NAME) + .addTerm(new Term(TEXT_FIELD_NAME, "first")) + .addTerm(new Term(TEXT_FIELD_NAME, "first")) + .build(), defaultOp)) .add(new BooleanQuery.Builder() - .add(new BooleanClause(new SynonymQuery(new Term(TEXT_FIELD_NAME, "foo"), - new Term(TEXT_FIELD_NAME, "foo")), defaultOp)) - .add(new BooleanClause(new SynonymQuery(new Term(TEXT_FIELD_NAME, "bar"), - new Term(TEXT_FIELD_NAME, "bar")), defaultOp)) + .add(new BooleanClause(new SynonymQuery.Builder(TEXT_FIELD_NAME) + .addTerm(new Term(TEXT_FIELD_NAME, "foo")) + .addTerm(new Term(TEXT_FIELD_NAME, "foo")) + .build(), defaultOp)) + .add(new BooleanClause(new SynonymQuery.Builder(TEXT_FIELD_NAME) + .addTerm(new Term(TEXT_FIELD_NAME, "bar")) + .addTerm(new Term(TEXT_FIELD_NAME, "bar")) + .build(), defaultOp)) .add(new BooleanQuery.Builder() .add(new BooleanClause(new PrefixQuery(new Term(TEXT_FIELD_NAME, "foobar")), BooleanClause.Occur.SHOULD)) @@ -601,8 +612,10 @@ public void testToQueryWilcardQueryWithSynonyms() throws Exception { BooleanClause.Occur.SHOULD)) .build(), defaultOp) .build(), defaultOp) - .add(new BooleanClause(new SynonymQuery(new Term(TEXT_FIELD_NAME, "last"), - new Term(TEXT_FIELD_NAME, "last")), defaultOp)) + .add(new BooleanClause(new SynonymQuery.Builder(TEXT_FIELD_NAME) + .addTerm(new Term(TEXT_FIELD_NAME, "last")) + .addTerm(new Term(TEXT_FIELD_NAME, "last")) + .build(), defaultOp)) .build(); assertThat(query, Matchers.equalTo(expectedQuery)); } diff --git a/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java index 157cbcddf756a..b9fc4becb6159 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java @@ -11,6 +11,10 @@ import org.apache.lucene.analysis.MockSynonymAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; @@ -22,10 +26,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.util.TestUtil; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; @@ -461,13 +461,19 @@ public void testAnalyzerWildcardWithSynonyms() throws IOException { parser.setDefaultOperator(defaultOp); Query query = parser.parse("first foo-bar-foobar* last"); Query expectedQuery = new BooleanQuery.Builder() - .add(new BooleanClause(new SynonymQuery(new Term(TEXT_FIELD_NAME, "first"), - new Term(TEXT_FIELD_NAME, "first")), defaultOp)) + .add(new BooleanClause(new SynonymQuery.Builder(TEXT_FIELD_NAME) + .addTerm(new Term(TEXT_FIELD_NAME, "first")) + .addTerm(new Term(TEXT_FIELD_NAME, "first")) + .build(), defaultOp)) .add(new BooleanQuery.Builder() - .add(new BooleanClause(new SynonymQuery(new Term(TEXT_FIELD_NAME, "foo"), - new Term(TEXT_FIELD_NAME, "foo")), defaultOp)) - .add(new BooleanClause(new SynonymQuery(new Term(TEXT_FIELD_NAME, "bar"), - new Term(TEXT_FIELD_NAME, "bar")), defaultOp)) + .add(new BooleanClause(new SynonymQuery.Builder(TEXT_FIELD_NAME) + .addTerm(new Term(TEXT_FIELD_NAME, "foo")) + .addTerm(new Term(TEXT_FIELD_NAME, "foo")) + .build(), defaultOp)) + .add(new BooleanClause(new SynonymQuery.Builder(TEXT_FIELD_NAME) + .addTerm(new Term(TEXT_FIELD_NAME, "bar")) + .addTerm(new Term(TEXT_FIELD_NAME, "bar")) + .build(), defaultOp)) .add(new BooleanQuery.Builder() .add(new BooleanClause(new PrefixQuery(new Term(TEXT_FIELD_NAME, "foobar")), BooleanClause.Occur.SHOULD)) @@ -475,8 +481,10 @@ public void testAnalyzerWildcardWithSynonyms() throws IOException { BooleanClause.Occur.SHOULD)) .build(), defaultOp) .build(), defaultOp) - .add(new BooleanClause(new SynonymQuery(new Term(TEXT_FIELD_NAME, "last"), - new Term(TEXT_FIELD_NAME, "last")), defaultOp)) + .add(new BooleanClause(new SynonymQuery.Builder(TEXT_FIELD_NAME) + .addTerm(new Term(TEXT_FIELD_NAME, "last")) + .addTerm(new Term(TEXT_FIELD_NAME, "last")) + .build(), defaultOp)) .build(); assertThat(query, equalTo(expectedQuery)); } diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanContainingQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanContainingQueryBuilderTests.java index 8a7286691a8ad..0981699a2b4a7 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanContainingQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanContainingQueryBuilderTests.java @@ -8,8 +8,8 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanContainingQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanContainingQuery; import org.elasticsearch.common.ParsingException; import org.elasticsearch.test.AbstractQueryTestCase; diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanFirstQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanFirstQueryBuilderTests.java index a7c9f71120385..4d01b52e109fd 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanFirstQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanFirstQueryBuilderTests.java @@ -8,8 +8,8 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanFirstQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanFirstQuery; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanGapQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanGapQueryBuilderTests.java index 5999be5e81850..2bc4825521eb5 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanGapQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanGapQueryBuilderTests.java @@ -8,10 +8,10 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.elasticsearch.test.AbstractQueryTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java index ecddbb6d39131..2cffbc2c220a0 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java @@ -15,6 +15,10 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.queries.SpanMatchNoDocsQuery; +import org.apache.lucene.queries.spans.FieldMaskingSpanQuery; +import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.IndexSearcher; @@ -23,10 +27,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopTermsRewrite; -import org.apache.lucene.search.spans.FieldMaskingSpanQuery; -import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.store.Directory; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java index 89eea3703a726..ae6eed6151b5d 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java @@ -9,10 +9,10 @@ package org.elasticsearch.index.query; import org.apache.lucene.queries.SpanMatchNoDocsQuery; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.elasticsearch.common.ParsingException; import org.elasticsearch.test.AbstractQueryTestCase; diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanNotQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanNotQueryBuilderTests.java index 508cd191918c5..398af982d36bd 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanNotQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanNotQueryBuilderTests.java @@ -8,8 +8,8 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanNotQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanNotQuery; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java index 8931656be7ba0..a7a486d5c833e 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java @@ -8,9 +8,9 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanQuery; import org.elasticsearch.common.ParsingException; import org.elasticsearch.test.AbstractQueryTestCase; diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java index 87074e5c74dc4..dfcf124378aa6 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java @@ -10,9 +10,9 @@ import com.fasterxml.jackson.core.io.JsonStringEncoder; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.index.mapper.MappedFieldType; diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanWithinQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanWithinQueryBuilderTests.java index ccc9fea91da87..7ae668fe7f19b 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanWithinQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanWithinQueryBuilderTests.java @@ -8,8 +8,8 @@ package org.elasticsearch.index.query; +import org.apache.lucene.queries.spans.SpanWithinQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.spans.SpanWithinQuery; import org.elasticsearch.common.ParsingException; import org.elasticsearch.test.AbstractQueryTestCase; diff --git a/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java index b6ac9d53136eb..c97e2f2830a13 100644 --- a/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java @@ -18,7 +18,7 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.Term; -import org.apache.lucene.search.CoveringQuery; +import org.apache.lucene.sandbox.search.CoveringQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; diff --git a/server/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java b/server/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java index d6ed7f8eb573d..7d017ff996420 100644 --- a/server/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java +++ b/server/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java @@ -11,6 +11,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Weight; import org.elasticsearch.plugins.Plugin; @@ -50,5 +51,10 @@ public boolean equals(Object obj) { public int hashCode() { return classHash(); } + + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } } } diff --git a/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryParserTests.java b/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryParserTests.java index 85d4316f0fae2..f0071f6d677f3 100644 --- a/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryParserTests.java @@ -243,16 +243,16 @@ public void testMultiMatchCrossFieldsWithSynonyms() throws IOException { // check that synonym query is used for a single field Query parsedQuery = parser.parse(MultiMatchQueryBuilder.Type.CROSS_FIELDS, fieldNames, "dogs", null); - Term[] terms = new Term[2]; - terms[0] = new Term("name.first", "dog"); - terms[1] = new Term("name.first", "dogs"); - Query expectedQuery = new SynonymQuery(terms); + Query expectedQuery = new SynonymQuery.Builder("name.first") + .addTerm(new Term("name.first", "dog")) + .addTerm(new Term("name.first", "dogs")) + .build(); assertThat(parsedQuery, equalTo(expectedQuery)); // check that blended term query is used for multiple fields fieldNames.put("name.last", 1.0f); parsedQuery = parser.parse(MultiMatchQueryBuilder.Type.CROSS_FIELDS, fieldNames, "dogs", null); - terms = new Term[4]; + Term[] terms = new Term[4]; terms[0] = new Term("name.first", "dog"); terms[1] = new Term("name.first", "dogs"); terms[2] = new Term("name.last", "dog"); diff --git a/server/src/test/java/org/elasticsearch/index/similarity/ScriptedSimilarityTests.java b/server/src/test/java/org/elasticsearch/index/similarity/ScriptedSimilarityTests.java index 827c0bb9cdab8..c546dc3bcec37 100644 --- a/server/src/test/java/org/elasticsearch/index/similarity/ScriptedSimilarityTests.java +++ b/server/src/test/java/org/elasticsearch/index/similarity/ScriptedSimilarityTests.java @@ -50,8 +50,7 @@ public void testSameNormsAsBM25DiscountOverlaps() { private void doTestSameNormsAsBM25(boolean discountOverlaps) { ScriptedSimilarity sim1 = new ScriptedSimilarity("foobar", null, "foobaz", null, discountOverlaps); - BM25Similarity sim2 = new BM25Similarity(); - sim2.setDiscountOverlaps(discountOverlaps); + BM25Similarity sim2 = new BM25Similarity(discountOverlaps); for (int iter = 0; iter < 100; ++iter) { final int length = TestUtil.nextInt(random(), 1, 100); final int position = random().nextInt(length); diff --git a/server/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java b/server/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java index 8f20ad11c8f8d..e960cc5cb24c8 100644 --- a/server/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java @@ -12,10 +12,10 @@ import org.apache.lucene.search.TermStatistics; import org.apache.lucene.search.similarities.BooleanSimilarity; import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.search.similarity.LegacyBM25Similarity; import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.lucene.similarity.LegacyBM25Similarity; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; import org.hamcrest.Matchers; diff --git a/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java b/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java index f71cb691a9fd6..7c0245dd61b8b 100644 --- a/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java +++ b/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java @@ -20,7 +20,6 @@ import org.apache.lucene.search.similarities.LMJelinekMercerSimilarity; import org.apache.lucene.search.similarities.LambdaTTF; import org.apache.lucene.search.similarities.NormalizationH2; -import org.apache.lucene.search.similarity.LegacyBM25Similarity; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; @@ -30,6 +29,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.lucene.similarity.LegacyBM25Similarity; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; diff --git a/server/src/test/java/org/elasticsearch/index/store/StoreTests.java b/server/src/test/java/org/elasticsearch/index/store/StoreTests.java index 5492f34cdffe8..a8810a9b08774 100644 --- a/server/src/test/java/org/elasticsearch/index/store/StoreTests.java +++ b/server/src/test/java/org/elasticsearch/index/store/StoreTests.java @@ -16,7 +16,6 @@ import org.apache.lucene.document.TextField; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.index.IndexNotFoundException; @@ -210,10 +209,10 @@ public void testChecksumCorrupted() throws IOException { BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024)); output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length); } - output.writeInt(CodecUtil.FOOTER_MAGIC); - output.writeInt(0); + CodecUtil.writeBEInt(output, CodecUtil.FOOTER_MAGIC); + CodecUtil.writeBEInt(output, 0); String checksum = Store.digestToString(output.getChecksum()); - output.writeLong(output.getChecksum() + 1); // write a wrong checksum to the file + CodecUtil.writeBELong(output, output.getChecksum() + 1); // write a wrong checksum to the file output.close(); IndexInput indexInput = dir.openInput("foo.bar", IOContext.DEFAULT); @@ -459,8 +458,7 @@ public void assertDeleteContent(Store store, Directory dir) throws IOException { public static void assertConsistent(Store store, Store.MetadataSnapshot metadata) throws IOException { for (String file : store.directory().listAll()) { - if (IndexWriter.WRITE_LOCK_NAME.equals(file) == false && - IndexFileNames.OLD_SEGMENTS_GEN.equals(file) == false && file.startsWith("extra") == false) { + if (IndexWriter.WRITE_LOCK_NAME.equals(file) == false && file.startsWith("extra") == false) { assertTrue(file + " is not in the map: " + metadata.asMap().size() + " vs. " + store.directory().listAll().length, metadata.asMap().containsKey(file)); } else { diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java index 27f320300bab4..df3865d3d3fe1 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java @@ -12,9 +12,9 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.ReleasableBytesReference; -import org.elasticsearch.core.Tuple; -import org.elasticsearch.core.Releasable; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Tuple; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index 5c9851b2b7c98..1c3e248baee3f 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -11,6 +11,7 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; @@ -21,6 +22,7 @@ import org.apache.lucene.mockfile.FilterFileSystemProvider; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.ByteArrayDataOutput; +import org.apache.lucene.store.DataOutput; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase; @@ -1259,7 +1261,7 @@ public void testTranslogWriter() throws IOException { boolean opsHaveValidSequenceNumbers = randomBoolean(); for (int i = 0; i < numOps; i++) { byte[] bytes = new byte[4]; - ByteArrayDataOutput out = new ByteArrayDataOutput(bytes); + DataOutput out = EndiannessReverserUtil.wrapDataOutput(new ByteArrayDataOutput(bytes)); out.writeInt(i); long seqNo; do { @@ -1291,7 +1293,7 @@ public void testTranslogWriter() throws IOException { assertThat(reader.getCheckpoint().maxSeqNo, equalTo(maxSeqNo)); byte[] bytes = new byte[4]; - ByteArrayDataOutput out = new ByteArrayDataOutput(bytes); + DataOutput out = EndiannessReverserUtil.wrapDataOutput(new ByteArrayDataOutput(bytes)); out.writeInt(2048); writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), randomNonNegativeLong()); @@ -1469,7 +1471,7 @@ ChannelFactory getChannelFactory() { TranslogWriter writer = translog.getCurrent(); byte[] bytes = new byte[4]; - ByteArrayDataOutput out = new ByteArrayDataOutput(new byte[4]); + DataOutput out = EndiannessReverserUtil.wrapDataOutput(new ByteArrayDataOutput(new byte[4])); out.writeInt(1); writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), 1); assertThat(persistedSeqNos, empty()); @@ -1501,8 +1503,7 @@ public void testCloseIntoReader() throws IOException { final int numOps = randomIntBetween(8, 128); for (int i = 0; i < numOps; i++) { final byte[] bytes = new byte[4]; - final ByteArrayDataOutput out = new ByteArrayDataOutput(bytes); - out.reset(bytes); + final DataOutput out = EndiannessReverserUtil.wrapDataOutput(new ByteArrayDataOutput(bytes)); out.writeInt(i); writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), randomNonNegativeLong()); } diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java index 12cf61ac2e87f..3a18ee5a03893 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java @@ -12,7 +12,6 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocIdSetIterator; @@ -21,20 +20,20 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryCachingPolicy; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.ScorerSupplier; import org.apache.lucene.search.Weight; import org.apache.lucene.store.Directory; -import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.cache.query.QueryCacheStats; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; import java.io.IOException; -import java.util.Set; public class IndicesQueryCacheTests extends ESTestCase { @@ -56,6 +55,11 @@ public int hashCode() { return 31 * classHash() + id; } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + @Override public String toString(String field) { return "dummy"; @@ -352,11 +356,6 @@ private static class DummyWeight extends Weight { this.weight = weight; } - @Override - public void extractTerms(Set terms) { - weight.extractTerms(terms); - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return weight.explain(context, doc); diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java index 9e6e5730cf6ee..e6529bb18041a 100644 --- a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java @@ -15,7 +15,6 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.hunspell.Dictionary; -import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.store.Directory; import org.elasticsearch.Version; @@ -66,7 +65,6 @@ import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; public class AnalysisModuleTests extends ESTestCase { private final Settings emptyNodeSettings = Settings.builder() @@ -119,21 +117,6 @@ public void testSimpleConfigurationYaml() throws IOException { assertWarnings("Setting [version] on analysis component [custom7] has no effect and is deprecated"); } - public void testVersionedAnalyzers() throws Exception { - String yaml = "/org/elasticsearch/index/analysis/test1.yml"; - Version version = VersionUtils.randomVersion(random()); - Settings settings2 = Settings.builder() - .loadFromStream(yaml, getClass().getResourceAsStream(yaml), false) - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) - .put(IndexMetadata.SETTING_VERSION_CREATED, version) - .build(); - AnalysisRegistry newRegistry = getNewRegistry(settings2); - IndexAnalyzers indexAnalyzers = getIndexAnalyzers(newRegistry, settings2); - - assertThat(indexAnalyzers.get("custom7").analyzer(), is(instanceOf(StandardAnalyzer.class))); - assertWarnings("Setting [version] on analysis component [custom7] has no effect and is deprecated"); - } - private void testSimpleConfiguration(Settings settings) throws IOException { IndexAnalyzers indexAnalyzers = getIndexAnalyzers(settings); Analyzer analyzer = indexAnalyzers.get("custom1").analyzer(); diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java index 87dd75998319c..9fef6653ca533 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java @@ -7,6 +7,7 @@ */ package org.elasticsearch.indices.recovery; +import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.store.IndexOutput; import org.elasticsearch.common.util.set.Sets; @@ -32,11 +33,10 @@ public void testRenameTempFiles() throws IOException { "foo.bar", new StoreFileMetadata("foo.bar", 8 + CodecUtil.footerLength(), "9z51nw", MIN_SUPPORTED_LUCENE_VERSION.toString()), indexShard.store())) { - - indexOutput.writeInt(1); + EndiannessReverserUtil.wrapDataOutput(indexOutput).writeInt(1); IndexOutput openIndexOutput = multiFileWriter.getOpenIndexOutput("foo.bar"); assertSame(openIndexOutput, indexOutput); - openIndexOutput.writeInt(1); + EndiannessReverserUtil.wrapDataOutput(openIndexOutput).writeInt(1); CodecUtil.writeFooter(indexOutput); } diff --git a/server/src/test/java/org/apache/lucene/analysis/miscellaneous/DeDuplicatingTokenFilterTests.java b/server/src/test/java/org/elasticsearch/lucene/analysis/miscellaneous/DeDuplicatingTokenFilterTests.java similarity index 94% rename from server/src/test/java/org/apache/lucene/analysis/miscellaneous/DeDuplicatingTokenFilterTests.java rename to server/src/test/java/org/elasticsearch/lucene/analysis/miscellaneous/DeDuplicatingTokenFilterTests.java index 5b44c260cab97..013e6ee2ab8f6 100644 --- a/server/src/test/java/org/apache/lucene/analysis/miscellaneous/DeDuplicatingTokenFilterTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/analysis/miscellaneous/DeDuplicatingTokenFilterTests.java @@ -6,12 +6,15 @@ * Side Public License, v 1. */ -package org.apache.lucene.analysis.miscellaneous; +package org.elasticsearch.lucene.analysis.miscellaneous; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.miscellaneous.DeDuplicatingTokenFilter; +import org.apache.lucene.analysis.miscellaneous.DuplicateByteSequenceSpotter; +import org.apache.lucene.analysis.miscellaneous.DuplicateSequenceAttribute; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.elasticsearch.test.ESTestCase; diff --git a/server/src/test/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilterTests.java b/server/src/test/java/org/elasticsearch/lucene/analysis/miscellaneous/TruncateTokenFilterTests.java similarity index 94% rename from server/src/test/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilterTests.java rename to server/src/test/java/org/elasticsearch/lucene/analysis/miscellaneous/TruncateTokenFilterTests.java index 91732345fd311..c5ad79670c401 100644 --- a/server/src/test/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilterTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/analysis/miscellaneous/TruncateTokenFilterTests.java @@ -6,12 +6,13 @@ * Side Public License, v 1. */ -package org.apache.lucene.analysis.miscellaneous; +package org.elasticsearch.lucene.analysis.miscellaneous; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.miscellaneous.TruncateTokenFilter; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.elasticsearch.test.ESTestCase; diff --git a/server/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorSearchAfterTests.java b/server/src/test/java/org/elasticsearch/lucene/grouping/CollapsingTopDocsCollectorSearchAfterTests.java similarity index 99% rename from server/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorSearchAfterTests.java rename to server/src/test/java/org/elasticsearch/lucene/grouping/CollapsingTopDocsCollectorSearchAfterTests.java index 347b99c332171..dc6668be6c40a 100644 --- a/server/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorSearchAfterTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/grouping/CollapsingTopDocsCollectorSearchAfterTests.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.apache.lucene.grouping; +package org.elasticsearch.lucene.grouping; import org.apache.lucene.document.Document; import org.apache.lucene.document.NumericDocValuesField; @@ -245,7 +245,7 @@ public void add(Document doc, BytesRef value) { @Override public SortField sortField(boolean reversed) { - SortField sortField = new SortField("field", SortField.Type.STRING_VAL, reversed); + SortField sortField = new SortField("field", SortField.Type.STRING, reversed); sortField.setMissingValue(reversed ? SortField.STRING_FIRST : SortField.STRING_LAST); return sortField; } diff --git a/server/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorTests.java b/server/src/test/java/org/elasticsearch/lucene/grouping/CollapsingTopDocsCollectorTests.java similarity index 99% rename from server/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorTests.java rename to server/src/test/java/org/elasticsearch/lucene/grouping/CollapsingTopDocsCollectorTests.java index 13a9da4727820..3c21965107a6b 100644 --- a/server/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/grouping/CollapsingTopDocsCollectorTests.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.apache.lucene.grouping; +package org.elasticsearch.lucene.grouping; import org.apache.lucene.document.Document; import org.apache.lucene.document.NumericDocValuesField; @@ -341,7 +341,7 @@ public SortField sortField(boolean multivalued) { if (multivalued) { return new SortedSetSortField("field", false); } else { - return new SortField("field", SortField.Type.STRING_VAL); + return new SortField("field", SortField.Type.STRING); } } }; @@ -413,7 +413,7 @@ public void testEmptySortedSegment() throws Exception { MappedFieldType fieldType = new MockFieldMapper.FakeFieldType("group"); - Sort sort = new Sort(new SortField("group", SortField.Type.STRING_VAL)); + Sort sort = new Sort(new SortField("group", SortField.Type.STRING)); final CollapsingTopDocsCollector collapsingCollector = CollapsingTopDocsCollector.createKeyword("group", fieldType, sort, 10, null); diff --git a/server/src/test/java/org/apache/lucene/index/LazySoftDeletesDirectoryReaderWrapperTests.java b/server/src/test/java/org/elasticsearch/lucene/index/LazySoftDeletesDirectoryReaderWrapperTests.java similarity index 94% rename from server/src/test/java/org/apache/lucene/index/LazySoftDeletesDirectoryReaderWrapperTests.java rename to server/src/test/java/org/elasticsearch/lucene/index/LazySoftDeletesDirectoryReaderWrapperTests.java index 59dbc08f72084..ece103a1b8e36 100644 --- a/server/src/test/java/org/apache/lucene/index/LazySoftDeletesDirectoryReaderWrapperTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/index/LazySoftDeletesDirectoryReaderWrapperTests.java @@ -6,12 +6,20 @@ * Side Public License, v 1. */ -package org.apache.lucene.index; +package org.elasticsearch.lucene.index; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LazySoftDeletesDirectoryReaderWrapper; +import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.index.SoftDeletesRetentionMergePolicy; +import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.TermQuery; diff --git a/server/src/test/java/org/apache/lucene/index/ShuffleForcedMergePolicyTests.java b/server/src/test/java/org/elasticsearch/lucene/index/ShuffleForcedMergePolicyTests.java similarity index 86% rename from server/src/test/java/org/apache/lucene/index/ShuffleForcedMergePolicyTests.java rename to server/src/test/java/org/elasticsearch/lucene/index/ShuffleForcedMergePolicyTests.java index ec1d03394fe25..debae061a0f34 100644 --- a/server/src/test/java/org/apache/lucene/index/ShuffleForcedMergePolicyTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/index/ShuffleForcedMergePolicyTests.java @@ -6,12 +6,21 @@ * Side Public License, v 1. */ -package org.apache.lucene.index; +package org.elasticsearch.lucene.index; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.StringField; +import org.apache.lucene.index.BaseMergePolicyTestCase; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.MergePolicy; +import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.index.ShuffleForcedMergePolicy; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.store.Directory; diff --git a/server/src/test/java/org/apache/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java b/server/src/test/java/org/elasticsearch/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java similarity index 98% rename from server/src/test/java/org/apache/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java rename to server/src/test/java/org/elasticsearch/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java index 0a3c858445e6c..ed78d8af92fbe 100644 --- a/server/src/test/java/org/apache/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Field; diff --git a/server/src/test/java/org/apache/lucene/queries/BinaryDocValuesRangeQueryTests.java b/server/src/test/java/org/elasticsearch/lucene/queries/BinaryDocValuesRangeQueryTests.java similarity index 99% rename from server/src/test/java/org/apache/lucene/queries/BinaryDocValuesRangeQueryTests.java rename to server/src/test/java/org/elasticsearch/lucene/queries/BinaryDocValuesRangeQueryTests.java index e799782e6cd8e..26a54e0b5175a 100644 --- a/server/src/test/java/org/apache/lucene/queries/BinaryDocValuesRangeQueryTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/BinaryDocValuesRangeQueryTests.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Document; diff --git a/server/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java b/server/src/test/java/org/elasticsearch/lucene/queries/BlendedTermQueryTests.java similarity index 97% rename from server/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java rename to server/src/test/java/org/elasticsearch/lucene/queries/BlendedTermQueryTests.java index 64528e25a9acb..3489316823285 100644 --- a/server/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/BlendedTermQueryTests.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; @@ -15,17 +15,17 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermStates; +import org.apache.lucene.queries.BlendedTermQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryUtils; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.similarities.BM25Similarity; @@ -211,8 +211,7 @@ public void testExtractTerms() throws IOException { BlendedTermQuery blendedTermQuery = BlendedTermQuery.dismaxBlendedQuery(terms.toArray(new Term[0]), random().nextFloat()); Set extracted = new HashSet<>(); - IndexSearcher searcher = new IndexSearcher(new MultiReader()); - searcher.createWeight(searcher.rewrite(blendedTermQuery), ScoreMode.COMPLETE_NO_SCORES, 1f).extractTerms(extracted); + blendedTermQuery.visit(QueryVisitor.termCollector(extracted)); assertThat(extracted.size(), equalTo(terms.size())); assertThat(extracted, containsInAnyOrder(terms.toArray(new Term[0]))); } diff --git a/server/src/test/java/org/apache/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java b/server/src/test/java/org/elasticsearch/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java similarity index 98% rename from server/src/test/java/org/apache/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java rename to server/src/test/java/org/elasticsearch/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java index ebecf6d36feb7..0b20c42d052f6 100644 --- a/server/src/test/java/org/apache/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.elasticsearch.index.mapper.RangeType; diff --git a/server/src/test/java/org/apache/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java b/server/src/test/java/org/elasticsearch/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java similarity index 98% rename from server/src/test/java/org/apache/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java rename to server/src/test/java/org/elasticsearch/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java index 681d5434b5618..907cde732c25c 100644 --- a/server/src/test/java/org/apache/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.elasticsearch.index.mapper.RangeType; diff --git a/server/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java b/server/src/test/java/org/elasticsearch/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java similarity index 78% rename from server/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java rename to server/src/test/java/org/elasticsearch/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java index 3123723b49659..995953cb3b5a6 100644 --- a/server/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java @@ -5,10 +5,9 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.apache.lucene.document.InetAddressPoint; -import org.apache.lucene.util.FutureArrays; import org.elasticsearch.index.mapper.RangeType; import java.net.InetAddress; @@ -33,7 +32,7 @@ protected Range nextRange(int dimensions) throws Exception { byte[] bMin = InetAddressPoint.encode(min); InetAddress max = nextInetaddress(); byte[] bMax = InetAddressPoint.encode(max); - if (FutureArrays.compareUnsigned(bMin, 0, bMin.length, bMax, 0, bMin.length) > 0) { + if (Arrays.compareUnsigned(bMin, 0, bMin.length, bMax, 0, bMin.length) > 0) { return new IpRange(max, min); } return new IpRange(min, max); @@ -80,7 +79,7 @@ protected void setMin(int dim, Object val) { InetAddress v = (InetAddress)val; byte[] e = InetAddressPoint.encode(v); - if (FutureArrays.compareUnsigned(min, 0, e.length, e, 0, e.length) < 0) { + if (Arrays.compareUnsigned(min, 0, e.length, e, 0, e.length) < 0) { max = e; maxAddress = v; } else { @@ -100,7 +99,7 @@ protected void setMax(int dim, Object val) { InetAddress v = (InetAddress)val; byte[] e = InetAddressPoint.encode(v); - if (FutureArrays.compareUnsigned(max, 0, e.length, e, 0, e.length) > 0) { + if (Arrays.compareUnsigned(max, 0, e.length, e, 0, e.length) > 0) { min = e; minAddress = v; } else { @@ -112,22 +111,22 @@ protected void setMax(int dim, Object val) { @Override protected boolean isDisjoint(Range o) { IpRange other = (IpRange) o; - return FutureArrays.compareUnsigned(min, 0, min.length, other.max, 0, min.length) > 0 || - FutureArrays.compareUnsigned(max, 0, max.length, other.min, 0, max.length) < 0; + return Arrays.compareUnsigned(min, 0, min.length, other.max, 0, min.length) > 0 || + Arrays.compareUnsigned(max, 0, max.length, other.min, 0, max.length) < 0; } @Override protected boolean isWithin(Range o) { IpRange other = (IpRange)o; - return FutureArrays.compareUnsigned(min, 0, min.length, other.min, 0, min.length) >= 0 && - FutureArrays.compareUnsigned(max, 0, max.length, other.max, 0, max.length) <= 0; + return Arrays.compareUnsigned(min, 0, min.length, other.min, 0, min.length) >= 0 && + Arrays.compareUnsigned(max, 0, max.length, other.max, 0, max.length) <= 0; } @Override protected boolean contains(Range o) { IpRange other = (IpRange)o; - return FutureArrays.compareUnsigned(min, 0, min.length, other.min, 0, min.length) <= 0 && - FutureArrays.compareUnsigned(max, 0, max.length, other.max, 0, max.length) >= 0; + return Arrays.compareUnsigned(min, 0, min.length, other.min, 0, min.length) <= 0 && + Arrays.compareUnsigned(max, 0, max.length, other.max, 0, max.length) >= 0; } } diff --git a/server/src/test/java/org/apache/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java b/server/src/test/java/org/elasticsearch/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java similarity index 98% rename from server/src/test/java/org/apache/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java rename to server/src/test/java/org/elasticsearch/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java index 72fd223c52501..6861bd7284128 100644 --- a/server/src/test/java/org/apache/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.apache.lucene.util.TestUtil; import org.elasticsearch.index.mapper.RangeType; diff --git a/server/src/test/java/org/apache/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java b/server/src/test/java/org/elasticsearch/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java similarity index 98% rename from server/src/test/java/org/apache/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java rename to server/src/test/java/org/elasticsearch/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java index 2f75d49bd82a1..87116153b11bd 100644 --- a/server/src/test/java/org/apache/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.apache.lucene.util.TestUtil; import org.elasticsearch.index.mapper.RangeType; diff --git a/server/src/test/java/org/apache/lucene/queries/MinDocQueryTests.java b/server/src/test/java/org/elasticsearch/lucene/queries/MinDocQueryTests.java similarity index 98% rename from server/src/test/java/org/apache/lucene/queries/MinDocQueryTests.java rename to server/src/test/java/org/elasticsearch/lucene/queries/MinDocQueryTests.java index c3290a4dc66b7..7f58ed9fe56d4 100644 --- a/server/src/test/java/org/apache/lucene/queries/MinDocQueryTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/MinDocQueryTests.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexReader; diff --git a/server/src/test/java/org/apache/lucene/queries/SearchAfterSortedDocQueryTests.java b/server/src/test/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQueryTests.java similarity index 99% rename from server/src/test/java/org/apache/lucene/queries/SearchAfterSortedDocQueryTests.java rename to server/src/test/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQueryTests.java index 5859f3fa35d73..f056b5b1308f1 100644 --- a/server/src/test/java/org/apache/lucene/queries/SearchAfterSortedDocQueryTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQueryTests.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.apache.lucene.document.Document; import org.apache.lucene.document.SortedDocValuesField; diff --git a/server/src/test/java/org/apache/lucene/queries/SpanMatchNoDocsQueryTests.java b/server/src/test/java/org/elasticsearch/lucene/queries/SpanMatchNoDocsQueryTests.java similarity index 92% rename from server/src/test/java/org/apache/lucene/queries/SpanMatchNoDocsQueryTests.java rename to server/src/test/java/org/elasticsearch/lucene/queries/SpanMatchNoDocsQueryTests.java index fed0f4408ea0d..2fef323146043 100644 --- a/server/src/test/java/org/apache/lucene/queries/SpanMatchNoDocsQueryTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/queries/SpanMatchNoDocsQueryTests.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.apache.lucene.queries; +package org.elasticsearch.lucene.queries; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; @@ -16,14 +16,15 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.SpanMatchNoDocsQuery; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanOrQuery; +import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryUtils; import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanOrQuery; -import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.store.Directory; import org.elasticsearch.test.ESTestCase; diff --git a/server/src/test/java/org/apache/lucene/search/uhighlight/BoundedBreakIteratorScannerTests.java b/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/BoundedBreakIteratorScannerTests.java similarity index 97% rename from server/src/test/java/org/apache/lucene/search/uhighlight/BoundedBreakIteratorScannerTests.java rename to server/src/test/java/org/elasticsearch/lucene/search/uhighlight/BoundedBreakIteratorScannerTests.java index 26cc2c6a5a01f..a9b34e15542ce 100644 --- a/server/src/test/java/org/apache/lucene/search/uhighlight/BoundedBreakIteratorScannerTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/BoundedBreakIteratorScannerTests.java @@ -6,8 +6,9 @@ * Side Public License, v 1. */ -package org.apache.lucene.search.uhighlight; +package org.elasticsearch.lucene.search.uhighlight; +import org.apache.lucene.search.uhighlight.BoundedBreakIteratorScanner; import org.elasticsearch.test.ESTestCase; import java.text.BreakIterator; diff --git a/server/src/test/java/org/apache/lucene/search/uhighlight/CustomPassageFormatterTests.java b/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomPassageFormatterTests.java similarity index 94% rename from server/src/test/java/org/apache/lucene/search/uhighlight/CustomPassageFormatterTests.java rename to server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomPassageFormatterTests.java index 6050c71638770..004ebd4d6ce82 100644 --- a/server/src/test/java/org/apache/lucene/search/uhighlight/CustomPassageFormatterTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomPassageFormatterTests.java @@ -6,10 +6,13 @@ * Side Public License, v 1. */ -package org.apache.lucene.search.uhighlight; +package org.elasticsearch.lucene.search.uhighlight; import org.apache.lucene.search.highlight.DefaultEncoder; import org.apache.lucene.search.highlight.SimpleHTMLEncoder; +import org.apache.lucene.search.uhighlight.CustomPassageFormatter; +import org.apache.lucene.search.uhighlight.Passage; +import org.apache.lucene.search.uhighlight.Snippet; import org.apache.lucene.util.BytesRef; import org.elasticsearch.test.ESTestCase; diff --git a/server/src/test/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java b/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java similarity index 97% rename from server/src/test/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java rename to server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java index 16bc9792ffe48..7120f40f5ac14 100644 --- a/server/src/test/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.apache.lucene.search.uhighlight; +package org.elasticsearch.lucene.search.uhighlight; import static org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter.MULTIVAL_SEP_CHAR; import static org.hamcrest.CoreMatchers.equalTo; @@ -38,6 +38,11 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.highlight.DefaultEncoder; +import org.apache.lucene.search.uhighlight.BoundedBreakIteratorScanner; +import org.apache.lucene.search.uhighlight.CustomPassageFormatter; +import org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter; +import org.apache.lucene.search.uhighlight.Snippet; +import org.apache.lucene.search.uhighlight.UnifiedHighlighter; import org.apache.lucene.store.Directory; import org.elasticsearch.common.Strings; import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery; diff --git a/server/src/test/java/org/apache/lucene/util/CombinedBitSetTests.java b/server/src/test/java/org/elasticsearch/lucene/util/CombinedBitSetTests.java similarity index 94% rename from server/src/test/java/org/apache/lucene/util/CombinedBitSetTests.java rename to server/src/test/java/org/elasticsearch/lucene/util/CombinedBitSetTests.java index 93f8e4b115cd9..ad3a94d70c9e1 100644 --- a/server/src/test/java/org/apache/lucene/util/CombinedBitSetTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/util/CombinedBitSetTests.java @@ -6,9 +6,13 @@ * Side Public License, v 1. */ -package org.apache.lucene.util; +package org.elasticsearch.lucene.util; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BitSet; +import org.apache.lucene.util.CombinedBitSet; +import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.SparseFixedBitSet; import org.elasticsearch.test.ESTestCase; public class CombinedBitSetTests extends ESTestCase { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java index 82f45e005ac75..23b0ace7e6168 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java @@ -299,11 +299,11 @@ public void testUnmappedFieldWithHistogram() throws Exception { final String mappedFieldName = "price"; dataset.addAll( Arrays.asList( - createDocument(mappedFieldName, 103L), - createDocument(mappedFieldName, 51L), - createDocument(mappedFieldName, 56L), - createDocument(mappedFieldName, 105L), - createDocument(mappedFieldName, 25L) + createDocument(mappedFieldName, 103), + createDocument(mappedFieldName, 51), + createDocument(mappedFieldName, 56), + createDocument(mappedFieldName, 105), + createDocument(mappedFieldName, 25) ) ); @@ -1316,8 +1316,8 @@ public void testMissingTermBucket() throws Exception { public void testMissingHistogramBucket() throws Exception { List>> dataset = Arrays.asList( - createDocument("const", 1, "long", 1), - createDocument("const", 1, "long", 2), + createDocument("const", 1, "long", 1L), + createDocument("const", 1, "long", 2L), createDocument("const", 1, "keyword", "a") ); @@ -1462,8 +1462,8 @@ public void testMissingTermBucketAfterKey() throws Exception { public void testMissingHistogramBucketAfterKey() throws Exception { List>> dataset = Arrays.asList( - createDocument("const", 1, "long", 1), - createDocument("const", 1, "long", 2), + createDocument("const", 1, "long", 1L), + createDocument("const", 1, "long", 2L), createDocument("const", 1, "keyword", "a"), createDocument("const", 1, "keyword", "b") ); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java index 23039bd599b01..5c6d8606f6e80 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java @@ -33,7 +33,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.FutureArrays; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.Tuple; @@ -385,7 +384,7 @@ public void visit(int docID, byte[] packedValue) { @Override public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { - if (FutureArrays.equals(maxPackedValue, 0, numBytes, maxValue, 0, numBytes)) { + if (Arrays.equals(maxPackedValue, 0, numBytes, maxValue, 0, numBytes)) { return PointValues.Relation.CELL_CROSSES_QUERY; } return PointValues.Relation.CELL_OUTSIDE_QUERY; diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index 548c47a4bbd7c..cd0921c64bed2 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -912,7 +912,7 @@ public void testTooManyUnmappedFieldWildcardPattern() throws IOException { XContentBuilder source = XContentFactory.jsonBuilder().startObject().field("a", "foo").endObject(); - List fieldAndFormatList = new ArrayList<>(10_000); + List fieldAndFormatList = new ArrayList<>(8_000); for (int i = 0; i < 8000; i++) { fieldAndFormatList.add(new FieldAndFormat(randomAlphaOfLength(150) + "*", null, true)); } diff --git a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java index 1d5cb729a96ca..f563f1eae9927 100644 --- a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java @@ -34,6 +34,7 @@ import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.Scorable; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -392,11 +393,6 @@ private static class CreateScorerOnceWeight extends Weight { this.weight = weight; } - @Override - public void extractTerms(Set terms) { - weight.extractTerms(terms); - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return weight.explain(context, doc); @@ -457,5 +453,10 @@ public boolean equals(Object obj) { public int hashCode() { return 31 * classHash() + query.hashCode(); } + + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } } } diff --git a/server/src/test/java/org/elasticsearch/search/lookup/LeafStoredFieldsLookupTests.java b/server/src/test/java/org/elasticsearch/search/lookup/LeafStoredFieldsLookupTests.java index 536b649e8fd2b..b0fe09e3ec697 100644 --- a/server/src/test/java/org/elasticsearch/search/lookup/LeafStoredFieldsLookupTests.java +++ b/server/src/test/java/org/elasticsearch/search/lookup/LeafStoredFieldsLookupTests.java @@ -10,6 +10,7 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.VectorSimilarityFunction; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.test.ESTestCase; import org.junit.Before; @@ -35,7 +36,7 @@ public void setUp() throws Exception { (Double) invocation.getArguments()[0] + 10); FieldInfo mockFieldInfo = new FieldInfo("field", 1, false, false, true, - IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, false); + IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, 0, VectorSimilarityFunction.EUCLIDEAN, false); fieldsLookup = new LeafStoredFieldsLookup(field -> field.equals("field") || field.equals("alias") ? fieldType : null, (doc, visitor) -> visitor.doubleField(mockFieldInfo, 2.718)); diff --git a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java index 680a749c6fcf3..344f7885c76a9 100644 --- a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java +++ b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java @@ -23,6 +23,7 @@ import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryCachingPolicy; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.RandomApproximationQuery; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -44,7 +45,6 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.Set; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -165,9 +165,11 @@ public void testUseIndexStats() throws IOException { QueryProfiler profiler = new QueryProfiler(); searcher.setProfiler(profiler); Query query = new TermQuery(new Term("foo", "bar")); - searcher.count(query); // will use index stats + searcher.count(query); // will use index stats - builds weight but never builds scorer List results = profiler.getTree(); - assertEquals(0, results.size()); + assertEquals(1, results.size()); + ProfileResult result = results.get(0); + assertEquals(0, (long) result.getTimeBreakdown().get("build_scorer_count")); long rewriteTime = profiler.getRewriteTime(); assertThat(rewriteTime, greaterThan(0L)); @@ -230,14 +232,14 @@ public int hashCode() { return 0; } + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new Weight(this) { - @Override - public void extractTerms(Set terms) { - throw new UnsupportedOperationException(); - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { throw new UnsupportedOperationException(); diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java index c741dd64d0bc3..75190cf35485e 100644 --- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java @@ -26,7 +26,8 @@ import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; -import org.apache.lucene.queries.MinDocQuery; +import org.apache.lucene.queries.spans.SpanNearQuery; +import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Collector; @@ -53,8 +54,6 @@ import org.apache.lucene.search.Weight; import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.search.join.ScoreMode; -import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; @@ -67,6 +66,7 @@ import org.elasticsearch.index.search.ESToParentBlockJoinQuery; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardTestCase; +import org.elasticsearch.lucene.queries.MinDocQuery; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.search.internal.ScrollContext; @@ -711,7 +711,7 @@ public void testNumericSortOptimization() throws Exception { searchContext.trackTotalHitsUpTo(10); searchContext.setSize(10); QueryPhase.executeInternal(searchContext); - assertTrue(searchContext.sort().sort.getSort()[0].getCanUsePoints()); + assertTrue(searchContext.sort().sort.getSort()[0].getOptimizeSortWithPoints()); assertSortResults(searchContext.queryResult().topDocs().topDocs, numDocs, false); } @@ -729,7 +729,7 @@ public void testNumericSortOptimization() throws Exception { searchContext.trackTotalHitsUpTo(10); searchContext.setSize(10); QueryPhase.executeInternal(searchContext); - assertTrue(searchContext.sort().sort.getSort()[0].getCanUsePoints()); + assertTrue(searchContext.sort().sort.getSort()[0].getOptimizeSortWithPoints()); final TopDocs topDocs = searchContext.queryResult().topDocs().topDocs; long firstResult = (long) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]; assertThat(firstResult, greaterThan(afterValue)); @@ -746,7 +746,7 @@ public void testNumericSortOptimization() throws Exception { searchContext.trackTotalHitsUpTo(10); searchContext.setSize(10); QueryPhase.executeInternal(searchContext); - assertTrue(searchContext.sort().sort.getSort()[0].getCanUsePoints()); + assertTrue(searchContext.sort().sort.getSort()[0].getOptimizeSortWithPoints()); assertSortResults(searchContext.queryResult().topDocs().topDocs, numDocs, true); } @@ -760,7 +760,7 @@ public void testNumericSortOptimization() throws Exception { searchContext.trackTotalHitsUpTo(10); searchContext.setSize(10); QueryPhase.executeInternal(searchContext); - assertTrue(searchContext.sort().sort.getSort()[0].getCanUsePoints()); + assertTrue(searchContext.sort().sort.getSort()[0].getOptimizeSortWithPoints()); assertSortResults(searchContext.queryResult().topDocs().topDocs, numDocs, false); } @@ -774,7 +774,7 @@ public void testNumericSortOptimization() throws Exception { searchContext.trackTotalHitsUpTo(10); searchContext.setSize(10); QueryPhase.executeInternal(searchContext); - assertTrue(searchContext.sort().sort.getSort()[0].getCanUsePoints()); + assertTrue(searchContext.sort().sort.getSort()[0].getOptimizeSortWithPoints()); assertSortResults(searchContext.queryResult().topDocs().topDocs, (long) numDocs, true); } @@ -789,7 +789,7 @@ public void testNumericSortOptimization() throws Exception { searchContext.from(5); searchContext.setSize(0); QueryPhase.executeInternal(searchContext); - assertTrue(searchContext.sort().sort.getSort()[0].getCanUsePoints()); + assertTrue(searchContext.sort().sort.getSort()[0].getOptimizeSortWithPoints()); assertSortResults(searchContext.queryResult().topDocs().topDocs, (long) numDocs, false); } diff --git a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java index 0996d9d8e3c8d..aab44654c83fc 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java @@ -13,7 +13,6 @@ import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; import org.apache.lucene.document.FloatPoint; -import org.apache.lucene.document.HalfFloatPoint; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.SortedNumericDocValuesField; @@ -21,6 +20,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; +import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.search.AssertingIndexSearcher; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.SortField; diff --git a/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java index 21de821e2592e..168bf10a8591a 100644 --- a/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java @@ -8,8 +8,8 @@ package org.elasticsearch.indices.analysis; -import org.apache.lucene.analysis.util.TokenFilterFactory; -import org.apache.lucene.analysis.util.TokenizerFactory; +import org.apache.lucene.analysis.TokenFilterFactory; +import org.apache.lucene.analysis.TokenizerFactory; import org.elasticsearch.index.analysis.HunspellTokenFilterFactory; import org.elasticsearch.index.analysis.ShingleTokenFilterFactory; import org.elasticsearch.index.analysis.StandardTokenizerFactory; @@ -111,6 +111,7 @@ public abstract class AnalysisFactoryTestCase extends ESTestCase { entry("ngram", MovedToAnalysisCommon.class), entry("norwegianlightstem", MovedToAnalysisCommon.class), entry("norwegianminimalstem", MovedToAnalysisCommon.class), + entry("norwegiannormalization", MovedToAnalysisCommon.class), entry("patterncapturegroup", MovedToAnalysisCommon.class), entry("patternreplace", MovedToAnalysisCommon.class), entry("persiannormalization", MovedToAnalysisCommon.class), @@ -132,8 +133,11 @@ public abstract class AnalysisFactoryTestCase extends ESTestCase { entry("stemmeroverride", MovedToAnalysisCommon.class), entry("stop", StopTokenFilterFactory.class), entry("swedishlightstem", MovedToAnalysisCommon.class), + entry("swedishminimalstem", MovedToAnalysisCommon.class), entry("synonym", MovedToAnalysisCommon.class), entry("synonymgraph", MovedToAnalysisCommon.class), + entry("telugunormalization", MovedToAnalysisCommon.class), + entry("telugustem", MovedToAnalysisCommon.class), entry("trim", MovedToAnalysisCommon.class), entry("truncate", MovedToAnalysisCommon.class), entry("turkishlowercase", MovedToAnalysisCommon.class), @@ -186,10 +190,9 @@ public abstract class AnalysisFactoryTestCase extends ESTestCase { entry("delimitedboost", Void.class), // LUCENE-9574 entry("dropifflagged", Void.class), + entry("japanesecompletion", Void.class), // LUCENE-9575 - entry("patterntyping", Void.class), - entry("telugustem", Void.class), - entry("telugunormalization", Void.class)); + entry("patterntyping", Void.class)); static final Map> KNOWN_CHARFILTERS = Map.of( @@ -264,7 +267,7 @@ public Map> getPreConfiguredCharFilters() { public void testTokenizers() { Set missing = new TreeSet(); - missing.addAll(org.apache.lucene.analysis.util.TokenizerFactory.availableTokenizers() + missing.addAll(org.apache.lucene.analysis.TokenizerFactory.availableTokenizers() .stream().map(key -> key.toLowerCase(Locale.ROOT)).collect(Collectors.toSet())); missing.removeAll(getTokenizers().keySet()); assertTrue("new tokenizers found, please update KNOWN_TOKENIZERS: " + missing.toString(), missing.isEmpty()); @@ -272,7 +275,7 @@ public void testTokenizers() { public void testCharFilters() { Set missing = new TreeSet(); - missing.addAll(org.apache.lucene.analysis.util.CharFilterFactory.availableCharFilters() + missing.addAll(org.apache.lucene.analysis.CharFilterFactory.availableCharFilters() .stream().map(key -> key.toLowerCase(Locale.ROOT)).collect(Collectors.toSet())); missing.removeAll(getCharFilters().keySet()); assertTrue("new charfilters found, please update KNOWN_CHARFILTERS: " + missing.toString(), missing.isEmpty()); @@ -280,7 +283,7 @@ public void testCharFilters() { public void testTokenFilters() { Set missing = new TreeSet(); - missing.addAll(org.apache.lucene.analysis.util.TokenFilterFactory.availableTokenFilters() + missing.addAll(org.apache.lucene.analysis.TokenFilterFactory.availableTokenFilters() .stream().map(key -> key.toLowerCase(Locale.ROOT)).collect(Collectors.toSet())); missing.removeAll(getTokenFilters().keySet()); assertTrue("new tokenfilters found, please update KNOWN_TOKENFILTERS: " + missing.toString(), missing.isEmpty()); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 0a0fe066677a4..9c8e472ad7bb8 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -10,7 +10,6 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Document; -import org.apache.lucene.document.HalfFloatPoint; import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.document.LatLonDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; @@ -24,6 +23,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.search.AssertingIndexSearcher; import org.apache.lucene.search.Collector; import org.apache.lucene.search.IndexSearcher; diff --git a/test/framework/src/main/java/org/elasticsearch/test/CorruptionUtils.java b/test/framework/src/main/java/org/elasticsearch/test/CorruptionUtils.java index e892278ca31d0..26b0cfc177f81 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/CorruptionUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/CorruptionUtils.java @@ -80,7 +80,7 @@ public static void corruptFile(Random random, Path... files) throws IOException input.seek(input.length() - CodecUtil.footerLength()); checksumAfterCorruption = input.getChecksum(); input.seek(input.length() - 8); - actualChecksumAfterCorruption = input.readLong(); + actualChecksumAfterCorruption = CodecUtil.readBELong(input); } // we need to add assumptions here that the checksums actually really don't match there is a small chance to get collisions // in the checksum which is ok though.... diff --git a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/BlockingQueryBuilder.java b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/BlockingQueryBuilder.java index 679654a67d2b8..7fb330e40e450 100644 --- a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/BlockingQueryBuilder.java +++ b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/BlockingQueryBuilder.java @@ -8,6 +8,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Weight; import org.elasticsearch.common.io.stream.StreamInput; @@ -103,6 +104,11 @@ public boolean equals(Object obj) { public int hashCode() { return 0; } + + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } }; } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/ThrowingQueryBuilder.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/ThrowingQueryBuilder.java index b7d2d8e422ea8..04ce3d4a7c12a 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/ThrowingQueryBuilder.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/ThrowingQueryBuilder.java @@ -8,6 +8,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Weight; import org.elasticsearch.common.io.stream.StreamInput; @@ -82,6 +83,11 @@ public boolean equals(Object obj) { public int hashCode() { return 0; } + + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } }; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java index 63ac440d5b4ec..a53ab745a4c89 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java @@ -23,6 +23,8 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.Terms; +import org.apache.lucene.index.VectorValues; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; @@ -213,6 +215,16 @@ public NumericDocValues getNormValues(String field) { throw new UnsupportedOperationException(); } + @Override + public VectorValues getVectorValues(String field) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs) throws IOException { + throw new UnsupportedOperationException(); + } + @Override public FieldInfos getFieldInfos() { return fieldInfos; @@ -238,7 +250,7 @@ public LeafMetaData getMetaData() { } @Override - public Fields getTermVectors(int docID) { + public Fields getTermVectors(int docId) { throw new UnsupportedOperationException(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SeqIdGeneratingFilterReader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SeqIdGeneratingFilterReader.java index 917b35afd5d75..01a324c105cdf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SeqIdGeneratingFilterReader.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SeqIdGeneratingFilterReader.java @@ -161,7 +161,7 @@ public Terms terms(String field) { @Override public PointValues getPointValues(String field) { - throw new UnsupportedOperationException("_source only indices can't be searched or filtered"); + return null; } }; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java index 6d3dfcdf81bf4..30f047a0e9a94 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.index.SoftDeletesDirectoryReaderWrapper; import org.apache.lucene.index.StandardDirectoryReader; +import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -61,9 +62,9 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; -import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.FIELDS_EXTENSION; -import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.INDEX_EXTENSION; -import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.META_EXTENSION; +import static org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsWriter.FIELDS_EXTENSION; +import static org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsWriter.INDEX_EXTENSION; +import static org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsWriter.META_EXTENSION; public class SourceOnlySnapshot { @@ -223,7 +224,7 @@ private SegmentCommitInfo syncSegment(SegmentCommitInfo segmentCommitInfo, LiveD for (FieldInfo fieldInfo : fieldInfos) { fieldInfoCopy.add(new FieldInfo(fieldInfo.name, fieldInfo.number, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, fieldInfo.attributes(), 0, 0, 0, - fieldInfo.isSoftDeletesField())); + 0, VectorSimilarityFunction.EUCLIDEAN, fieldInfo.isSoftDeletesField())); } FieldInfos newFieldInfos = new FieldInfos(fieldInfoCopy.toArray(new FieldInfo[0])); codec.fieldInfosFormat().write(trackingDir, newSegmentInfo, segmentSuffix, newFieldInfos, IOContext.DEFAULT); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java index 91925062b8f6b..76e6516f89f1c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java @@ -29,12 +29,12 @@ import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.core.Tuple; import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; @@ -321,11 +321,6 @@ public void checkIntegrity() throws IOException { public void close() throws IOException { reader.close(); } - - @Override - public long ramBytesUsed() { - return reader.ramBytesUsed(); - } } /** @@ -353,7 +348,7 @@ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { } @Override - public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { + public void stringField(FieldInfo fieldInfo, String value) throws IOException { visitor.stringField(fieldInfo, value); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java index 4f81dc425892c..0ac4a98542539 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java @@ -473,7 +473,7 @@ public void testSortedDocValues() throws Exception { SortedDocValues values = segmentReader.getSortedDocValues("fieldA"); assertNotNull(values); assertTrue(values.advanceExact(0)); - assertEquals(new BytesRef("testA"), values.binaryValue()); + assertEquals(new BytesRef("testA"), values.lookupOrd(values.ordValue())); assertNull(segmentReader.getSortedDocValues("fieldB")); TestUtil.checkReader(ir); diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 3abdb53df0313..fbacf7844be5a 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -14,8 +14,8 @@ import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; +import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; -import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java index 6cd918afee846..4f7e4b853939e 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java @@ -24,7 +24,6 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; -import org.apache.lucene.util.FutureArrays; import org.elasticsearch.action.bulk.BackoffPolicy; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkProcessor; @@ -380,7 +379,7 @@ private static Comparator keyComparator() { return (o1, o2) -> { int keySize1 = readInt(o1.bytes, o1.offset); int keySize2 = readInt(o2.bytes, o2.offset); - return FutureArrays.compareUnsigned( + return Arrays.compareUnsigned( o1.bytes, o1.offset + Integer.BYTES, keySize1 + o1.offset + Integer.BYTES, diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScoreWeight.java b/x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScoreWeight.java index 42790a4f68b8e..6128920858ec7 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScoreWeight.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScoreWeight.java @@ -8,10 +8,8 @@ package org.apache.lucene.search; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; import java.io.IOException; -import java.util.Set; /** * A Weight that caps scores of the wrapped query to a maximum value @@ -27,11 +25,6 @@ protected CappedScoreWeight(Query query, Weight innerWeight, float maxScore) { this.innerWeight = innerWeight; } - @Override - public void extractTerms(Set terms) { - innerWeight.extractTerms(terms); - } - @Override public boolean isCacheable(LeafReaderContext ctx) { return innerWeight.isCacheable(ctx); diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectory.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectory.java index d3f2d0ff8da3c..f6cc9ca40036c 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectory.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectory.java @@ -86,7 +86,6 @@ import java.util.function.LongSupplier; import java.util.function.Supplier; -import static org.apache.lucene.store.BufferedIndexInput.bufferSize; import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE; import static org.elasticsearch.xpack.core.searchablesnapshots.SearchableSnapshotsConstants.SNAPSHOT_PARTIAL_SETTING; @@ -427,15 +426,7 @@ public IndexInput openInput(final String name, final IOContext context) throws I ); } } else { - return new DirectBlobContainerIndexInput( - name, - this, - fileInfo, - context, - inputStats, - getUncachedChunkSize(), - bufferSize(context) - ); + return new DirectBlobContainerIndexInput(name, this, fileInfo, context, inputStats, getUncachedChunkSize()); } } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/BaseSearchableSnapshotIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/BaseSearchableSnapshotIndexInput.java index 4777699fcf692..c7916e47b64ad 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/BaseSearchableSnapshotIndexInput.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/BaseSearchableSnapshotIndexInput.java @@ -300,9 +300,12 @@ protected final boolean assertCurrentThreadMayAccessBlobStore() { // Cache prewarming also runs on a dedicated thread pool. || threadName.contains('[' + SearchableSnapshots.CACHE_PREWARMING_THREAD_POOL_NAME + ']') - // Unit tests access the blob store on the main test thread; simplest just to permit this rather than have them override this + // Unit tests access the blob store on the main test thread, or via an asynchronous + // checkindex call; + // simplest just to permit this rather than have them override this // method somehow. || threadName.startsWith("TEST-") + || threadName.startsWith("async-check-index") || threadName.startsWith("LuceneTestCase") : "current thread [" + Thread.currentThread() + "] may not read " + fileInfo; return true; } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/ChecksumBlobContainerIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/ChecksumBlobContainerIndexInput.java index aac42c81334e6..db02872a89c84 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/ChecksumBlobContainerIndexInput.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/ChecksumBlobContainerIndexInput.java @@ -140,9 +140,9 @@ public static byte[] checksumToBytesArray(String checksum) throws IOException { assert result.length >= Integer.BYTES + Integer.BYTES + Long.BYTES; // ensure that nobody changed the file format under us final ByteArrayDataOutput output = new ByteArrayDataOutput(result); // reverse CodecUtil.writeFooter() - output.writeInt(CodecUtil.FOOTER_MAGIC); - output.writeInt(0); - output.writeLong(Long.parseLong(checksum, Character.MAX_RADIX)); + CodecUtil.writeBEInt(output, CodecUtil.FOOTER_MAGIC); + CodecUtil.writeBEInt(output, 0); + CodecUtil.writeBELong(output, Long.parseLong(checksum, Character.MAX_RADIX)); assert output.getPosition() == result.length; return result; } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInput.java index 1617e3ec7f4f2..0daf828cc47f5 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInput.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInput.java @@ -73,10 +73,9 @@ public DirectBlobContainerIndexInput( FileInfo fileInfo, IOContext context, IndexInputStats stats, - long sequentialReadSize, - int bufferSize + long sequentialReadSize ) { - this(name, directory, fileInfo, context, stats, 0L, 0L, fileInfo.length(), sequentialReadSize, bufferSize); + this(name, directory, fileInfo, context, stats, 0L, 0L, fileInfo.length(), sequentialReadSize); stats.incrementOpenCount(); } @@ -89,15 +88,13 @@ private DirectBlobContainerIndexInput( long position, long offset, long length, - long sequentialReadSize, - int bufferSize + long sequentialReadSize ) { super(logger, name, directory, fileInfo, context, stats, offset, length, ByteRange.EMPTY, ByteRange.EMPTY); // TODO should use blob // cache this.position = position; assert sequentialReadSize >= 0; this.sequentialReadSize = sequentialReadSize; - setBufferSize(bufferSize); } @Override @@ -272,8 +269,7 @@ public DirectBlobContainerIndexInput clone() { length, // Clones might not be closed when they are no longer needed, but we must always close streamForSequentialReads. The simple // solution: do not optimize sequential reads on clones. - NO_SEQUENTIAL_READ_OPTIMIZATION, - getBufferSize() + NO_SEQUENTIAL_READ_OPTIMIZATION ); clone.isClone = true; return clone; @@ -293,8 +289,7 @@ public IndexInput slice(String sliceName, long offset, long length) throws IOExc length, // Slices might not be closed when they are no longer needed, but we must always close streamForSequentialReads. The simple // solution: do not optimize sequential reads on slices. - NO_SEQUENTIAL_READ_OPTIMIZATION, - getBufferSize() + NO_SEQUENTIAL_READ_OPTIMIZATION ); slice.isClone = true; slice.seek(0L); diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInputTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInputTests.java index bef6d3ea29622..fd7b1d1792a39 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInputTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInputTests.java @@ -133,8 +133,7 @@ public int read(byte[] b, int off, int len) throws IOException { fileInfo, randomIOContext(), new IndexInputStats(1L, fileInfo.length(), fileInfo.length(), fileInfo.length(), () -> 0L), - minimumReadSize, - randomBoolean() ? BufferedIndexInput.BUFFER_SIZE : between(BufferedIndexInput.MIN_BUFFER_SIZE, BufferedIndexInput.BUFFER_SIZE) + minimumReadSize ); assertEquals(input.length, indexInput.length()); return indexInput; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractor.java index 820fac1d30c78..309ca6da30da0 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractor.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractor.java @@ -7,11 +7,12 @@ package org.elasticsearch.xpack.security.authz.accesscontrol; import org.apache.lucene.index.PrefixCodedTerms.TermIterator; +import org.apache.lucene.queries.spans.SpanTermQuery; +import org.apache.lucene.sandbox.search.DocValuesNumbersQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.DocValuesFieldExistsQuery; -import org.apache.lucene.search.DocValuesNumbersQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; @@ -23,8 +24,6 @@ import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.Weight; -import org.apache.lucene.search.spans.SpanTermQuery; import java.util.HashSet; import java.util.Set; @@ -32,9 +31,11 @@ /** * Extracts fields from a query, or throws UnsupportedOperationException. *

- * Lucene queries have {@link Weight#extractTerms}, but this is really geared at things + * Lucene queries used to have Weight#extractTerms, but this was really geared at things * such as highlighting, not security. For example terms in a Boolean {@code MUST_NOT} clause * are not included, TermsQuery doesn't implement the method as it could be terribly slow, etc. + * + * TODO reimplement this using QueryVisitors */ class FieldExtractor { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractorTests.java index a39431e56a408..fc30c61cd0aee 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractorTests.java @@ -9,12 +9,13 @@ import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.spans.SpanTermQuery; +import org.apache.lucene.sandbox.search.DocValuesNumbersQuery; import org.apache.lucene.search.AssertingQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.DocValuesFieldExistsQuery; -import org.apache.lucene.search.DocValuesNumbersQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; @@ -24,7 +25,6 @@ import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.test.ESTestCase; @@ -69,7 +69,10 @@ public void testTerm() { public void testSynonym() { Set fields = new HashSet<>(); - SynonymQuery query = new SynonymQuery(new Term("foo", "bar"), new Term("foo", "baz")); + SynonymQuery query = new SynonymQuery.Builder("foo") + .addTerm(new Term("foo", "bar")) + .addTerm(new Term("foo", "baz")) + .build(); FieldExtractor.extractFields(query, fields); assertEquals(asSet("foo"), fields); } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialUtils.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialUtils.java index 48d1690665419..080ca9f05a693 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialUtils.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialUtils.java @@ -45,8 +45,8 @@ public static Polygon createRegularGeoShapePolygon(Circle circle, int gons) { for(int i=0; i