Upgrade to lucene-9.0.0-snapshot-32a0a16aff0

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.0.0
-lucene            = 8.9.0
+lucene            = 9.0.0-snapshot-7450a7e64be
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 16.0.2+7

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java

@@ -8,7 +8,9 @@
 
 package org.elasticsearch.common.settings;
 
+import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
 import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.store.DataOutput;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexOutput;
@@ -345,13 +347,14 @@ private void possiblyAlterEncryptedBytes(
         byte[] encryptedBytes,
         int truncEncryptedDataLength
     ) throws Exception {
-        indexOutput.writeInt(4 + salt.length + 4 + iv.length + 4 + encryptedBytes.length);
-        indexOutput.writeInt(salt.length);
-        indexOutput.writeBytes(salt, salt.length);
-        indexOutput.writeInt(iv.length);
-        indexOutput.writeBytes(iv, iv.length);
-        indexOutput.writeInt(encryptedBytes.length - truncEncryptedDataLength);
-        indexOutput.writeBytes(encryptedBytes, encryptedBytes.length);
+        DataOutput out = EndiannessReverserUtil.wrapDataOutput(indexOutput);
+        out.writeInt(4 + salt.length + 4 + iv.length + 4 + encryptedBytes.length);
+        out.writeInt(salt.length);
+        out.writeBytes(salt, salt.length);
+        out.writeInt(iv.length);
+        out.writeBytes(iv, iv.length);
+        out.writeInt(encryptedBytes.length - truncEncryptedDataLength);
+        out.writeBytes(encryptedBytes, encryptedBytes.length);
     }
 
     public void testUpgradeAddsSeed() throws Exception {
@@ -382,7 +385,7 @@ public void testBackcompatV1() throws Exception {
         Path configDir = env.configFile();
         try (
             Directory directory = newFSDirectory(configDir);
-            IndexOutput output = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)
+            IndexOutput output = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT);
         ) {
             CodecUtil.writeHeader(output, "elasticsearch.keystore", 1);
             output.writeByte((byte) 0); // hasPassword = false
@@ -417,9 +420,8 @@ public void testBackcompatV2() throws Exception {
         random().nextBytes(fileBytes);
         try (
             Directory directory = newFSDirectory(configDir);
-            IndexOutput output = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)
+            IndexOutput output = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT);
         ) {
-
             CodecUtil.writeHeader(output, "elasticsearch.keystore", 2);
             output.writeByte((byte) 0); // hasPassword = false
             output.writeString("PKCS12");

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        8.9.0
-:lucene_version_path:   8_9_0
+:lucene_version:        9.0.0
+:lucene_version_path:   9_0_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_flavor:          default

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicFilterFactory.java

@@ -8,7 +8,7 @@
 package org.elasticsearch.analysis.common;
 
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.standard.ClassicFilter;
+import org.apache.lucene.analysis.classic.ClassicFilter;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicTokenizerFactory.java

@@ -9,7 +9,7 @@
 package org.elasticsearch.analysis.common;
 
 import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.standard.ClassicTokenizer;
+import org.apache.lucene.analysis.classic.ClassicTokenizer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java

@@ -27,6 +27,8 @@
 import org.apache.lucene.analysis.cjk.CJKWidthFilter;
 import org.apache.lucene.analysis.ckb.SoraniAnalyzer;
 import org.apache.lucene.analysis.ckb.SoraniNormalizationFilter;
+import org.apache.lucene.analysis.classic.ClassicFilter;
+import org.apache.lucene.analysis.classic.ClassicTokenizer;
 import org.apache.lucene.analysis.commongrams.CommonGramsFilter;
 import org.apache.lucene.analysis.core.DecimalDigitFilter;
 import org.apache.lucene.analysis.core.KeywordTokenizer;
@@ -40,6 +42,7 @@
 import org.apache.lucene.analysis.de.GermanNormalizationFilter;
 import org.apache.lucene.analysis.de.GermanStemFilter;
 import org.apache.lucene.analysis.el.GreekAnalyzer;
+import org.apache.lucene.analysis.email.UAX29URLEmailTokenizer;
 import org.apache.lucene.analysis.en.EnglishAnalyzer;
 import org.apache.lucene.analysis.en.KStemFilter;
 import org.apache.lucene.analysis.en.PorterStemFilter;
@@ -89,10 +92,7 @@
 import org.apache.lucene.analysis.ru.RussianAnalyzer;
 import org.apache.lucene.analysis.shingle.ShingleFilter;
 import org.apache.lucene.analysis.snowball.SnowballFilter;
-import org.apache.lucene.analysis.standard.ClassicFilter;
-import org.apache.lucene.analysis.standard.ClassicTokenizer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer;
 import org.apache.lucene.analysis.sv.SwedishAnalyzer;
 import org.apache.lucene.analysis.th.ThaiAnalyzer;
 import org.apache.lucene.analysis.th.ThaiTokenizer;
@@ -289,6 +289,7 @@ public TokenStream create(TokenStream tokenStream) {
 
             };
         });
+        filters.put("norwegiannormalization", NorwegianNormalizationFilterFactory::new);
         filters.put("pattern_capture", requiresAnalysisSettings(PatternCaptureGroupTokenFilterFactory::new));
         filters.put("pattern_replace", requiresAnalysisSettings(PatternReplaceTokenFilterFactory::new));
         filters.put("persian_normalization", PersianNormalizationFilterFactory::new);
@@ -305,6 +306,7 @@ public TokenStream create(TokenStream tokenStream) {
         filters.put("sorani_normalization", SoraniNormalizationFilterFactory::new);
         filters.put("stemmer_override", requiresAnalysisSettings(StemmerOverrideTokenFilterFactory::new));
         filters.put("stemmer", StemmerTokenFilterFactory::new);
+        filters.put("swedishminimalstem", SwedishMinimalStemFilterFactory::new);
         filters.put("synonym", requiresAnalysisSettings(SynonymTokenFilterFactory::new));
         filters.put("synonym_graph", requiresAnalysisSettings(SynonymGraphTokenFilterFactory::new));
         filters.put("trim", TrimTokenFilterFactory::new);

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MinHashTokenFilterFactory.java

@@ -38,10 +38,18 @@ public TokenStream create(TokenStream tokenStream) {
 
     private Map<String, String> convertSettings(Settings settings) {
         Map<String, String> settingMap = new HashMap<>();
-        settingMap.put("hashCount", settings.get("hash_count"));
-        settingMap.put("bucketCount", settings.get("bucket_count"));
-        settingMap.put("hashSetSize", settings.get("hash_set_size"));
-        settingMap.put("withRotation", settings.get("with_rotation"));
+        if (settings.hasValue("hash_count")) {
+            settingMap.put("hashCount", settings.get("hash_count"));
+        }
+        if (settings.hasValue("bucketCount")) {
+            settingMap.put("bucketCount", settings.get("bucket_count"));
+        }
+        if (settings.hasValue("hashSetSize")) {
+            settingMap.put("hashSetSize", settings.get("hash_set_size"));
+        }
+        if (settings.hasValue("with_rotation")) {
+            settingMap.put("withRotation", settings.get("with_rotation"));
+        }
         return settingMap;
     }
 }

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NorwegianNormalizationFilterFactory.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.analysis.common;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.no.NorwegianNormalizationFilter;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
+import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory;
+
+/**
+ * Factory for {@link NorwegianNormalizationFilter}
+ */
+public class NorwegianNormalizationFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory {
+
+    public NorwegianNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
+        super(indexSettings, name, settings);
+    }
+
+    @Override
+    public TokenStream create(TokenStream tokenStream) {
+        return new NorwegianNormalizationFilter(tokenStream);
+    }
+}

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SwedishMinimalStemFilterFactory.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.analysis.common;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.sv.SwedishMinimalStemFilter;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
+
+/**
+ * Factory for {@link SwedishMinimalStemFilter}
+ */
+public class SwedishMinimalStemFilterFactory extends AbstractTokenFilterFactory {
+
+    public SwedishMinimalStemFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
+        super(indexSettings, name, settings);
+    }
+
+    @Override
+    public TokenStream create(TokenStream tokenStream) {
+        return new SwedishMinimalStemFilter(tokenStream);
+    }
+}

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UAX29URLEmailTokenizerFactory.java

@@ -9,8 +9,8 @@
 package org.elasticsearch.analysis.common;
 
 import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.email.UAX29URLEmailTokenizer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;

modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java

@@ -84,13 +84,15 @@ protected Map<String, Class<?>> getTokenFilters() {
         filters.put("latvianstem", StemmerTokenFilterFactory.class);
         filters.put("norwegianlightstem", StemmerTokenFilterFactory.class);
         filters.put("norwegianminimalstem", StemmerTokenFilterFactory.class);
+        filters.put("norwegiannormalization", NorwegianNormalizationFilterFactory.class);
         filters.put("portuguesestem", StemmerTokenFilterFactory.class);
         filters.put("portugueselightstem", StemmerTokenFilterFactory.class);
         filters.put("portugueseminimalstem", StemmerTokenFilterFactory.class);
         filters.put("russianlightstem", StemmerTokenFilterFactory.class);
         filters.put("soranistem", StemmerTokenFilterFactory.class);
         filters.put("spanishlightstem", StemmerTokenFilterFactory.class);
         filters.put("swedishlightstem", StemmerTokenFilterFactory.class);
+        filters.put("swedishminimalstem", SwedishMinimalStemFilterFactory.class);
         filters.put("stemmeroverride", StemmerOverrideTokenFilterFactory.class);
         filters.put("kstem", KStemTokenFilterFactory.class);
         filters.put("synonym", SynonymTokenFilterFactory.class);
@@ -212,7 +214,7 @@ protected Map<String, Class<?>> getPreConfiguredTokenizers() {
         tokenizers.put("keyword", null);
         tokenizers.put("lowercase", Void.class);
         tokenizers.put("classic", null);
-        tokenizers.put("uax_url_email", org.apache.lucene.analysis.standard.UAX29URLEmailTokenizerFactory.class);
+        tokenizers.put("uax_url_email", org.apache.lucene.analysis.email.UAX29URLEmailTokenizerFactory.class);
         tokenizers.put("path_hierarchy", null);
         tokenizers.put("letter", null);
         tokenizers.put("whitespace", null);

modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java

@@ -80,15 +80,17 @@ public void setup() {
         // that ignores position length attribute
          expectedQueryWithUnigram= new BooleanQuery.Builder()
             .add(
-                new SynonymQuery(
-                    new Term("text_shingle_unigram", "foo"),
-                    new Term("text_shingle_unigram", "foo bar")
-                ), BooleanClause.Occur.SHOULD)
+                new SynonymQuery.Builder("text_shingle_unigram")
+                    .addTerm(new Term("text_shingle_unigram", "foo"))
+                    .addTerm(new Term("text_shingle_unigram", "foo bar"))
+                    .build(),
+                BooleanClause.Occur.SHOULD)
             .add(
-                new SynonymQuery(
-                    new Term("text_shingle_unigram", "bar"),
-                    new Term("text_shingle_unigram", "bar baz")
-            ), BooleanClause.Occur.SHOULD)
+                new SynonymQuery.Builder("text_shingle_unigram")
+                    .addTerm(new Term("text_shingle_unigram", "bar"))
+                    .addTerm(new Term("text_shingle_unigram", "bar baz"))
+                    .build(),
+                BooleanClause.Occur.SHOULD)
             .add(
                 new TermQuery(
                     new Term("text_shingle_unigram", "baz")

modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-7450a7e64be.jar.sha1

@@ -0,0 +1 @@
+42eaef59486b8d205136510bb4adbcda64b07ff9

modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java

@@ -13,7 +13,6 @@
 import org.apache.lucene.expressions.js.JavascriptCompiler;
 import org.apache.lucene.expressions.js.VariableContext;
 import org.apache.lucene.search.DoubleValuesSource;
-import org.apache.lucene.search.SortField;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -232,7 +231,7 @@ private static NumberSortScript.LeafFactory newSortScript(Expression expr, Searc
         for (String variable : expr.variables) {
             try {
                 if (variable.equals("_score")) {
-                    bindings.add(new SortField("_score", SortField.Type.SCORE));
+                    bindings.add("_score", DoubleValuesSource.SCORES);
                     needsScores = true;
                 } else if (vars != null && vars.containsKey(variable)) {
                     bindFromParams(vars, bindings, variable);
@@ -283,7 +282,7 @@ private static AggregationScript.LeafFactory newAggregationScript(Expression exp
         for (String variable : expr.variables) {
             try {
                 if (variable.equals("_score")) {
-                    bindings.add(new SortField("_score", SortField.Type.SCORE));
+                    bindings.add("_score", DoubleValuesSource.SCORES);
                     needsScores = true;
                 } else if (variable.equals("_value")) {
                     specialValue = new ReplaceableConstDoubleValueSource();
@@ -355,7 +354,7 @@ private static ScoreScript.LeafFactory newScoreScript(Expression expr, SearchLoo
         for (String variable : expr.variables) {
             try {
                 if (variable.equals("_score")) {
-                    bindings.add(new SortField("_score", SortField.Type.SCORE));
+                    bindings.add("_score", DoubleValuesSource.SCORES);
                     needsScores = true;
                 } else if (variable.equals("_value")) {
                     specialValue = new ReplaceableConstDoubleValueSource();

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java

@@ -20,6 +20,10 @@
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.queries.spans.FieldMaskingSpanQuery;
+import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper;
+import org.apache.lucene.queries.spans.SpanQuery;
+import org.apache.lucene.queries.spans.SpanTermQuery;
 import org.apache.lucene.search.AutomatonQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
@@ -28,10 +32,6 @@
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.spans.FieldMaskingSpanQuery;
-import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
-import org.apache.lucene.search.spans.SpanQuery;
-import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.util.automaton.Automata;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.Operations;

modules/mapper-extras/src/main/java/org/elasticsearch/index/query/SourceConfirmedTextQuery.java

@@ -183,6 +183,11 @@ public int hashCode() {
         return 31 * Objects.hash(in, valueFetcherProvider, indexAnalyzer) + classHash();
     }
 
+    @Override
+    public void visit(QueryVisitor visitor) {
+        in.visit(visitor.getSubVisitor(Occur.MUST, this));
+    }
+
     @Override
     public Query rewrite(IndexReader reader) throws IOException {
         Query inRewritten = in.rewrite(reader);
@@ -244,11 +249,6 @@ public boolean isCacheable(LeafReaderContext ctx) {
                 return false;
             }
 
-            @Override
-            public void extractTerms(Set<Term> termSet) {
-                termSet.addAll(terms);
-            }
-
             @Override
             public Explanation explain(LeafReaderContext context, int doc) throws IOException {
                 RuntimePhraseScorer scorer = scorer(context);

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java

@@ -15,6 +15,9 @@
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.IndexableFieldType;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.queries.spans.FieldMaskingSpanQuery;
+import org.apache.lucene.queries.spans.SpanNearQuery;
+import org.apache.lucene.queries.spans.SpanTermQuery;
 import org.apache.lucene.queryparser.classic.ParseException;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
@@ -26,9 +29,6 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.SynonymQuery;
 import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.spans.FieldMaskingSpanQuery;
-import org.apache.lucene.search.spans.SpanNearQuery;
-import org.apache.lucene.search.spans.SpanTermQuery;
 import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.IndexSettings;