Upgrade to lucene-9.0.0-snapshot-32a0a16aff0

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.0.0
-lucene            = 8.10.0-snapshot-bf2fcb53079
+lucene            = 9.0.0-snapshot-32a0a16aff0
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 16.0.2+7
@@ -11,7 +11,7 @@ spatial4j         = 0.7
 jts               = 1.15.0
 jackson           = 2.10.4
 snakeyaml         = 1.26
-icu4j             = 62.1
+icu4j             = 68.2
 supercsv          = 2.4.0
 # when updating log4j, please update also docs/java-api/index.asciidoc
 log4j             = 2.11.1

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java

@@ -8,7 +8,9 @@
 
 package org.elasticsearch.common.settings;
 
+import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
 import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.store.DataOutput;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexOutput;
@@ -345,13 +347,14 @@ private void possiblyAlterEncryptedBytes(
         byte[] encryptedBytes,
         int truncEncryptedDataLength
     ) throws Exception {
-        indexOutput.writeInt(4 + salt.length + 4 + iv.length + 4 + encryptedBytes.length);
-        indexOutput.writeInt(salt.length);
-        indexOutput.writeBytes(salt, salt.length);
-        indexOutput.writeInt(iv.length);
-        indexOutput.writeBytes(iv, iv.length);
-        indexOutput.writeInt(encryptedBytes.length - truncEncryptedDataLength);
-        indexOutput.writeBytes(encryptedBytes, encryptedBytes.length);
+        DataOutput out = EndiannessReverserUtil.wrapDataOutput(indexOutput);
+        out.writeInt(4 + salt.length + 4 + iv.length + 4 + encryptedBytes.length);
+        out.writeInt(salt.length);
+        out.writeBytes(salt, salt.length);
+        out.writeInt(iv.length);
+        out.writeBytes(iv, iv.length);
+        out.writeInt(encryptedBytes.length - truncEncryptedDataLength);
+        out.writeBytes(encryptedBytes, encryptedBytes.length);
     }
 
     public void testUpgradeAddsSeed() throws Exception {
@@ -382,7 +385,7 @@ public void testBackcompatV1() throws Exception {
         Path configDir = env.configFile();
         try (
             Directory directory = newFSDirectory(configDir);
-            IndexOutput output = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)
+            IndexOutput output = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT);
         ) {
             CodecUtil.writeHeader(output, "elasticsearch.keystore", 1);
             output.writeByte((byte) 0); // hasPassword = false
@@ -417,9 +420,8 @@ public void testBackcompatV2() throws Exception {
         random().nextBytes(fileBytes);
         try (
             Directory directory = newFSDirectory(configDir);
-            IndexOutput output = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)
+            IndexOutput output = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT);
         ) {
-
             CodecUtil.writeHeader(output, "elasticsearch.keystore", 2);
             output.writeByte((byte) 0); // hasPassword = false
             output.writeString("PKCS12");

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        8.10.0
-:lucene_version_path:   8_10_0
+:lucene_version:        9.0.0
+:lucene_version_path:   9_0_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_flavor:          default

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicFilterFactory.java

@@ -8,7 +8,7 @@
 package org.elasticsearch.analysis.common;
 
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.standard.ClassicFilter;
+import org.apache.lucene.analysis.classic.ClassicFilter;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicTokenizerFactory.java

@@ -9,7 +9,7 @@
 package org.elasticsearch.analysis.common;
 
 import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.standard.ClassicTokenizer;
+import org.apache.lucene.analysis.classic.ClassicTokenizer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java

@@ -27,6 +27,8 @@
 import org.apache.lucene.analysis.cjk.CJKWidthFilter;
 import org.apache.lucene.analysis.ckb.SoraniAnalyzer;
 import org.apache.lucene.analysis.ckb.SoraniNormalizationFilter;
+import org.apache.lucene.analysis.classic.ClassicFilter;
+import org.apache.lucene.analysis.classic.ClassicTokenizer;
 import org.apache.lucene.analysis.commongrams.CommonGramsFilter;
 import org.apache.lucene.analysis.core.DecimalDigitFilter;
 import org.apache.lucene.analysis.core.KeywordTokenizer;
@@ -40,6 +42,7 @@
 import org.apache.lucene.analysis.de.GermanNormalizationFilter;
 import org.apache.lucene.analysis.de.GermanStemFilter;
 import org.apache.lucene.analysis.el.GreekAnalyzer;
+import org.apache.lucene.analysis.email.UAX29URLEmailTokenizer;
 import org.apache.lucene.analysis.en.EnglishAnalyzer;
 import org.apache.lucene.analysis.en.KStemFilter;
 import org.apache.lucene.analysis.en.PorterStemFilter;
@@ -89,10 +92,7 @@
 import org.apache.lucene.analysis.ru.RussianAnalyzer;
 import org.apache.lucene.analysis.shingle.ShingleFilter;
 import org.apache.lucene.analysis.snowball.SnowballFilter;
-import org.apache.lucene.analysis.standard.ClassicFilter;
-import org.apache.lucene.analysis.standard.ClassicTokenizer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer;
 import org.apache.lucene.analysis.sv.SwedishAnalyzer;
 import org.apache.lucene.analysis.th.ThaiAnalyzer;
 import org.apache.lucene.analysis.th.ThaiTokenizer;
@@ -289,6 +289,7 @@ public TokenStream create(TokenStream tokenStream) {
 
             };
         });
+        filters.put("norwegiannormalization", NorwegianNormalizationFilterFactory::new);
         filters.put("pattern_capture", requiresAnalysisSettings(PatternCaptureGroupTokenFilterFactory::new));
         filters.put("pattern_replace", requiresAnalysisSettings(PatternReplaceTokenFilterFactory::new));
         filters.put("persian_normalization", PersianNormalizationFilterFactory::new);
@@ -305,8 +306,11 @@ public TokenStream create(TokenStream tokenStream) {
         filters.put("sorani_normalization", SoraniNormalizationFilterFactory::new);
         filters.put("stemmer_override", requiresAnalysisSettings(StemmerOverrideTokenFilterFactory::new));
         filters.put("stemmer", StemmerTokenFilterFactory::new);
+        filters.put("swedishminimalstem", SwedishMinimalStemFilterFactory::new);
         filters.put("synonym", requiresAnalysisSettings(SynonymTokenFilterFactory::new));
         filters.put("synonym_graph", requiresAnalysisSettings(SynonymGraphTokenFilterFactory::new));
+        filters.put("telugunormalization", TeluguNormalizationFilterFactory::new);
+        filters.put("telugustem", TeluguStemFilterFactory::new);
         filters.put("trim", TrimTokenFilterFactory::new);
         filters.put("truncate", requiresAnalysisSettings(TruncateTokenFilterFactory::new));
         filters.put("unique", UniqueTokenFilterFactory::new);

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MinHashTokenFilterFactory.java

@@ -38,10 +38,18 @@ public TokenStream create(TokenStream tokenStream) {
 
     private Map<String, String> convertSettings(Settings settings) {
         Map<String, String> settingMap = new HashMap<>();
-        settingMap.put("hashCount", settings.get("hash_count"));
-        settingMap.put("bucketCount", settings.get("bucket_count"));
-        settingMap.put("hashSetSize", settings.get("hash_set_size"));
-        settingMap.put("withRotation", settings.get("with_rotation"));
+        if (settings.hasValue("hash_count")) {
+            settingMap.put("hashCount", settings.get("hash_count"));
+        }
+        if (settings.hasValue("bucketCount")) {
+            settingMap.put("bucketCount", settings.get("bucket_count"));
+        }
+        if (settings.hasValue("hashSetSize")) {
+            settingMap.put("hashSetSize", settings.get("hash_set_size"));
+        }
+        if (settings.hasValue("with_rotation")) {
+            settingMap.put("withRotation", settings.get("with_rotation"));
+        }
         return settingMap;
     }
 }

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NorwegianNormalizationFilterFactory.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.analysis.common;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.no.NorwegianNormalizationFilter;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
+import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory;
+
+/**
+ * Factory for {@link NorwegianNormalizationFilter}
+ */
+public class NorwegianNormalizationFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory {
+
+    public NorwegianNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
+        super(indexSettings, name, settings);
+    }
+
+    @Override
+    public TokenStream create(TokenStream tokenStream) {
+        return new NorwegianNormalizationFilter(tokenStream);
+    }
+}

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SwedishMinimalStemFilterFactory.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.analysis.common;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.sv.SwedishMinimalStemFilter;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
+
+/**
+ * Factory for {@link SwedishMinimalStemFilter}
+ */
+public class SwedishMinimalStemFilterFactory extends AbstractTokenFilterFactory {
+
+    public SwedishMinimalStemFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
+        super(indexSettings, name, settings);
+    }
+
+    @Override
+    public TokenStream create(TokenStream tokenStream) {
+        return new SwedishMinimalStemFilter(tokenStream);
+    }
+}

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TeluguNormalizationFilterFactory.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.analysis.common;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.te.TeluguNormalizationFilter;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
+
+public class TeluguNormalizationFilterFactory extends AbstractTokenFilterFactory {
+
+    public TeluguNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
+        super(indexSettings, name, settings);
+    }
+
+    @Override
+    public TokenStream create(TokenStream tokenStream) {
+        return new TeluguNormalizationFilter(tokenStream);
+    }
+}

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TeluguStemFilterFactory.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.analysis.common;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.te.TeluguStemFilter;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
+
+public class TeluguStemFilterFactory extends AbstractTokenFilterFactory {
+
+    public TeluguStemFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
+        super(indexSettings, name, settings);
+    }
+
+    @Override
+    public TokenStream create(TokenStream tokenStream) {
+        return new TeluguStemFilter(tokenStream);
+    }
+}

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UAX29URLEmailTokenizerFactory.java

@@ -9,8 +9,8 @@
 package org.elasticsearch.analysis.common;
 
 import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.email.UAX29URLEmailTokenizer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;

modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java

@@ -86,14 +86,18 @@ protected Map<String, Class<?>> getTokenFilters() {
         filters.put("latvianstem", StemmerTokenFilterFactory.class);
         filters.put("norwegianlightstem", StemmerTokenFilterFactory.class);
         filters.put("norwegianminimalstem", StemmerTokenFilterFactory.class);
+        filters.put("norwegiannormalization", NorwegianNormalizationFilterFactory.class);
         filters.put("portuguesestem", StemmerTokenFilterFactory.class);
         filters.put("portugueselightstem", StemmerTokenFilterFactory.class);
         filters.put("portugueseminimalstem", StemmerTokenFilterFactory.class);
         filters.put("russianlightstem", StemmerTokenFilterFactory.class);
         filters.put("soranistem", StemmerTokenFilterFactory.class);
         filters.put("spanishlightstem", StemmerTokenFilterFactory.class);
         filters.put("swedishlightstem", StemmerTokenFilterFactory.class);
+        filters.put("swedishminimalstem", SwedishMinimalStemFilterFactory.class);
         filters.put("stemmeroverride", StemmerOverrideTokenFilterFactory.class);
+        filters.put("telugunormalization", TeluguNormalizationFilterFactory.class);
+        filters.put("telugustem", TeluguStemFilterFactory.class);
         filters.put("kstem", KStemTokenFilterFactory.class);
         filters.put("synonym", SynonymTokenFilterFactory.class);
         filters.put("synonymgraph", SynonymGraphTokenFilterFactory.class);
@@ -131,8 +135,6 @@ protected Map<String, Class<?>> getTokenFilters() {
         filters.put("brazilianstem", BrazilianStemTokenFilterFactory.class);
         filters.put("czechstem", CzechStemTokenFilterFactory.class);
         filters.put("germanstem", GermanStemTokenFilterFactory.class);
-        filters.put("telugunormalization", TeluguNormalizationFilterFactory.class);
-        filters.put("telugustem", TeluguStemFilterFactory.class);
         // this filter is not exposed and should only be used internally
         filters.put("fixedshingle", Void.class);
         return filters;
@@ -216,7 +218,7 @@ protected Map<String, Class<?>> getPreConfiguredTokenizers() {
         tokenizers.put("keyword", null);
         tokenizers.put("lowercase", Void.class);
         tokenizers.put("classic", null);
-        tokenizers.put("uax_url_email", org.apache.lucene.analysis.standard.UAX29URLEmailTokenizerFactory.class);
+        tokenizers.put("uax_url_email", org.apache.lucene.analysis.email.UAX29URLEmailTokenizerFactory.class);
         tokenizers.put("path_hierarchy", null);
         tokenizers.put("letter", null);
         tokenizers.put("whitespace", null);

modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java

@@ -80,15 +80,17 @@ public void setup() {
         // that ignores position length attribute
          expectedQueryWithUnigram= new BooleanQuery.Builder()
             .add(
-                new SynonymQuery(
-                    new Term("text_shingle_unigram", "foo"),
-                    new Term("text_shingle_unigram", "foo bar")
-                ), BooleanClause.Occur.SHOULD)
+                new SynonymQuery.Builder("text_shingle_unigram")
+                    .addTerm(new Term("text_shingle_unigram", "foo"))
+                    .addTerm(new Term("text_shingle_unigram", "foo bar"))
+                    .build(),
+                BooleanClause.Occur.SHOULD)
             .add(
-                new SynonymQuery(
-                    new Term("text_shingle_unigram", "bar"),
-                    new Term("text_shingle_unigram", "bar baz")
-            ), BooleanClause.Occur.SHOULD)
+                new SynonymQuery.Builder("text_shingle_unigram")
+                    .addTerm(new Term("text_shingle_unigram", "bar"))
+                    .addTerm(new Term("text_shingle_unigram", "bar baz"))
+                    .build(),
+                BooleanClause.Occur.SHOULD)
             .add(
                 new TermQuery(
                     new Term("text_shingle_unigram", "baz")

modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-32a0a16aff0.jar.sha1

@@ -0,0 +1 @@
+561301335ba14e07ec89adff0e2321cb79146a48