Merge remote-tracking branch 'opensearch-main/2.x' into remote-checks…

…um-fix-2x
opensearch-project · Sep 11, 2024 · cbaf0d2 · cbaf0d2
2 parents 4d67722 + fa6de0d
commit cbaf0d2
Show file tree

Hide file tree

Showing 8 changed files with 77 additions and 18 deletions.
diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml
@@ -32,7 +32,10 @@ jobs:
         if: runner.os == 'macos'
         continue-on-error: true
         run: |
-          brew install docker colima coreutils
+          # Force QEMU 9.0.2 usage
+          curl https://raw.githubusercontent.com/Homebrew/homebrew-core/f1a9cf104a9a51779c7a532b658c490f69974839/Formula/q/qemu.rb > qemu.rb
+          brew install qemu.rb
+          HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 HOMEBREW_NO_AUTO_UPDATE=1 brew install docker colima coreutils
           gtimeout 15m colima start
         shell: bash
       - name: Run Gradle (assemble)

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578))
 - Bump `protobuf` from 3.22.3 to 3.25.4 ([#15684](https://github.com/opensearch-project/OpenSearch/pull/15684))
 - Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863))
+- Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862))
 
 ### Changed
 
@@ -20,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Removed
 
 ### Fixed
+- Fix wildcard query containing escaped character ([#15737](https://github.com/opensearch-project/OpenSearch/pull/15737))
 
 ### Security
 

diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -11,7 +11,7 @@
 
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.1-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=682b4df7fe5accdca84a4d1ef6a3a6ab096b3efd5edf7de2bd8c758d95a93703
+distributionSha256Sum=fdfca5dbc2834f0ece5020465737538e5ba679deeff5ab6c09621d67f8bb1a15
diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
@@ -62,7 +62,7 @@ dependencies {
   api 'com.microsoft.azure:msal4j-persistence-extension:1.3.0'
   api "net.java.dev.jna:jna-platform:${versions.jna}"
   api 'com.microsoft.azure:msal4j:1.17.0'
-  api 'com.nimbusds:oauth2-oidc-sdk:11.9.1'
+  api 'com.nimbusds:oauth2-oidc-sdk:11.19.1'
   api 'com.nimbusds:nimbus-jose-jwt:9.40'
   api 'com.nimbusds:content-type:2.3'
   api 'com.nimbusds:lang-tag:1.7'

diff --git a/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.19.1.jar.sha1 b/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.19.1.jar.sha1
@@ -0,0 +1 @@
+58db85a807a56ae76baffa519772271ad5808195
diff --git a/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 b/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1
diff --git a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java
@@ -40,7 +40,6 @@
 import org.apache.lucene.util.automaton.RegExp;
 import org.opensearch.common.lucene.BytesRefs;
 import org.opensearch.common.lucene.Lucene;
-import org.opensearch.common.regex.Regex;
 import org.opensearch.common.unit.Fuzziness;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.index.analysis.IndexAnalyzers;
@@ -430,22 +429,27 @@ public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, bo
                 finalValue = value;
             }
             Predicate<String> matchPredicate;
-            if (value.contains("?")) {
-                Automaton automaton = WildcardQuery.toAutomaton(new Term(name(), finalValue));
-                CompiledAutomaton compiledAutomaton = new CompiledAutomaton(automaton);
+            Automaton automaton = WildcardQuery.toAutomaton(new Term(name(), finalValue));
+            CompiledAutomaton compiledAutomaton = new CompiledAutomaton(automaton);
+            if (compiledAutomaton.type == CompiledAutomaton.AUTOMATON_TYPE.SINGLE) {
+                // when type equals SINGLE, #compiledAutomaton.runAutomaton is null
                 matchPredicate = s -> {
                     if (caseInsensitive) {
                         s = s.toLowerCase(Locale.ROOT);
                     }
-                    BytesRef valueBytes = BytesRefs.toBytesRef(s);
-                    return compiledAutomaton.runAutomaton.run(valueBytes.bytes, valueBytes.offset, valueBytes.length);
+                    return s.equals(finalValue);
                 };
+            } else if (compiledAutomaton.type == CompiledAutomaton.AUTOMATON_TYPE.ALL) {
+                return existsQuery(context);
+            } else if (compiledAutomaton.type == CompiledAutomaton.AUTOMATON_TYPE.NONE) {
+                return new MatchNoDocsQuery("Wildcard expression matches nothing");
             } else {
                 matchPredicate = s -> {
                     if (caseInsensitive) {
                         s = s.toLowerCase(Locale.ROOT);
                     }
-                    return Regex.simpleMatch(finalValue, s);
+                    BytesRef valueBytes = BytesRefs.toBytesRef(s);
+                    return compiledAutomaton.runAutomaton.run(valueBytes.bytes, valueBytes.offset, valueBytes.length);
                 };
             }
 
@@ -468,22 +472,30 @@ public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, bo
         // Package-private for testing
         static Set<String> getRequiredNGrams(String value) {
             Set<String> terms = new HashSet<>();
+
+            if (value.isEmpty()) {
+                return terms;
+            }
+
             int pos = 0;
+            String rawSequence = null;
             String currentSequence = null;
             if (!value.startsWith("?") && !value.startsWith("*")) {
                 // Can add prefix term
-                currentSequence = getNonWildcardSequence(value, 0);
+                rawSequence = getNonWildcardSequence(value, 0);
+                currentSequence = performEscape(rawSequence);
                 if (currentSequence.length() == 1) {
                     terms.add(new String(new char[] { 0, currentSequence.charAt(0) }));
                 } else {
                     terms.add(new String(new char[] { 0, currentSequence.charAt(0), currentSequence.charAt(1) }));
                 }
             } else {
                 pos = findNonWildcardSequence(value, pos);
-                currentSequence = getNonWildcardSequence(value, pos);
+                rawSequence = getNonWildcardSequence(value, pos);
             }
             while (pos < value.length()) {
-                boolean isEndOfValue = pos + currentSequence.length() == value.length();
+                boolean isEndOfValue = pos + rawSequence.length() == value.length();
+                currentSequence = performEscape(rawSequence);
                 if (!currentSequence.isEmpty() && currentSequence.length() < 3 && !isEndOfValue && pos > 0) {
                     // If this is a prefix or suffix of length < 3, then we already have a longer token including the anchor.
                     terms.add(currentSequence);
@@ -502,16 +514,16 @@ static Set<String> getRequiredNGrams(String value) {
                         terms.add(new String(new char[] { a, b, 0 }));
                     }
                 }
-                pos = findNonWildcardSequence(value, pos + currentSequence.length());
-                currentSequence = getNonWildcardSequence(value, pos);
+                pos = findNonWildcardSequence(value, pos + rawSequence.length());
+                rawSequence = getNonWildcardSequence(value, pos);
             }
             return terms;
         }
 
         private static String getNonWildcardSequence(String value, int startFrom) {
             for (int i = startFrom; i < value.length(); i++) {
                 char c = value.charAt(i);
-                if (c == '?' || c == '*') {
+                if ((c == '?' || c == '*') && (i == 0 || value.charAt(i - 1) != '\\')) {
                     return value.substring(startFrom, i);
                 }
             }
@@ -529,6 +541,22 @@ private static int findNonWildcardSequence(String value, int startFrom) {
             return value.length();
         }
 
+        private static String performEscape(String str) {
+            StringBuilder sb = new StringBuilder();
+            for (int i = 0; i < str.length(); i++) {
+                if (str.charAt(i) == '\\' && (i + 1) < str.length()) {
+                    char c = str.charAt(i + 1);
+                    if (c == '*' || c == '?') {
+                        i++;
+                    }
+                }
+                sb.append(str.charAt(i));
+            }
+            assert !sb.toString().contains("\\*");
+            assert !sb.toString().contains("\\?");
+            return sb.toString();
+        }
+
         @Override
         public Query regexpQuery(
             String value,

diff --git a/server/src/test/java/org/opensearch/index/mapper/WildcardFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/WildcardFieldTypeTests.java
@@ -88,6 +88,32 @@ public void testWildcardQuery() {
         );
     }
 
+    public void testEscapedWildcardQuery() {
+        MappedFieldType ft = new WildcardFieldMapper.WildcardFieldType("field");
+        Set<String> expectedTerms = new HashSet<>();
+        expectedTerms.add(prefixAnchored("*"));
+        expectedTerms.add(suffixAnchored("*"));
+
+        BooleanQuery.Builder builder = new BooleanQuery.Builder();
+        for (String term : expectedTerms) {
+            builder.add(new TermQuery(new Term("field", term)), BooleanClause.Occur.FILTER);
+        }
+
+        assertEquals(
+            new WildcardFieldMapper.WildcardMatchingQuery("field", builder.build(), "\\**\\*"),
+            ft.wildcardQuery("\\**\\*", null, null)
+        );
+
+        assertEquals(new WildcardFieldMapper.WildcardMatchingQuery("field", builder.build(), "\\*"), ft.wildcardQuery("\\*", null, null));
+
+        expectedTerms.remove(suffixAnchored("*"));
+        builder = new BooleanQuery.Builder();
+        for (String term : expectedTerms) {
+            builder.add(new TermQuery(new Term("field", term)), BooleanClause.Occur.FILTER);
+        }
+        assertEquals(new WildcardFieldMapper.WildcardMatchingQuery("field", builder.build(), "\\**"), ft.wildcardQuery("\\**", null, null));
+    }
+
     public void testMultipleWildcardsInQuery() {
         final String pattern = "a?cd*efg?h";
         MappedFieldType ft = new WildcardFieldMapper.WildcardFieldType("field");