Skip to content

Commit

Permalink
Merge branch 'main' into primary-rebalacing
Browse files Browse the repository at this point in the history
Signed-off-by: Arpit-Bandejiya <abandeji@amazon.com>
  • Loading branch information
Arpit-Bandejiya committed Apr 2, 2024
2 parents e823b85 + 2dc071f commit 9f94ba5
Show file tree
Hide file tree
Showing 12 changed files with 549 additions and 1 deletion.
24 changes: 24 additions & 0 deletions .github/workflows/detect-breaking-change.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
name: "Detect Breaking Changes"
on: [push, pull_request]
jobs:
detect-breaking-change:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
with:
distribution: temurin # Temurin is a distribution of adoptium
java-version: 21
- uses: gradle/gradle-build-action@v3
with:
arguments: japicmp
gradle-version: 8.7
build-root-directory: server
- if: failure()
run: cat server/build/reports/java-compatibility/report.txt
- if: failure()
uses: actions/upload-artifact@v4
with:
name: java-compatibility-report.html
path: ${{ github.workspace }}/server/build/reports/java-compatibility/report.html

3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -103,11 +103,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),

## [Unreleased 2.x]
### Added
- Constant Keyword Field ([#12285](https://github.com/opensearch-project/OpenSearch/pull/12285))
- Convert ingest processor supports ip type ([#12818](https://github.com/opensearch-project/OpenSearch/pull/12818))
- Add a counter to node stat api to track shard going from idle to non-idle ([#12768](https://github.com/opensearch-project/OpenSearch/pull/12768))
- Allow setting KEYSTORE_PASSWORD through env variable ([#12865](https://github.com/opensearch-project/OpenSearch/pull/12865))
- [Concurrent Segment Search] Perform buildAggregation concurrently and support Composite Aggregations ([#12697](https://github.com/opensearch-project/OpenSearch/pull/12697))
- [Concurrent Segment Search] Disable concurrent segment search for system indices and throttled requests ([#12954](https://github.com/opensearch-project/OpenSearch/pull/12954))
- Detect breaking changes on pull requests ([#9044](https://github.com/opensearch-project/OpenSearch/pull/9044))
- Add cluster primary balance contraint for rebalancing with buffer ([#12656](https://github.com/opensearch-project/OpenSearch/pull/12656))

### Dependencies
Expand All @@ -127,6 +129,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),

### Fixed
- Fix issue with feature flags where default value may not be honored ([#12849](https://github.com/opensearch-project/OpenSearch/pull/12849))
- Fix UOE While building Exists query for nested search_as_you_type field ([#12048](https://github.com/opensearch-project/OpenSearch/pull/12048))

### Security

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import org.apache.lucene.search.DisjunctionMaxQuery;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.MultiPhraseQuery;
import org.apache.lucene.search.NormsFieldExistsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SynonymQuery;
import org.apache.lucene.search.TermQuery;
Expand All @@ -68,6 +69,7 @@
import org.opensearch.index.query.MatchPhraseQueryBuilder;
import org.opensearch.index.query.MultiMatchQueryBuilder;
import org.opensearch.index.query.QueryShardContext;
import org.opensearch.index.query.QueryStringQueryBuilder;
import org.opensearch.plugins.Plugin;

import java.io.IOException;
Expand Down Expand Up @@ -541,6 +543,31 @@ public void testMatchPhrase() throws IOException {
}
}

public void testNestedExistsQuery() throws IOException {
MapperService mapperService = createMapperService(mapping(b -> {
b.startObject("field");
{
b.field("type", "object");
b.startObject("properties");
{
b.startObject("nested_field");
{
b.field("type", "search_as_you_type");
}
b.endObject();
}
b.endObject();
}
b.endObject();
}));
QueryShardContext queryShardContext = createQueryShardContext(mapperService);
Query actual = new QueryStringQueryBuilder("field:*").toQuery(queryShardContext);
Query expected = new ConstantScoreQuery(
new BooleanQuery.Builder().add(new NormsFieldExistsQuery("field.nested_field"), BooleanClause.Occur.SHOULD).build()
);
assertEquals(expected, actual);
}

private static BooleanQuery buildBoolPrefixQuery(String shingleFieldName, String prefixFieldName, List<String> terms) {
final BooleanQuery.Builder builder = new BooleanQuery.Builder();
for (int i = 0; i < terms.size() - 1; i++) {
Expand Down
79 changes: 79 additions & 0 deletions server/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ plugins {
id('opensearch.publish')
id('opensearch.internal-cluster-test')
id('opensearch.optional-dependencies')
id('me.champeau.gradle.japicmp') version '0.4.2'
}

publishing {
Expand Down Expand Up @@ -378,3 +379,81 @@ tasks.named("sourcesJar").configure {
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
}
}

/** Compares the current build against a snapshot build */
tasks.register("japicmp", me.champeau.gradle.japicmp.JapicmpTask) {
oldClasspath.from(files("${buildDir}/snapshot/opensearch-${version}.jar"))
newClasspath.from(tasks.named('jar'))
onlyModified = true
failOnModification = true
ignoreMissingClasses = true
annotationIncludes = ['@org.opensearch.common.annotation.PublicApi']
txtOutputFile = layout.buildDirectory.file("reports/java-compatibility/report.txt")
htmlOutputFile = layout.buildDirectory.file("reports/java-compatibility/report.html")
dependsOn downloadSnapshot
}

/** If the Java API Comparison task failed, print a hint if the change should be merged from its target branch */
gradle.taskGraph.afterTask { Task task, TaskState state ->
if (task.name == 'japicmp' && state.failure != null) {
def sha = getGitShaFromJar("${buildDir}/snapshot/opensearch-${version}.jar")
logger.info("Incompatiable java api from snapshot jar built off of commit ${sha}")

if (!inHistory(sha)) {
logger.warn('\u001B[33mPlease merge from the target branch and run this task again.\u001B[0m')
}
}
}

/** Downloads latest snapshot from maven repository */
tasks.register("downloadSnapshot", Copy) {
def mavenSnapshotRepoUrl = "https://aws.oss.sonatype.org/content/repositories/snapshots/"
def groupId = "org.opensearch"
def artifactId = "opensearch"

repositories {
maven {
url mavenSnapshotRepoUrl
}
}

configurations {
snapshotArtifact
}

dependencies {
snapshotArtifact("${groupId}:${artifactId}:${version}:")
}

from configurations.snapshotArtifact
into "$buildDir/snapshot"
}

/** Check if the sha is in the current history */
def inHistory(String sha) {
try {
def commandCheckSha = "git merge-base --is-ancestor ${sha} HEAD"
commandCheckSha.execute()
return true
} catch (Exception) {
return false
}
}

/** Extracts the Git SHA used to build a jar from its manifest */
def getGitShaFromJar(String jarPath) {
def sha = ''
try {
// Open the JAR file
def jarFile = new java.util.jar.JarFile(jarPath)
// Get the manifest from the JAR file
def manifest = jarFile.manifest
def attributes = manifest.mainAttributes
// Assuming the Git SHA is stored under an attribute named 'Git-SHA'
sha = attributes.getValue('Change')
jarFile.close()
} catch (IOException e) {
println "Failed to read the JAR file: $e.message"
}
return sha
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.index.mapper;

import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.opensearch.OpenSearchParseException;
import org.opensearch.common.annotation.PublicApi;
import org.opensearch.common.regex.Regex;
import org.opensearch.index.fielddata.IndexFieldData;
import org.opensearch.index.fielddata.plain.ConstantIndexFieldData;
import org.opensearch.index.query.QueryShardContext;
import org.opensearch.search.aggregations.support.CoreValuesSourceType;
import org.opensearch.search.lookup.SearchLookup;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;

/**
* Index specific field mapper
*
* @opensearch.api
*/
@PublicApi(since = "2.14.0")
public class ConstantKeywordFieldMapper extends ParametrizedFieldMapper {

public static final String CONTENT_TYPE = "constant_keyword";

private static final String valuePropertyName = "value";

/**
* A {@link Mapper.TypeParser} for the constant keyword field.
*
* @opensearch.internal
*/
public static class TypeParser implements Mapper.TypeParser {
@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
if (!node.containsKey(valuePropertyName)) {
throw new OpenSearchParseException("Field [" + name + "] is missing required parameter [value]");
}
Object value = node.remove(valuePropertyName);
if (!(value instanceof String)) {
throw new OpenSearchParseException("Field [" + name + "] is expected to be a string value");
}
return new Builder(name, (String) value);
}
}

private static ConstantKeywordFieldMapper toType(FieldMapper in) {
return (ConstantKeywordFieldMapper) in;
}

/**
* Builder for the binary field mapper
*
* @opensearch.internal
*/
public static class Builder extends ParametrizedFieldMapper.Builder {

private final Parameter<String> value;

public Builder(String name, String value) {
super(name);
this.value = Parameter.stringParam(valuePropertyName, false, m -> toType(m).value, value);
}

@Override
public List<Parameter<?>> getParameters() {
return Arrays.asList(value);
}

@Override
public ConstantKeywordFieldMapper build(BuilderContext context) {
return new ConstantKeywordFieldMapper(
name,
new ConstantKeywordFieldMapper.ConstantKeywordFieldType(buildFullName(context), value.getValue()),
multiFieldsBuilder.build(this, context),
copyTo.build(),
this
);
}
}

/**
* Field type for Index field mapper
*
* @opensearch.internal
*/
@PublicApi(since = "2.14.0")
protected static final class ConstantKeywordFieldType extends ConstantFieldType {

protected final String value;

public ConstantKeywordFieldType(String name, String value) {
super(name, Collections.emptyMap());
this.value = value;
}

@Override
public String typeName() {
return CONTENT_TYPE;
}

@Override
protected boolean matches(String pattern, boolean caseInsensitive, QueryShardContext context) {
return Regex.simpleMatch(pattern, value, caseInsensitive);
}

@Override
public Query existsQuery(QueryShardContext context) {
return new MatchAllDocsQuery();
}

@Override
public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier<SearchLookup> searchLookup) {
return new ConstantIndexFieldData.Builder(fullyQualifiedIndexName, name(), CoreValuesSourceType.BYTES);
}

@Override
public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) {
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't " + "support formats.");
}

return new SourceValueFetcher(name(), context) {
@Override
protected Object parseSourceValue(Object value) {
String keywordValue = value.toString();
return Collections.singletonList(keywordValue);
}
};
}
}

private final String value;

protected ConstantKeywordFieldMapper(
String simpleName,
MappedFieldType mappedFieldType,
MultiFields multiFields,
CopyTo copyTo,
ConstantKeywordFieldMapper.Builder builder
) {
super(simpleName, mappedFieldType, multiFields, copyTo);
this.value = builder.value.getValue();
}

public ParametrizedFieldMapper.Builder getMergeBuilder() {
return new ConstantKeywordFieldMapper.Builder(simpleName(), this.value).init(this);
}

@Override
protected void parseCreateField(ParseContext context) throws IOException {

final String value;
if (context.externalValueSet()) {
value = context.externalValue().toString();
} else {
value = context.parser().textOrNull();
}
if (value == null) {
throw new IllegalArgumentException("constant keyword field [" + name() + "] must have a value");
}

if (!value.equals(fieldType().value)) {
throw new IllegalArgumentException("constant keyword field [" + name() + "] must have a value of [" + this.value + "]");
}

}

@Override
public ConstantKeywordFieldMapper.ConstantKeywordFieldType fieldType() {
return (ConstantKeywordFieldMapper.ConstantKeywordFieldType) super.fieldType();
}

@Override
protected String contentType() {
return CONTENT_TYPE;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,11 @@ private static Query newObjectFieldExistsQuery(QueryShardContext context, String
BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
Collection<String> fields = context.simpleMatchToIndexNames(objField + ".*");
for (String field : fields) {
int dotPos = field.lastIndexOf('.');
if (dotPos > 0 && field.charAt(dotPos + 1) == '_') {
// This is a subfield (e.g. prefix) of a complex field type. Skip it.
continue;
}
Query existsQuery = context.getMapperService().fieldType(field).existsQuery(context);
booleanQuery.add(existsQuery, Occur.SHOULD);
}
Expand Down
Loading

0 comments on commit 9f94ba5

Please sign in to comment.