Skip to content

Commit

Permalink
Merge remote-tracking branch 'es/ccr' into ccr_more_validation
Browse files Browse the repository at this point in the history 
* es/ccr: (37 commits)
  Default to one shard (elastic#30539)
  Unmute IndexUpgradeIT tests
  Forbid expensive query parts in ranking evaluation (elastic#30151)
  Docs: Update HighLevelRestClient migration docs (elastic#30544)
  Clients: Switch to new performRequest (elastic#30543)
  [TEST] Fix typo in MovAvgIT test
  Add missing dependencies on testClasses (elastic#30527)
  [TEST] Mute ML test that needs updating to following ml-cpp changes
  Document woes between auto-expand-replicas and allocation filtering (elastic#30531)
  Moved tokenizers to analysis common module (elastic#30538)
  Adjust copy settings versions
  Mute ShrinkIndexIT suite
  SQL: SYS TABLES ordered according to *DBC specs (elastic#30530)
  Deprecate not copy settings and explicitly disallow (elastic#30404)
  [ML] Improve state persistence log message
  Build: Add mavenPlugin cluster configuration method (elastic#30541)
  Re-enable FlushIT tests
  Bump Gradle heap to 2 GB (elastic#30535)
  SQL: Use request flavored methods in tests (elastic#30345)
  Suppress hdfsFixture if there are spaces in the path (elastic#30302)
  ...
  • Loading branch information
@martijnvg
martijnvg committed May 15, 2018
2 parents 80f5ccb + b971a81 commit 56e584a
Show file tree
Hide file tree
Showing 182 changed files with 2,106 additions and 1,391 deletions.
1 change: 1 addition & 0 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -744,6 +744,7 @@ class BuildPlugin implements Plugin<Project> {
additionalTest.testClassesDir = test.testClassesDir
additionalTest.configure(commonTestConfig(project))
additionalTest.configure(config)
additionalTest.dependsOn(project.tasks.testClasses)
test.dependsOn(additionalTest)
});
return test
Expand Down
1 change: 1 addition & 0 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
* warning every time. */
current.println(" - skip:")
current.println(" features: ")
current.println(" - default_shards")
current.println(" - stash_in_key")
current.println(" - stash_in_path")
current.println(" - stash_path_replace")
Expand Down
7 changes: 6 additions & 1 deletion buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ class ClusterConfiguration {
// map from destination path, to source file
Map<String, Object> extraConfigFiles = new HashMap<>()

LinkedHashMap<String, Project> plugins = new LinkedHashMap<>()
LinkedHashMap<String, Object> plugins = new LinkedHashMap<>()

List<Project> modules = new ArrayList<>()

Expand Down Expand Up @@ -185,6 +185,11 @@ class ClusterConfiguration {
plugins.put(pluginProject.name, pluginProject)
}

@Input
void mavenPlugin(String name, String mavenCoords) {
plugins.put(name, mavenCoords)
}

/** Add a module to the cluster. The project must be an esplugin and have a single zip default artifact. */
@Input
void module(Project moduleProject) {
Expand Down
103 changes: 61 additions & 42 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,8 @@ class ClusterFormationTasks {
// from mirrors using gradles built-in mechanism etc.

configureDistributionDependency(project, config.distribution, bwcDistro, config.bwcVersion)
for (Map.Entry<String, Project> entry : config.plugins.entrySet()) {
configureBwcPluginDependency("${prefix}_elasticsearchBwcPlugins", project, entry.getValue(), bwcPlugins, config.bwcVersion)
for (Map.Entry<String, Object> entry : config.plugins.entrySet()) {
configureBwcPluginDependency(project, entry.getValue(), bwcPlugins, config.bwcVersion)
}
bwcDistro.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS)
bwcPlugins.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS)
Expand Down Expand Up @@ -150,10 +150,15 @@ class ClusterFormationTasks {
}

/** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */
static void configureBwcPluginDependency(String name, Project project, Project pluginProject, Configuration configuration, Version elasticsearchVersion) {
verifyProjectHasBuildPlugin(name, elasticsearchVersion, project, pluginProject)
final String pluginName = findPluginName(pluginProject)
project.dependencies.add(configuration.name, "org.elasticsearch.plugin:${pluginName}:${elasticsearchVersion}@zip")
static void configureBwcPluginDependency(Project project, Object plugin, Configuration configuration, Version elasticsearchVersion) {
if (plugin instanceof Project) {
Project pluginProject = (Project)plugin
verifyProjectHasBuildPlugin(configuration.name, elasticsearchVersion, project, pluginProject)
final String pluginName = findPluginName(pluginProject)
project.dependencies.add(configuration.name, "org.elasticsearch.plugin:${pluginName}:${elasticsearchVersion}@zip")
} else {
project.dependencies.add(configuration.name, "${plugin}@zip")
}
}

/**
Expand Down Expand Up @@ -210,9 +215,9 @@ class ClusterFormationTasks {
}

// install plugins
for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
String actionName = pluginTaskName('install', plugin.getKey(), 'Plugin')
setup = configureInstallPluginTask(taskName(prefix, node, actionName), project, setup, node, plugin.getValue(), prefix)
for (String pluginName : node.config.plugins.keySet()) {
String actionName = pluginTaskName('install', pluginName, 'Plugin')
setup = configureInstallPluginTask(taskName(prefix, node, actionName), project, setup, node, pluginName, prefix)
}

// sets up any extra config files that need to be copied over to the ES instance;
Expand Down Expand Up @@ -444,31 +449,40 @@ class ClusterFormationTasks {
Copy copyPlugins = project.tasks.create(name: name, type: Copy, dependsOn: setup)

List<FileCollection> pluginFiles = []
for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
for (Map.Entry<String, Object> plugin : node.config.plugins.entrySet()) {

Project pluginProject = plugin.getValue()
verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)
String configurationName = pluginConfigurationName(prefix, pluginProject)
String configurationName = pluginConfigurationName(prefix, plugin.key)
Configuration configuration = project.configurations.findByName(configurationName)
if (configuration == null) {
configuration = project.configurations.create(configurationName)
}
project.dependencies.add(configurationName, project.dependencies.project(path: pluginProject.path, configuration: 'zip'))
setup.dependsOn(pluginProject.tasks.bundlePlugin)

// also allow rest tests to use the rest spec from the plugin
String copyRestSpecTaskName = pluginTaskName('copy', plugin.getKey(), 'PluginRestSpec')
Copy copyRestSpec = project.tasks.findByName(copyRestSpecTaskName)
for (File resourceDir : pluginProject.sourceSets.test.resources.srcDirs) {
File restApiDir = new File(resourceDir, 'rest-api-spec/api')
if (restApiDir.exists() == false) continue
if (copyRestSpec == null) {
copyRestSpec = project.tasks.create(name: copyRestSpecTaskName, type: Copy)
copyPlugins.dependsOn(copyRestSpec)
copyRestSpec.into(project.sourceSets.test.output.resourcesDir)

if (plugin.getValue() instanceof Project) {
Project pluginProject = plugin.getValue()
verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)

project.dependencies.add(configurationName, project.dependencies.project(path: pluginProject.path, configuration: 'zip'))
setup.dependsOn(pluginProject.tasks.bundlePlugin)

// also allow rest tests to use the rest spec from the plugin
String copyRestSpecTaskName = pluginTaskName('copy', plugin.getKey(), 'PluginRestSpec')
Copy copyRestSpec = project.tasks.findByName(copyRestSpecTaskName)
for (File resourceDir : pluginProject.sourceSets.test.resources.srcDirs) {
File restApiDir = new File(resourceDir, 'rest-api-spec/api')
if (restApiDir.exists() == false) continue
if (copyRestSpec == null) {
copyRestSpec = project.tasks.create(name: copyRestSpecTaskName, type: Copy)
copyPlugins.dependsOn(copyRestSpec)
copyRestSpec.into(project.sourceSets.test.output.resourcesDir)
}
copyRestSpec.from(resourceDir).include('rest-api-spec/api/**')
}
copyRestSpec.from(resourceDir).include('rest-api-spec/api/**')
} else {
project.dependencies.add(configurationName, "${plugin.getValue()}@zip")
}



pluginFiles.add(configuration)
}

Expand All @@ -477,32 +491,37 @@ class ClusterFormationTasks {
return copyPlugins
}

private static String pluginConfigurationName(final String prefix, final Project project) {
return "_plugin_${prefix}_${project.path}".replace(':', '_')
private static String pluginConfigurationName(final String prefix, final String name) {
return "_plugin_${prefix}_${name}".replace(':', '_')
}

private static String pluginBwcConfigurationName(final String prefix, final Project project) {
return "_plugin_bwc_${prefix}_${project.path}".replace(':', '_')
private static String pluginBwcConfigurationName(final String prefix, final String name) {
return "_plugin_bwc_${prefix}_${name}".replace(':', '_')
}

/** Configures task to copy a plugin based on a zip file resolved using dependencies for an older version */
static Task configureCopyBwcPluginsTask(String name, Project project, Task setup, NodeInfo node, String prefix) {
Configuration bwcPlugins = project.configurations.getByName("${prefix}_elasticsearchBwcPlugins")
for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
Project pluginProject = plugin.getValue()
verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)
String configurationName = pluginBwcConfigurationName(prefix, pluginProject)
for (Map.Entry<String, Object> plugin : node.config.plugins.entrySet()) {
String configurationName = pluginBwcConfigurationName(prefix, plugin.key)
Configuration configuration = project.configurations.findByName(configurationName)
if (configuration == null) {
configuration = project.configurations.create(configurationName)
}

final String depName = findPluginName(pluginProject)
if (plugin.getValue() instanceof Project) {
Project pluginProject = plugin.getValue()
verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)

Dependency dep = bwcPlugins.dependencies.find {
it.name == depName
final String depName = findPluginName(pluginProject)

Dependency dep = bwcPlugins.dependencies.find {
it.name == depName
}
configuration.dependencies.add(dep)
} else {
project.dependencies.add(configurationName, "${plugin.getValue()}@zip")
}
configuration.dependencies.add(dep)
}

Copy copyPlugins = project.tasks.create(name: name, type: Copy, dependsOn: setup) {
Expand All @@ -527,12 +546,12 @@ class ClusterFormationTasks {
return installModule
}

static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, Project plugin, String prefix) {
static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, String pluginName, String prefix) {
final FileCollection pluginZip;
if (node.nodeVersion != VersionProperties.elasticsearch) {
pluginZip = project.configurations.getByName(pluginBwcConfigurationName(prefix, plugin))
pluginZip = project.configurations.getByName(pluginBwcConfigurationName(prefix, pluginName))
} else {
pluginZip = project.configurations.getByName(pluginConfigurationName(prefix, plugin))
pluginZip = project.configurations.getByName(pluginConfigurationName(prefix, pluginName))
}
// delay reading the file location until execution time by wrapping in a closure within a GString
final Object file = "${-> new File(node.pluginsTmpDir, pluginZip.singleFile.getName()).toURI().toURL().toString()}"
Expand Down
20 changes: 7 additions & 13 deletions .../benchmark/src/main/java/org/elasticsearch/client/benchmark/rest/RestClientBenchmark.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,27 +18,19 @@
*/
package org.elasticsearch.client.benchmark.rest;

import org.apache.http.HttpEntity;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.conn.ConnectionKeepAliveStrategy;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.apache.http.message.BasicHeader;
import org.apache.http.nio.entity.NStringEntity;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.benchmark.AbstractBenchmark;
import org.elasticsearch.client.benchmark.ops.bulk.BulkRequestExecutor;
import org.elasticsearch.client.benchmark.ops.search.SearchRequestExecutor;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
Expand Down Expand Up @@ -86,9 +78,10 @@ public boolean bulkIndex(List<String> bulkData) {
bulkRequestBody.append(bulkItem);
bulkRequestBody.append("\n");
}
HttpEntity entity = new NStringEntity(bulkRequestBody.toString(), ContentType.APPLICATION_JSON);
Request request = new Request("POST", "/geonames/type/_noop_bulk");
request.setJsonEntity(bulkRequestBody.toString());
try {
Response response = client.performRequest("POST", "/geonames/type/_noop_bulk", Collections.emptyMap(), entity);
Response response = client.performRequest(request);
return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK;
} catch (Exception e) {
throw new ElasticsearchException(e);
Expand All @@ -107,9 +100,10 @@ private RestSearchRequestExecutor(RestClient client, String indexName) {

@Override
public boolean search(String source) {
HttpEntity searchBody = new NStringEntity(source, StandardCharsets.UTF_8);
Request request = new Request("GET", endpoint);
request.setJsonEntity(source);
try {
Response response = client.performRequest("GET", endpoint, Collections.emptyMap(), searchBody);
Response response = client.performRequest(request);
return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK;
} catch (IOException e) {
throw new ElasticsearchException(e);
Expand Down
10 changes: 4 additions & 6 deletions client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,18 +194,16 @@ public void testBulkProcessorWaitOnClose() throws Exception {
}

public void testBulkProcessorConcurrentRequestsReadOnlyIndex() throws Exception {

String createIndexBody = "{\n" +
Request request = new Request("PUT", "/test-ro");
request.setJsonEntity("{\n" +
" \"settings\" : {\n" +
" \"index\" : {\n" +
" \"blocks.write\" : true\n" +
" }\n" +
" }\n" +
" \n" +
"}";

NStringEntity entity = new NStringEntity(createIndexBody, ContentType.APPLICATION_JSON);
Response response = client().performRequest("PUT", "/test-ro", Collections.emptyMap(), entity);
"}");
Response response = client().performRequest(request);
assertThat(response.getStatusLine().getStatusCode(), equalTo(200));

int bulkActions = randomIntBetween(10, 100);
Expand Down
43 changes: 18 additions & 25 deletions client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,6 @@

package org.elasticsearch.client;

import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.action.DocWriteRequest;
Expand All @@ -39,6 +36,7 @@
import org.elasticsearch.action.get.MultiGetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.common.Strings;
Expand Down Expand Up @@ -147,11 +145,10 @@ public void testExists() throws IOException {
GetRequest getRequest = new GetRequest("index", "type", "id");
assertFalse(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync));
}
String document = "{\"field1\":\"value1\",\"field2\":\"value2\"}";
StringEntity stringEntity = new StringEntity(document, ContentType.APPLICATION_JSON);
Response response = client().performRequest(HttpPut.METHOD_NAME, "/index/type/id", Collections.singletonMap("refresh", "wait_for"),
stringEntity);
assertEquals(201, response.getStatusLine().getStatusCode());
IndexRequest index = new IndexRequest("index", "type", "id");
index.source("{\"field1\":\"value1\",\"field2\":\"value2\"}", XContentType.JSON);
index.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
highLevelClient().index(index);
{
GetRequest getRequest = new GetRequest("index", "type", "id");
assertTrue(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync));
Expand All @@ -175,12 +172,11 @@ public void testGet() throws IOException {
assertEquals("Elasticsearch exception [type=index_not_found_exception, reason=no such index]", exception.getMessage());
assertEquals("index", exception.getMetadata("es.index").get(0));
}

IndexRequest index = new IndexRequest("index", "type", "id");
String document = "{\"field1\":\"value1\",\"field2\":\"value2\"}";
StringEntity stringEntity = new StringEntity(document, ContentType.APPLICATION_JSON);
Response response = client().performRequest(HttpPut.METHOD_NAME, "/index/type/id", Collections.singletonMap("refresh", "wait_for"),
stringEntity);
assertEquals(201, response.getStatusLine().getStatusCode());
index.source(document, XContentType.JSON);
index.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
highLevelClient().index(index);
{
GetRequest getRequest = new GetRequest("index", "type", "id").version(2);
ElasticsearchException exception = expectThrows(ElasticsearchException.class,
Expand Down Expand Up @@ -271,18 +267,15 @@ public void testMultiGet() throws IOException {
assertEquals("Elasticsearch exception [type=index_not_found_exception, reason=no such index]",
response.getResponses()[1].getFailure().getFailure().getMessage());
}

String document = "{\"field\":\"value1\"}";
StringEntity stringEntity = new StringEntity(document, ContentType.APPLICATION_JSON);
Response r = client().performRequest(HttpPut.METHOD_NAME, "/index/type/id1", Collections.singletonMap("refresh", "true"),
stringEntity);
assertEquals(201, r.getStatusLine().getStatusCode());

document = "{\"field\":\"value2\"}";
stringEntity = new StringEntity(document, ContentType.APPLICATION_JSON);
r = client().performRequest(HttpPut.METHOD_NAME, "/index/type/id2", Collections.singletonMap("refresh", "true"), stringEntity);
assertEquals(201, r.getStatusLine().getStatusCode());

BulkRequest bulk = new BulkRequest();
bulk.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
IndexRequest index = new IndexRequest("index", "type", "id1");
index.source("{\"field\":\"value1\"}", XContentType.JSON);
bulk.add(index);
index = new IndexRequest("index", "type", "id2");
index.source("{\"field\":\"value2\"}", XContentType.JSON);
bulk.add(index);
highLevelClient().bulk(bulk);
{
MultiGetRequest multiGetRequest = new MultiGetRequest();
multiGetRequest.add("index", "type", "id1");
Expand Down
Loading

0 comments on commit 56e584a

Please sign in to comment.