From 92b07cd0f010f33d812a7eacacde0ed42549750c Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 18 Jul 2017 10:36:48 +0200 Subject: [PATCH 01/11] move tribe to its own module --- .../resources/checkstyle_suppressions.xml | 2 +- .../cluster/MergableCustomMetaData.java | 43 +++++ .../common/settings/ClusterSettings.java | 9 - .../common/settings/SettingsModule.java | 21 +-- .../discovery/DiscoveryModule.java | 1 - .../java/org/elasticsearch/node/Node.java | 49 +++--- modules/tribe/build.gradle | 26 +++ .../elasticsearch/tribe}/TribeDiscovery.java | 5 +- .../org/elasticsearch/tribe/TribePlugin.java | 159 ++++++++++++++++++ .../org/elasticsearch/tribe/TribeService.java | 83 ++------- .../java/org/elasticsearch/tribe/TribeIT.java | 42 ++++- .../tribe/TribeServiceTests.java | 17 +- settings.gradle | 1 + .../java/org/elasticsearch/node/MockNode.java | 2 +- .../elasticsearch/test/ESIntegTestCase.java | 14 +- .../test/discovery/TestZenDiscovery.java | 2 +- 16 files changed, 333 insertions(+), 143 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/cluster/MergableCustomMetaData.java create mode 100644 modules/tribe/build.gradle rename {core/src/main/java/org/elasticsearch/discovery => modules/tribe/src/main/java/org/elasticsearch/tribe}/TribeDiscovery.java (94%) create mode 100644 modules/tribe/src/main/java/org/elasticsearch/tribe/TribePlugin.java rename {core => modules/tribe}/src/main/java/org/elasticsearch/tribe/TribeService.java (87%) rename {core => modules/tribe}/src/test/java/org/elasticsearch/tribe/TribeIT.java (94%) rename {core => modules/tribe}/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java (94%) diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index be73b64b0b396..ed1940954b4e1 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -418,7 +418,7 @@ - + diff --git a/core/src/main/java/org/elasticsearch/cluster/MergableCustomMetaData.java b/core/src/main/java/org/elasticsearch/cluster/MergableCustomMetaData.java new file mode 100644 index 0000000000000..2a0a79ae80a37 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/cluster/MergableCustomMetaData.java @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster; + +import org.elasticsearch.cluster.metadata.MetaData; + +/** + * Interface to allow merging {@link org.elasticsearch.cluster.metadata.MetaData.Custom}. + * When multiple Mergable Custom metadata of the same type are found (from underlying clusters), the + * Custom metadata can be merged using {@link #merge(MetaData.Custom)}. + * + * @param type of custom meta data + */ +public interface MergableCustomMetaData { + + /** + * Merges this custom metadata with other, returning either this or other custom metadata. + * This method should not mutate either this or the other custom metadata. + * + * @param other custom meta data + * @return the same instance or other custom metadata based on implementation + * if both the instances are considered equal, implementations should return this + * instance to avoid redundant cluster state changes. + */ + T merge(T other); +} diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index 1e6b8b5b444ff..e0665ae2c8223 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -71,7 +71,6 @@ import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.indices.store.IndicesStore; -import org.elasticsearch.ingest.IngestService; import org.elasticsearch.monitor.fs.FsService; import org.elasticsearch.monitor.jvm.JvmGcMonitorService; import org.elasticsearch.monitor.jvm.JvmService; @@ -91,7 +90,6 @@ import org.elasticsearch.transport.TcpTransport; import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.tribe.TribeService; import org.elasticsearch.watcher.ResourceWatcherService; import java.util.Arrays; @@ -369,13 +367,6 @@ public void apply(Settings value, Settings current, Settings previous) { ThreadContext.DEFAULT_HEADERS_SETTING, ESLoggerFactory.LOG_DEFAULT_LEVEL_SETTING, ESLoggerFactory.LOG_LEVEL_SETTING, - TribeService.BLOCKS_METADATA_SETTING, - TribeService.BLOCKS_WRITE_SETTING, - TribeService.BLOCKS_WRITE_INDICES_SETTING, - TribeService.BLOCKS_READ_INDICES_SETTING, - TribeService.BLOCKS_METADATA_INDICES_SETTING, - TribeService.ON_CONFLICT_SETTING, - TribeService.TRIBE_NAME_SETTING, NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING, NodeEnvironment.ENABLE_LUCENE_SEGMENT_INFOS_TRACE_SETTING, OsService.REFRESH_INTERVAL_SETTING, diff --git a/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java b/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java index 51ee7ed5f44e8..45b511e1cc168 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java +++ b/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.tribe.TribeService; import java.io.IOException; import java.util.Arrays; @@ -49,8 +48,6 @@ public class SettingsModule implements Module { private final Set settingsFilterPattern = new HashSet<>(); private final Map> nodeSettings = new HashMap<>(); private final Map> indexSettings = new HashMap<>(); - private static final Predicate TRIBE_CLIENT_NODE_SETTINGS_PREDICATE = (s) -> s.startsWith("tribe.") - && TribeService.TRIBE_SETTING_KEYS.contains(s) == false; private final Logger logger; private final IndexScopedSettings indexScopedSettings; private final ClusterSettings clusterSettings; @@ -135,9 +132,7 @@ public SettingsModule(Settings settings, List> additionalSettings, Li } } // by now we are fully configured, lets check node level settings for unregistered index settings - final Predicate acceptOnlyClusterSettings = TRIBE_CLIENT_NODE_SETTINGS_PREDICATE.negate(); - clusterSettings.validate(settings.filter(acceptOnlyClusterSettings)); - validateTribeSettings(settings, clusterSettings); + clusterSettings.validate(settings); this.settingsFilter = new SettingsFilter(settings, settingsFilterPattern); } @@ -195,20 +190,6 @@ private void registerSettingsFilter(String filter) { settingsFilterPattern.add(filter); } - private void validateTribeSettings(Settings settings, ClusterSettings clusterSettings) { - Map groups = settings.filter(TRIBE_CLIENT_NODE_SETTINGS_PREDICATE).getGroups("tribe.", true); - for (Map.Entry tribeSettings : groups.entrySet()) { - Settings thisTribesSettings = tribeSettings.getValue(); - for (Map.Entry entry : thisTribesSettings.getAsMap().entrySet()) { - try { - clusterSettings.validate(entry.getKey(), thisTribesSettings); - } catch (IllegalArgumentException ex) { - throw new IllegalArgumentException("tribe." + tribeSettings.getKey() +" validation failed: "+ ex.getMessage(), ex); - } - } - } - } - public Settings getSettings() { return settings; } diff --git a/core/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java b/core/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java index c410cb88d691b..0ecf40e65a1ba 100644 --- a/core/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java +++ b/core/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java @@ -86,7 +86,6 @@ public DiscoveryModule(Settings settings, ThreadPool threadPool, TransportServic discoveryTypes.put("zen", () -> new ZenDiscovery(settings, threadPool, transportService, namedWriteableRegistry, masterService, clusterApplier, clusterSettings, hostsProvider, allocationService)); - discoveryTypes.put("tribe", () -> new TribeDiscovery(settings, transportService, masterService, clusterApplier)); discoveryTypes.put("single-node", () -> new SingleNodeDiscovery(settings, transportService, masterService, clusterApplier)); for (DiscoveryPlugin plugin : plugins) { plugin.getDiscoveryTypes(threadPool, transportService, namedWriteableRegistry, diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java index 201513373b8ae..efe11eadc1d85 100644 --- a/core/src/main/java/org/elasticsearch/node/Node.java +++ b/core/src/main/java/org/elasticsearch/node/Node.java @@ -133,7 +133,6 @@ import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportInterceptor; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.tribe.TribeService; import org.elasticsearch.usage.UsageService; import org.elasticsearch.watcher.ResourceWatcherService; @@ -153,6 +152,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -229,6 +229,7 @@ public static final Settings addNodeNameIfNeeded(Settings settings, final String private final Collection pluginLifecycleComponents; private final LocalNodeFactory localNodeFactory; private final NodeService nodeService; + private final List onStartedListeners = new CopyOnWriteArrayList<>(); /** * Constructs a node with the given settings. @@ -256,8 +257,6 @@ protected Node(final Environment environment, Collection Settings tmpSettings = Settings.builder().put(environment.settings()) .put(Client.CLIENT_TYPE_SETTING_S.getKey(), CLIENT_TYPE).build(); - tmpSettings = TribeService.processSettings(tmpSettings); - // create the node environment as soon as possible, to recover the node id and enable logging try { nodeEnvironment = new NodeEnvironment(tmpSettings, environment); @@ -385,15 +384,6 @@ protected Node(final Environment environment, Collection .flatMap(p -> p.getNamedXContent().stream()), ClusterModule.getNamedXWriteables().stream()) .flatMap(Function.identity()).collect(toList())); - final TribeService tribeService = - new TribeService( - settings, - environment.configFile(), - clusterService, - nodeId, - namedWriteableRegistry, - (s, p) -> newTribeClientNode(s, classpathPlugins, p)); - resourcesToClose.add(tribeService); modules.add(new RepositoriesModule(this.environment, pluginsService.filterPlugins(RepositoryPlugin.class), xContentRegistry)); final MetaStateService metaStateService = new MetaStateService(settings, nodeEnvironment, xContentRegistry); final IndicesService indicesService = new IndicesService(settings, pluginsService, nodeEnvironment, xContentRegistry, @@ -449,6 +439,7 @@ protected Node(final Environment environment, Collection transportService, indicesService, pluginsService, circuitBreakerService, scriptModule.getScriptService(), httpServerTransport, ingestService, clusterService, settingsModule.getSettingsFilter()); modules.add(b -> { + b.bind(NodeBuilder.class).toInstance(new NodeBuilder(this, classpathPlugins)); b.bind(Node.class).toInstance(this); b.bind(NodeService.class).toInstance(nodeService); b.bind(NamedXContentRegistry.class).toInstance(xContentRegistry); @@ -458,7 +449,6 @@ protected Node(final Environment environment, Collection b.bind(Environment.class).toInstance(this.environment); b.bind(ThreadPool.class).toInstance(threadPool); b.bind(NodeEnvironment.class).toInstance(nodeEnvironment); - b.bind(TribeService.class).toInstance(tribeService); b.bind(ResourceWatcherService.class).toInstance(resourceWatcherService); b.bind(CircuitBreakerService.class).toInstance(circuitBreakerService); b.bind(BigArrays.class).toInstance(bigArrays); @@ -527,6 +517,10 @@ protected Node(final Environment environment, Collection } } + public void addOnStartedListener(Runnable runnable) { + onStartedListeners.add(runnable); + } + // visible for testing static void warnIfPreRelease(final Version version, final boolean isSnapshot, final Logger logger) { if (!version.isRelease() || isSnapshot) { @@ -612,10 +606,6 @@ public Node start() throws NodeValidationException { Discovery discovery = injector.getInstance(Discovery.class); clusterService.getMasterService().setClusterStatePublisher(discovery::publish); - // start before the cluster service since it adds/removes initial Cluster state blocks - final TribeService tribeService = injector.getInstance(TribeService.class); - tribeService.start(); - // Start the transport service now so the publish address will be added to the local disco node in ClusterService TransportService transportService = injector.getInstance(TransportService.class); transportService.getTaskManager().setTaskResultsService(injector.getInstance(TaskResultsService.class)); @@ -682,10 +672,10 @@ public void onTimeout(TimeValue timeout) { writePortsFile("transport", transport.boundAddress()); } - // start nodes now, after the http server, because it may take some time - tribeService.startNodes(); logger.info("started"); + onStartedListeners.forEach(Runnable::run); + return this; } @@ -696,7 +686,6 @@ private Node stop() { Logger logger = Loggers.getLogger(Node.class, NODE_NAME_SETTING.get(settings)); logger.info("stopping ..."); - injector.getInstance(TribeService.class).stop(); injector.getInstance(ResourceWatcherService.class).stop(); if (NetworkModule.HTTP_ENABLED.get(settings)) { injector.getInstance(HttpServerTransport.class).stop(); @@ -744,7 +733,6 @@ public synchronized void close() throws IOException { List toClose = new ArrayList<>(); StopWatch stopWatch = new StopWatch("node_close"); toClose.add(() -> stopWatch.start("tribe")); - toClose.add(injector.getInstance(TribeService.class)); toClose.add(() -> stopWatch.stop().start("node_service")); toClose.add(nodeService); toClose.add(() -> stopWatch.stop().start("http")); @@ -920,8 +908,23 @@ private List getCustomNameResolvers(List> classpathPlugins, Path configPath) { + public static class NodeBuilder { + + private final Node node; + private final Collection> classpathPlugins; + + public NodeBuilder(Node node, Collection> classpathPlugins) { + this.node = node; + this.classpathPlugins = classpathPlugins; + } + + public Node newNode(Settings settings, Path configPath) { + return node.newNode(settings, classpathPlugins, configPath); + } + } + + /** Constructs a new node based on the following settings. Overridden by tests */ + protected Node newNode(Settings settings, Collection> classpathPlugins, Path configPath) { return new Node(new Environment(settings, configPath), classpathPlugins); } diff --git a/modules/tribe/build.gradle b/modules/tribe/build.gradle new file mode 100644 index 0000000000000..59c99c1735659 --- /dev/null +++ b/modules/tribe/build.gradle @@ -0,0 +1,26 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'Tribe module' + classname 'org.elasticsearch.tribe.TribePlugin' +} + +compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked" +compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked" diff --git a/core/src/main/java/org/elasticsearch/discovery/TribeDiscovery.java b/modules/tribe/src/main/java/org/elasticsearch/tribe/TribeDiscovery.java similarity index 94% rename from core/src/main/java/org/elasticsearch/discovery/TribeDiscovery.java rename to modules/tribe/src/main/java/org/elasticsearch/tribe/TribeDiscovery.java index f3200be4562e8..d8e88e51e2232 100644 --- a/core/src/main/java/org/elasticsearch/discovery/TribeDiscovery.java +++ b/modules/tribe/src/main/java/org/elasticsearch/tribe/TribeDiscovery.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.discovery; +package org.elasticsearch.tribe; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterName; @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.service.MasterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.discovery.Discovery; import org.elasticsearch.discovery.single.SingleNodeDiscovery; import org.elasticsearch.transport.TransportService; @@ -37,7 +38,7 @@ import static org.elasticsearch.tribe.TribeService.TRIBE_WRITE_BLOCK; /** - * A {@link Discovery} implementation that is used by {@link org.elasticsearch.tribe.TribeService}. This implementation + * A {@link Discovery} implementation that is used by {@link TribeService}. This implementation * doesn't support any clustering features. Most notably {@link #startInitialJoin()} does nothing and * {@link #publish(ClusterChangedEvent, AckListener)} delegates state updates directly to the * {@link org.elasticsearch.cluster.service.ClusterApplierService}. diff --git a/modules/tribe/src/main/java/org/elasticsearch/tribe/TribePlugin.java b/modules/tribe/src/main/java/org/elasticsearch/tribe/TribePlugin.java new file mode 100644 index 0000000000000..6a7705b4a68bf --- /dev/null +++ b/modules/tribe/src/main/java/org/elasticsearch/tribe/TribePlugin.java @@ -0,0 +1,159 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.tribe; + +import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.cluster.routing.allocation.AllocationService; +import org.elasticsearch.cluster.service.ClusterApplier; +import org.elasticsearch.cluster.service.MasterService; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.component.LifecycleComponent; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.discovery.Discovery; +import org.elasticsearch.discovery.DiscoveryModule; +import org.elasticsearch.discovery.DiscoverySettings; +import org.elasticsearch.discovery.zen.UnicastHostsProvider; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.node.Node; +import org.elasticsearch.plugins.DiscoveryPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.function.Supplier; + +public class TribePlugin extends Plugin implements DiscoveryPlugin { + + private final Settings settings; + + public TribePlugin(Settings settings) { + this.settings = settings; + } + + + @Override + public Map> getDiscoveryTypes(ThreadPool threadPool, TransportService transportService, + NamedWriteableRegistry namedWriteableRegistry, + MasterService masterService, ClusterApplier clusterApplier, + ClusterSettings clusterSettings, UnicastHostsProvider hostsProvider, + AllocationService allocationService) { + return Collections.singletonMap("tribe", () -> new TribeDiscovery(settings, transportService, masterService, clusterApplier)); + } + + @Override + public Collection> getGuiceServiceClasses() { + return Collections.singleton(TribeService.class); + } + + @Override + public Settings additionalSettings() { + if (TribeService.TRIBE_NAME_SETTING.exists(settings) == false) { + Map nodesSettings = settings.getGroups("tribe", true); + if (nodesSettings.isEmpty()) { + return Settings.EMPTY; + } + Settings.Builder sb = Settings.builder(); + + if (Node.NODE_MASTER_SETTING.exists(settings)) { + if (Node.NODE_MASTER_SETTING.get(settings)) { + throw new IllegalArgumentException("node cannot be tribe as well as master node"); + } + } else { + sb.put(Node.NODE_MASTER_SETTING.getKey(), false); + } + if (Node.NODE_DATA_SETTING.exists(settings)) { + if (Node.NODE_DATA_SETTING.get(settings)) { + throw new IllegalArgumentException("node cannot be tribe as well as data node"); + } + } else { + sb.put(Node.NODE_DATA_SETTING.getKey(), false); + } + if (Node.NODE_INGEST_SETTING.exists(settings)) { + if (Node.NODE_INGEST_SETTING.get(settings)) { + throw new IllegalArgumentException("node cannot be tribe as well as ingest node"); + } + } else { + sb.put(Node.NODE_INGEST_SETTING.getKey(), false); + } + + if (!NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.exists(settings)) { + sb.put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), nodesSettings.size()); + } + sb.put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "tribe"); // there is a special discovery implementation for tribe + // nothing is going to be discovered, since no master will be elected + sb.put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), 0); + if (sb.get("cluster.name") == null) { + sb.put("cluster.name", "tribe_" + UUIDs.randomBase64UUID()); // make sure it won't join other tribe nodes in the same JVM + } + sb.put(TransportMasterNodeReadAction.FORCE_LOCAL_SETTING.getKey(), true); + + return sb.build(); + } else { + for (String s : settings.getAsMap().keySet()) { + if (s.startsWith("tribe.") && !s.equals(TribeService.TRIBE_NAME_SETTING.getKey())) { + throw new IllegalArgumentException("tribe cannot contain inner tribes: " + s); + } + } + } + return Settings.EMPTY; + } + + @Override + public List> getSettings() { + List> defaults = Arrays.asList( + TribeService.BLOCKS_METADATA_SETTING, + TribeService.BLOCKS_WRITE_SETTING, + TribeService.BLOCKS_WRITE_INDICES_SETTING, + TribeService.BLOCKS_READ_INDICES_SETTING, + TribeService.BLOCKS_METADATA_INDICES_SETTING, + TribeService.ON_CONFLICT_SETTING, + TribeService.TRIBE_NAME_SETTING + ); + Map nodesSettings = settings.getGroups("tribe", true); + if (nodesSettings.isEmpty()) { + return defaults; + } + List> allSettings = new ArrayList<>(defaults); + for (Map.Entry entry : nodesSettings.entrySet()) { + String prefix = "tribe." + entry.getKey() + "."; + if (TribeService.TRIBE_SETTING_KEYS.stream().anyMatch(s -> s.startsWith(prefix))) { + continue; + } + // create dummy setting just so that setting validation does not complain, these settings are going to be validated + // again by the SettingsModule of the nested tribe node. + Setting setting = Setting.prefixKeySetting(prefix, (key) -> new Setting<>(key, "", Function.identity(), + Setting.Property.Dynamic, Setting.Property.NodeScope)); + allSettings.add(setting); + } + + return allSettings; + } + +} diff --git a/core/src/main/java/org/elasticsearch/tribe/TribeService.java b/modules/tribe/src/main/java/org/elasticsearch/tribe/TribeService.java similarity index 87% rename from core/src/main/java/org/elasticsearch/tribe/TribeService.java rename to modules/tribe/src/main/java/org/elasticsearch/tribe/TribeService.java index b07e838653cc5..016d9b0091db6 100644 --- a/core/src/main/java/org/elasticsearch/tribe/TribeService.java +++ b/modules/tribe/src/main/java/org/elasticsearch/tribe/TribeService.java @@ -23,12 +23,12 @@ import org.apache.logging.log4j.util.Supplier; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.ClusterStateTaskConfig; import org.elasticsearch.cluster.ClusterStateTaskExecutor; +import org.elasticsearch.cluster.MergableCustomMetaData; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.block.ClusterBlocks; @@ -41,10 +41,10 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.hash.MurmurHash3; +import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -59,8 +59,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.discovery.DiscoveryModule; -import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.node.Node; @@ -68,7 +66,6 @@ import org.elasticsearch.transport.TcpTransport; import java.io.IOException; -import java.nio.file.Path; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -78,7 +75,6 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.Collectors; @@ -88,10 +84,6 @@ * The tribe service holds a list of node clients connected to a list of tribe members, and uses their * cluster state events to update this local node cluster state with the merged view of it. *

- * The {@link #processSettings(org.elasticsearch.common.settings.Settings)} method should be called before - * starting the node, so it will make sure to configure this current node properly with the relevant tribe node - * settings. - *

* The tribe node settings make sure the discovery used is "local", but with no master elected. This means no * write level master node operations will work ({@link org.elasticsearch.discovery.MasterNotDiscoveredException} * will be thrown), and state level metadata operations with automatically use the local flag. @@ -109,63 +101,6 @@ public class TribeService extends AbstractLifecycleComponent { public static final ClusterBlock TRIBE_WRITE_BLOCK = new ClusterBlock(11, "tribe node, write not allowed", false, false, false, RestStatus.BAD_REQUEST, EnumSet.of(ClusterBlockLevel.WRITE)); - public static Settings processSettings(Settings settings) { - if (TRIBE_NAME_SETTING.exists(settings)) { - // if its a node client started by this service as tribe, remove any tribe group setting - // to avoid recursive configuration - Settings.Builder sb = Settings.builder().put(settings); - for (String s : settings.getAsMap().keySet()) { - if (s.startsWith("tribe.") && !s.equals(TRIBE_NAME_SETTING.getKey())) { - sb.remove(s); - } - } - return sb.build(); - } - Map nodesSettings = settings.getGroups("tribe", true); - if (nodesSettings.isEmpty()) { - return settings; - } - // its a tribe configured node..., force settings - Settings.Builder sb = Settings.builder().put(settings); - sb.put(Node.NODE_MASTER_SETTING.getKey(), false); - sb.put(Node.NODE_DATA_SETTING.getKey(), false); - sb.put(Node.NODE_INGEST_SETTING.getKey(), false); - if (!NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.exists(settings)) { - sb.put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), nodesSettings.size()); - } - sb.put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "tribe"); // there is a special discovery implementation for tribe - // nothing is going to be discovered, since no master will be elected - sb.put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), 0); - if (sb.get("cluster.name") == null) { - sb.put("cluster.name", "tribe_" + UUIDs.randomBase64UUID()); // make sure it won't join other tribe nodes in the same JVM - } - sb.put(TransportMasterNodeReadAction.FORCE_LOCAL_SETTING.getKey(), true); - return sb.build(); - } - - /** - * Interface to allow merging {@link org.elasticsearch.cluster.metadata.MetaData.Custom} in tribe node - * When multiple Mergable Custom metadata of the same type is found (from underlying clusters), the - * Custom metadata will be merged using {@link #merge(MetaData.Custom)} and the result will be stored - * in the tribe cluster state - * - * @param type of custom meta data - */ - public interface MergableCustomMetaData { - - /** - * Merges this custom metadata with other, returning either this or other custom metadata - * for tribe cluster state. This method should not mutate either this or the - * other custom metadata. - * - * @param other custom meta data - * @return the same instance or other custom metadata based on implementation - * if both the instances are considered equal, implementations should return this - * instance to avoid redundant cluster state changes. - */ - T merge(T other); - } - // internal settings only public static final Setting TRIBE_NAME_SETTING = Setting.simpleString("tribe.name", Property.NodeScope); private final ClusterService clusterService; @@ -200,7 +135,8 @@ public interface MergableCustomMetaData { Setting.listSetting("tribe.blocks.metadata.indices", Collections.emptyList(), Function.identity(), Property.NodeScope); public static final Set TRIBE_SETTING_KEYS = Sets.newHashSet(TRIBE_NAME_SETTING.getKey(), ON_CONFLICT_SETTING.getKey(), - BLOCKS_METADATA_INDICES_SETTING.getKey(), BLOCKS_METADATA_SETTING.getKey(), BLOCKS_READ_INDICES_SETTING.getKey(), BLOCKS_WRITE_INDICES_SETTING.getKey(), BLOCKS_WRITE_SETTING.getKey()); + BLOCKS_METADATA_INDICES_SETTING.getKey(), BLOCKS_METADATA_SETTING.getKey(), BLOCKS_READ_INDICES_SETTING.getKey(), + BLOCKS_WRITE_INDICES_SETTING.getKey(), BLOCKS_WRITE_SETTING.getKey()); // these settings should be passed through to each tribe client, if they are not set explicitly private static final List> PASS_THROUGH_SETTINGS = Arrays.asList( @@ -218,8 +154,9 @@ public interface MergableCustomMetaData { private final NamedWriteableRegistry namedWriteableRegistry; - public TribeService(Settings settings, Path configPath, ClusterService clusterService, final String tribeNodeId, - NamedWriteableRegistry namedWriteableRegistry, BiFunction clientNodeBuilder) { + @Inject + public TribeService(Settings settings, Environment environment, NodeEnvironment nodeEnvironment, ClusterService clusterService, + Node node, NamedWriteableRegistry namedWriteableRegistry, Node.NodeBuilder clientNodeBuilder) { super(settings); this.clusterService = clusterService; this.namedWriteableRegistry = namedWriteableRegistry; @@ -227,8 +164,8 @@ public TribeService(Settings settings, Path configPath, ClusterService clusterSe nodesSettings.remove("blocks"); // remove prefix settings that don't indicate a client nodesSettings.remove("on_conflict"); // remove prefix settings that don't indicate a client for (Map.Entry entry : nodesSettings.entrySet()) { - Settings clientSettings = buildClientSettings(entry.getKey(), tribeNodeId, settings, entry.getValue()); - nodes.add(clientNodeBuilder.apply(clientSettings, configPath)); + Settings clientSettings = buildClientSettings(entry.getKey(), nodeEnvironment.nodeId(), settings, entry.getValue()); + nodes.add(clientNodeBuilder.newNode(clientSettings, environment.configFile())); } this.blockIndicesMetadata = BLOCKS_METADATA_INDICES_SETTING.get(settings).toArray(Strings.EMPTY_ARRAY); @@ -239,6 +176,7 @@ public TribeService(Settings settings, Path configPath, ClusterService clusterSe .deprecated("tribe nodes are deprecated in favor of cross-cluster search and will be removed in Elasticsearch 7.0.0"); } this.onConflict = ON_CONFLICT_SETTING.get(settings); + node.addOnStartedListener(this::startNodes); } // pkg private for testing @@ -279,7 +217,6 @@ static Settings buildClientSettings(String tribeName, String parentNodeId, Setti return sb.build(); } - @Override protected void doStart() { diff --git a/core/src/test/java/org/elasticsearch/tribe/TribeIT.java b/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeIT.java similarity index 94% rename from core/src/test/java/org/elasticsearch/tribe/TribeIT.java rename to modules/tribe/src/test/java/org/elasticsearch/tribe/TribeIT.java index 0307e69cfbeba..150134e69e2c0 100644 --- a/core/src/test/java/org/elasticsearch/tribe/TribeIT.java +++ b/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeIT.java @@ -33,12 +33,14 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.inject.ProvisionException; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.discovery.MasterNotDiscoveredException; import org.elasticsearch.node.Node; @@ -47,6 +49,7 @@ import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.NodeConfigurationSource; import org.elasticsearch.test.TestCustomMetaData; +import org.elasticsearch.test.discovery.TestZenDiscovery; import org.elasticsearch.transport.MockTcpTransportPlugin; import org.elasticsearch.tribe.TribeServiceTests.MergableCustomMetaData1; import org.elasticsearch.tribe.TribeServiceTests.MergableCustomMetaData2; @@ -76,6 +79,7 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.core.Is.is; @@ -149,10 +153,33 @@ public List getNamedWriteables() { protected Collection> nodePlugins() { ArrayList> plugins = new ArrayList<>(); plugins.addAll(getMockPlugins()); + plugins.add(TribePlugin.class); + plugins.add(TribeAwareTestZenDiscoveryPlugin.class); plugins.add(TestCustomMetaDataPlugin.class); return plugins; } + @Override + protected boolean addTestZenDiscovery() { + return false; + } + + public static class TribeAwareTestZenDiscoveryPlugin extends TestZenDiscovery.TestPlugin { + + public TribeAwareTestZenDiscoveryPlugin(Settings settings) { + super(settings); + } + + @Override + public Settings additionalSettings() { + if (settings.getGroups("tribe", true).isEmpty()) { + return super.additionalSettings(); + } else { + return Settings.EMPTY; + } + } + } + @Before public void startRemoteClusters() { final int minNumDataNodes = 2; @@ -249,9 +276,12 @@ private Settings.Builder createTribeSettings(Predicate filt final Settings.Builder settings = Settings.builder(); settings.put(Node.NODE_NAME_SETTING.getKey(), TRIBE_NODE); settings.put(Node.NODE_DATA_SETTING.getKey(), false); - settings.put(Node.NODE_MASTER_SETTING.getKey(), true); + settings.put(Node.NODE_MASTER_SETTING.getKey(), false); + settings.put(Node.NODE_INGEST_SETTING.getKey(), false); settings.put(NetworkModule.HTTP_ENABLED.getKey(), false); settings.put(NetworkModule.TRANSPORT_TYPE_SETTING.getKey(), MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME); + // add dummy tribe setting so that node is always identifiable as tribe in this test even if the set of connecting cluster is empty + settings.put(TribeService.BLOCKS_WRITE_SETTING.getKey(), TribeService.BLOCKS_WRITE_SETTING.getDefault(Settings.EMPTY)); doWithAllClusters(filter, c -> { String tribeSetting = "tribe." + c.getClusterName() + "."; @@ -263,6 +293,16 @@ private Settings.Builder createTribeSettings(Predicate filt return settings; } + public void testTribeNodeWithBadSettings() throws Exception { + Settings brokenSettings = Settings.builder() + .put("tribe.some.setting.that.does.not.exist", true) + .build(); + + ProvisionException e = expectThrows(ProvisionException.class, () -> startTribeNode(ALL, brokenSettings)); // <3 Guice + assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); + assertThat(e.getCause().getMessage(), containsString("unknown setting [setting.that.does.not.exist]")); + } + public void testGlobalReadWriteBlocks() throws Exception { Settings additionalSettings = Settings.builder() .put("tribe.blocks.write", true) diff --git a/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java b/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java similarity index 94% rename from core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java rename to modules/tribe/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java index ac9e3156e1c2d..c6341d1f734e5 100644 --- a/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java +++ b/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java @@ -20,14 +20,13 @@ package org.elasticsearch.tribe; import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.MergableCustomMetaData; import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.network.NetworkModule; -import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.env.Environment; import org.elasticsearch.node.MockNode; import org.elasticsearch.node.Node; import org.elasticsearch.test.ESTestCase; @@ -137,7 +136,7 @@ public void testMergeCustomMetaData() { } public void testMergeMultipleCustomMetaData() { - Map> inputMap = new HashMap<>(); + Map> inputMap = new HashMap<>(); inputMap.put(MergableCustomMetaData1.TYPE, Arrays.asList(new MergableCustomMetaData1("data10"), new MergableCustomMetaData1("data11"))); inputMap.put(MergableCustomMetaData2.TYPE, @@ -155,15 +154,15 @@ public void testMergeMultipleCustomMetaData() { } public void testMergeCustomMetaDataFromMany() { - Map> inputMap = new HashMap<>(); + Map> inputMap = new HashMap<>(); int n = randomIntBetween(3, 5); - List customList1 = new ArrayList<>(); + List customList1 = new ArrayList<>(); for (int i = 0; i <= n; i++) { customList1.add(new MergableCustomMetaData1("data1"+String.valueOf(i))); } Collections.shuffle(customList1, random()); inputMap.put(MergableCustomMetaData1.TYPE, customList1); - List customList2 = new ArrayList<>(); + List customList2 = new ArrayList<>(); for (int i = 0; i <= n; i++) { customList2.add(new MergableCustomMetaData2("data2"+String.valueOf(i))); } @@ -197,7 +196,7 @@ public void testTribeNodeDeprecation() throws IOException { settings.put(tribeSetting + ClusterName.CLUSTER_NAME_SETTING.getKey(), clusterName) .put(tribeSetting + NetworkModule.TRANSPORT_TYPE_SETTING.getKey(), "mock-socket-network"); } - try (Node node = new MockNode(settings.build(),Collections.singleton(MockTcpTransportPlugin.class) )) { + try (Node node = new MockNode(settings.build(), Arrays.asList(TribePlugin.class, MockTcpTransportPlugin.class))) { if (tribeServiceEnable) { assertWarnings("tribe nodes are deprecated in favor of cross-cluster search and will be removed in Elasticsearch 7.0.0"); } @@ -205,7 +204,7 @@ public void testTribeNodeDeprecation() throws IOException { } static class MergableCustomMetaData1 extends TestCustomMetaData - implements TribeService.MergableCustomMetaData { + implements MergableCustomMetaData { public static final String TYPE = "custom_md_1"; protected MergableCustomMetaData1(String data) { @@ -237,7 +236,7 @@ public MergableCustomMetaData1 merge(MergableCustomMetaData1 other) { } static class MergableCustomMetaData2 extends TestCustomMetaData - implements TribeService.MergableCustomMetaData { + implements MergableCustomMetaData { public static final String TYPE = "custom_md_2"; protected MergableCustomMetaData2(String data) { diff --git a/settings.gradle b/settings.gradle index 152f892f774cb..9cb695b466188 100644 --- a/settings.gradle +++ b/settings.gradle @@ -40,6 +40,7 @@ List projects = [ 'modules:reindex', 'modules:repository-url', 'modules:transport-netty4', + 'modules:tribe', 'plugins:analysis-icu', 'plugins:analysis-kuromoji', 'plugins:analysis-phonetic', diff --git a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java index 311b177830950..82b4dd854205e 100644 --- a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java +++ b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java @@ -114,7 +114,7 @@ protected TransportService newTransportService(Settings settings, Transport tran } @Override - protected Node newTribeClientNode(Settings settings, Collection> classpathPlugins, Path configPath) { + protected Node newNode(Settings settings, Collection> classpathPlugins, Path configPath) { return new MockNode(settings, classpathPlugins, configPath); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 0eaa3a57df0e3..03e22125b2978 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -1862,12 +1862,20 @@ public Collection> transportClientPlugins() { /** * Iff this returns true mock transport implementations are used for the test runs. Otherwise not mock transport impls are used. - * The defautl is true + * The default is true */ protected boolean addMockTransportService() { return true; } + /** + * Iff this returns true test zen discovery implementations is used for the test runs. + * The default is true + */ + protected boolean addTestZenDiscovery() { + return true; + } + /** * Returns a function that allows to wrap / filter all clients that are exposed by the test cluster. This is useful * for debugging or request / response pre and post processing. It also allows to intercept all calls done by the test @@ -1905,7 +1913,9 @@ protected Collection> getMockPlugins() { mocks.add(MockTcpTransportPlugin.class); } - mocks.add(TestZenDiscovery.TestPlugin.class); + if (addTestZenDiscovery()) { + mocks.add(TestZenDiscovery.TestPlugin.class); + } mocks.add(TestSeedPlugin.class); return Collections.unmodifiableList(mocks); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java b/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java index be6792d463d97..63212cddc39b1 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java +++ b/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java @@ -52,7 +52,7 @@ public class TestZenDiscovery extends ZenDiscovery { /** A plugin which installs mock discovery and configures it to be used. */ public static class TestPlugin extends Plugin implements DiscoveryPlugin { - private Settings settings; + protected final Settings settings; public TestPlugin(Settings settings) { this.settings = settings; } From 36454ec1bd15172cdc22ee2189f6bb08d6ee74b5 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 18 Jul 2017 18:57:30 +0200 Subject: [PATCH 02/11] remove obsolete tribe tests --- .../common/settings/SettingsModuleTests.java | 51 ------------------- 1 file changed, 51 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java index f2d9016a09a5d..a1c2711e5acc2 100644 --- a/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java +++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java @@ -21,13 +21,10 @@ import org.elasticsearch.common.inject.ModuleTestCase; import org.elasticsearch.common.settings.Setting.Property; -import org.joda.time.MonthDay; import java.util.Arrays; -import java.util.Collections; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.is; public class SettingsModuleTests extends ModuleTestCase { @@ -87,54 +84,6 @@ public void testRegisterSettings() { } } - public void testTribeSetting() { - { - Settings settings = Settings.builder().put("tribe.t1.cluster.routing.allocation.balance.shard", "2.0").build(); - SettingsModule module = new SettingsModule(settings); - assertInstanceBinding(module, Settings.class, (s) -> s == settings); - } - { - Settings settings = Settings.builder().put("tribe.t1.cluster.routing.allocation.balance.shard", "[2.0]").build(); - try { - new SettingsModule(settings); - fail(); - } catch (IllegalArgumentException ex) { - assertEquals( - "tribe.t1 validation failed: Failed to parse value [[2.0]] for setting [cluster.routing.allocation.balance.shard]", - ex.getMessage()); - } - } - } - - public void testSpecialTribeSetting() { - { - Settings settings = Settings.builder().put("tribe.blocks.write", "false").build(); - SettingsModule module = new SettingsModule(settings); - assertInstanceBinding(module, Settings.class, (s) -> s == settings); - } - { - Settings settings = Settings.builder().put("tribe.blocks.write", "BOOM").build(); - try { - new SettingsModule(settings); - fail(); - } catch (IllegalArgumentException ex) { - assertEquals("Failed to parse value [BOOM] as only [true] or [false] are allowed.", - ex.getMessage()); - } - } - { - Settings settings = Settings.builder().put("tribe.blocks.wtf", "BOOM").build(); - try { - new SettingsModule(settings); - fail(); - } catch (IllegalArgumentException ex) { - assertEquals("tribe.blocks validation failed: unknown setting [wtf] please check that any required plugins are" + - " installed, or check the breaking changes documentation for removed settings", ex.getMessage()); - } - } - } - - public void testLoggerSettings() { { Settings settings = Settings.builder().put("logger._root", "TRACE").put("logger.transport", "INFO").build(); From 462a600b3cefceb36104ce7b7edc83f94c8b3fb2 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 18 Jul 2017 19:36:14 +0200 Subject: [PATCH 03/11] remove stale test --- .../threadpool/SimpleThreadPoolIT.java | 40 ------------------- 1 file changed, 40 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java b/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java index bff004579ff91..fbfcf6bbd9510 100644 --- a/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java +++ b/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java @@ -21,32 +21,22 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.env.Environment; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.node.Node; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.hamcrest.RegexMatcher; -import java.io.IOException; import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; import java.util.HashSet; -import java.util.Map; import java.util.Set; import java.util.regex.Pattern; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -import static org.hamcrest.Matchers.containsString; @ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0) public class SimpleThreadPoolIT extends ESIntegTestCase { @@ -111,34 +101,4 @@ public void testThreadNames() throws Exception { } } - public void testThreadPoolLeakingThreadsWithTribeNode() { - Settings settings = Settings.builder() - .put("node.name", "thread_pool_leaking_threads_tribe_node") - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) - .put("tribe.t1.cluster.name", "non_existing_cluster") - //trigger initialization failure of one of the tribes (doesn't require starting the node) - .put("tribe.t1.plugin.mandatory", "non_existing").build(); - - try { - new Node(settings); - fail("The node startup is supposed to fail"); - } catch(Exception e) { - //all good - assertThat(e.getMessage(), containsString("mandatory plugins [non_existing]")); - } - } - - private Map getPoolSettingsThroughJson(ThreadPoolInfo info, String poolName) throws IOException { - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - info.toXContent(builder, ToXContent.EMPTY_PARAMS); - builder.endObject(); - builder.close(); - Map poolsMap; - try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) { - poolsMap = parser.map(); - } - return (Map) ((Map) poolsMap.get("thread_pool")).get(poolName); - } - } From e6064eadcc92ffd827973ebf38f7c45d50b957b1 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 18 Jul 2017 20:58:15 +0200 Subject: [PATCH 04/11] fix test name --- modules/tribe/build.gradle | 2 ++ .../tribe/{TribeIT.java => TribeIntegrationTests.java} | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) rename modules/tribe/src/test/java/org/elasticsearch/tribe/{TribeIT.java => TribeIntegrationTests.java} (99%) diff --git a/modules/tribe/build.gradle b/modules/tribe/build.gradle index 59c99c1735659..21c1c9f9647f3 100644 --- a/modules/tribe/build.gradle +++ b/modules/tribe/build.gradle @@ -24,3 +24,5 @@ esplugin { compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked" compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked" + +integTestRunner.enabled = false diff --git a/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeIT.java b/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeIntegrationTests.java similarity index 99% rename from modules/tribe/src/test/java/org/elasticsearch/tribe/TribeIT.java rename to modules/tribe/src/test/java/org/elasticsearch/tribe/TribeIntegrationTests.java index 150134e69e2c0..f532ee048a3ca 100644 --- a/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeIT.java +++ b/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeIntegrationTests.java @@ -88,7 +88,7 @@ * does it by default. */ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0) -public class TribeIT extends ESIntegTestCase { +public class TribeIntegrationTests extends ESIntegTestCase { private static final String TRIBE_NODE = "tribe_node"; From fdcc93a95529896e5bcf7444d73147f084756928 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Wed, 19 Jul 2017 00:17:44 +0200 Subject: [PATCH 05/11] Fix TribeUnitTests --- qa/evil-tests/build.gradle | 1 + .../elasticsearch/tribe/TribeUnitTests.java | 23 ++++++++++++++++--- .../java/org/elasticsearch/node/MockNode.java | 9 ++++++-- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/qa/evil-tests/build.gradle b/qa/evil-tests/build.gradle index 472fc87261602..2ac8cd89783d5 100644 --- a/qa/evil-tests/build.gradle +++ b/qa/evil-tests/build.gradle @@ -27,6 +27,7 @@ apply plugin: 'elasticsearch.standalone-test' dependencies { testCompile 'com.google.jimfs:jimfs:1.1' + testCompile project(path: ':modules:tribe', configuration: 'runtime') } // TODO: give each evil test its own fresh JVM for more isolation. diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java index adff57f517d0e..1a36c01fe09fb 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java @@ -67,7 +67,8 @@ public static void createTribes() throws NodeValidationException { .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), 2) .build(); - final List> mockPlugins = Arrays.asList(MockTcpTransportPlugin.class, TestZenDiscovery.TestPlugin.class); + final List> mockPlugins = Arrays.asList(MockTcpTransportPlugin.class, + TribeAwareTestZenDiscoveryPlugin.class, TribePlugin.class); tribe1 = new MockNode( Settings.builder() .put(baseSettings) @@ -91,6 +92,22 @@ public static void closeTribes() throws IOException { tribe2 = null; } + public static class TribeAwareTestZenDiscoveryPlugin extends TestZenDiscovery.TestPlugin { + + public TribeAwareTestZenDiscoveryPlugin(Settings settings) { + super(settings); + } + + @Override + public Settings additionalSettings() { + if (settings.getGroups("tribe", true).isEmpty()) { + return super.additionalSettings(); + } else { + return Settings.EMPTY; + } + } + } + public void testThatTribeClientsIgnoreGlobalConfig() throws Exception { assertTribeNodeSuccessfullyCreated(getDataPath("elasticsearch.yml").getParent()); assertWarnings("tribe nodes are deprecated in favor of cross-cluster search and will be removed in Elasticsearch 7.0.0"); @@ -99,14 +116,14 @@ public void testThatTribeClientsIgnoreGlobalConfig() throws Exception { private static void assertTribeNodeSuccessfullyCreated(Path configPath) throws Exception { // the tribe clients do need it to make sure they can find their corresponding tribes using the proper transport Settings settings = Settings.builder().put(NetworkModule.HTTP_ENABLED.getKey(), false).put("node.name", "tribe_node") - .put("transport.type", MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME).put("discovery.type", "local") + .put("transport.type", MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME) .put("tribe.t1.transport.type", MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME) .put("tribe.t2.transport.type",MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME) .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) .build(); final List> classpathPlugins = - Arrays.asList(MockTcpTransportPlugin.class, TestZenDiscovery.TestPlugin.class); + Arrays.asList(TribeAwareTestZenDiscoveryPlugin.class, TribePlugin.class, MockTcpTransportPlugin.class); try (Node node = new MockNode(settings, classpathPlugins, configPath).start()) { try (Client client = node.client()) { assertBusy(() -> { diff --git a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java index 82b4dd854205e..e7adcbf8d457a 100644 --- a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java +++ b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.env.Environment; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.recovery.RecoverySettings; @@ -66,7 +67,11 @@ public MockNode(Settings settings, Collection> classpath } public MockNode(Settings settings, Collection> classpathPlugins, Path configPath) { - super(InternalSettingsPreparer.prepareEnvironment(settings, null, Collections.emptyMap(), configPath), classpathPlugins); + this(InternalSettingsPreparer.prepareEnvironment(settings, null, Collections.emptyMap(), configPath), classpathPlugins); + } + + public MockNode(Environment environment, Collection> classpathPlugins) { + super(environment, classpathPlugins); this.classpathPlugins = classpathPlugins; } @@ -115,7 +120,7 @@ protected TransportService newTransportService(Settings settings, Transport tran @Override protected Node newNode(Settings settings, Collection> classpathPlugins, Path configPath) { - return new MockNode(settings, classpathPlugins, configPath); + return new MockNode(new Environment(settings, configPath), classpathPlugins); } @Override From f290c59cf50aa0c28335df7202a0161bed895954 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Wed, 19 Jul 2017 09:24:19 +0200 Subject: [PATCH 06/11] properly clean up if inner tribe fails to initialize --- .../org/elasticsearch/tribe/TribeService.java | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/modules/tribe/src/main/java/org/elasticsearch/tribe/TribeService.java b/modules/tribe/src/main/java/org/elasticsearch/tribe/TribeService.java index 016d9b0091db6..ec6fdd681c18c 100644 --- a/modules/tribe/src/main/java/org/elasticsearch/tribe/TribeService.java +++ b/modules/tribe/src/main/java/org/elasticsearch/tribe/TribeService.java @@ -23,6 +23,7 @@ import org.apache.logging.log4j.util.Supplier; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; @@ -165,7 +166,20 @@ public TribeService(Settings settings, Environment environment, NodeEnvironment nodesSettings.remove("on_conflict"); // remove prefix settings that don't indicate a client for (Map.Entry entry : nodesSettings.entrySet()) { Settings clientSettings = buildClientSettings(entry.getKey(), nodeEnvironment.nodeId(), settings, entry.getValue()); - nodes.add(clientNodeBuilder.newNode(clientSettings, environment.configFile())); + try { + nodes.add(clientNodeBuilder.newNode(clientSettings, environment.configFile())); + } catch (Exception e) { + // calling close is safe for non started nodes, we can just iterate over all + for (Node otherNode : nodes) { + try { + otherNode.close(); + } catch (Exception inner) { + inner.addSuppressed(e); + logger.warn((Supplier) () -> new ParameterizedMessage("failed to close node {} on failed start", otherNode), inner); + } + } + throw ExceptionsHelper.convertToRuntime(e); + } } this.blockIndicesMetadata = BLOCKS_METADATA_INDICES_SETTING.get(settings).toArray(Strings.EMPTY_ARRAY); @@ -237,10 +251,7 @@ public void startNodes() { logger.warn((Supplier) () -> new ParameterizedMessage("failed to close node {} on failed start", otherNode), inner); } } - if (e instanceof RuntimeException) { - throw (RuntimeException) e; - } - throw new ElasticsearchException(e); + throw ExceptionsHelper.convertToRuntime(e); } } } From 202515049afa9a070e9dca01590e6bb33678175f Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 25 Jul 2017 17:57:18 +0200 Subject: [PATCH 07/11] unguice --- .../elasticsearch/cluster/ClusterModule.java | 6 ++++ .../java/org/elasticsearch/node/Node.java | 28 +++---------------- .../elasticsearch/plugins/ClusterPlugin.java | 7 +++++ .../org/elasticsearch/plugins/Plugin.java | 9 +++++- .../metadata/TemplateUpgradeServiceIT.java | 13 +++++++-- .../org/elasticsearch/tribe/TribePlugin.java | 28 +++++++++++++++---- .../org/elasticsearch/tribe/TribeService.java | 10 +++---- .../tribe/TribeIntegrationTests.java | 10 ++----- .../file/FileBasedDiscoveryPlugin.java | 17 ++++++----- 9 files changed, 76 insertions(+), 52 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java index 603952101659d..0d7d13bab77aa 100644 --- a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java +++ b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java @@ -94,6 +94,7 @@ public class ClusterModule extends AbstractModule { private final IndexNameExpressionResolver indexNameExpressionResolver; private final AllocationDeciders allocationDeciders; private final AllocationService allocationService; + private final Runnable onStarted; // pkg private for tests final Collection deciderList; final ShardsAllocator shardsAllocator; @@ -106,6 +107,7 @@ public ClusterModule(Settings settings, ClusterService clusterService, List clusterPlugins.forEach(plugin -> plugin.onNodeStarted()); } @@ -241,4 +243,8 @@ protected void configure() { bind(AllocationDeciders.class).toInstance(allocationDeciders); bind(ShardsAllocator.class).toInstance(shardsAllocator); } + + public Runnable onStarted() { + return onStarted; + } } diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java index efe11eadc1d85..789e777a3c6ac 100644 --- a/core/src/main/java/org/elasticsearch/node/Node.java +++ b/core/src/main/java/org/elasticsearch/node/Node.java @@ -152,7 +152,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -229,7 +228,6 @@ public static final Settings addNodeNameIfNeeded(Settings settings, final String private final Collection pluginLifecycleComponents; private final LocalNodeFactory localNodeFactory; private final NodeService nodeService; - private final List onStartedListeners = new CopyOnWriteArrayList<>(); /** * Constructs a node with the given settings. @@ -394,7 +392,9 @@ protected Node(final Environment environment, Collection Collection pluginComponents = pluginsService.filterPlugins(Plugin.class).stream() .flatMap(p -> p.createComponents(client, clusterService, threadPool, resourceWatcherService, - scriptModule.getScriptService(), xContentRegistry).stream()) + scriptModule.getScriptService(), xContentRegistry, environment, nodeEnvironment, + namedWriteableRegistry, + (settings, configPath) -> newNode(settings, classpathPlugins, configPath)).stream()) .collect(Collectors.toList()); final RestController restController = actionModule.getRestController(); final NetworkModule networkModule = new NetworkModule(settings, false, pluginsService.filterPlugins(NetworkPlugin.class), @@ -439,7 +439,6 @@ protected Node(final Environment environment, Collection transportService, indicesService, pluginsService, circuitBreakerService, scriptModule.getScriptService(), httpServerTransport, ingestService, clusterService, settingsModule.getSettingsFilter()); modules.add(b -> { - b.bind(NodeBuilder.class).toInstance(new NodeBuilder(this, classpathPlugins)); b.bind(Node.class).toInstance(this); b.bind(NodeService.class).toInstance(nodeService); b.bind(NamedXContentRegistry.class).toInstance(xContentRegistry); @@ -517,10 +516,6 @@ protected Node(final Environment environment, Collection } } - public void addOnStartedListener(Runnable runnable) { - onStartedListeners.add(runnable); - } - // visible for testing static void warnIfPreRelease(final Version version, final boolean isSnapshot, final Logger logger) { if (!version.isRelease() || isSnapshot) { @@ -674,7 +669,7 @@ public void onTimeout(TimeValue timeout) { logger.info("started"); - onStartedListeners.forEach(Runnable::run); + pluginsService.filterPlugins(ClusterPlugin.class).forEach(ClusterPlugin::onNodeStarted); return this; } @@ -908,21 +903,6 @@ private List getCustomNameResolvers(List> classpathPlugins; - - public NodeBuilder(Node node, Collection> classpathPlugins) { - this.node = node; - this.classpathPlugins = classpathPlugins; - } - - public Node newNode(Settings settings, Path configPath) { - return node.newNode(settings, classpathPlugins, configPath); - } - } - /** Constructs a new node based on the following settings. Overridden by tests */ protected Node newNode(Settings settings, Collection> classpathPlugins, Path configPath) { return new Node(new Environment(settings, configPath), classpathPlugins); diff --git a/core/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java b/core/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java index 7de805b704553..612a0f2f9fc56 100644 --- a/core/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java +++ b/core/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java @@ -58,4 +58,11 @@ default Collection createAllocationDeciders(Settings settings default Map> getShardsAllocators(Settings settings, ClusterSettings clusterSettings) { return Collections.emptyMap(); } + + /** + * Called when the node is started + */ + default void onNodeStarted() { + + } } diff --git a/core/src/main/java/org/elasticsearch/plugins/Plugin.java b/core/src/main/java/org/elasticsearch/plugins/Plugin.java index bf5bc49d279c0..b03befef561f3 100644 --- a/core/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/core/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -38,8 +38,11 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.discovery.DiscoveryModule; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexModule; import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.node.Node; import org.elasticsearch.repositories.RepositoriesModule; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; @@ -50,10 +53,12 @@ import java.io.Closeable; import java.io.IOException; +import java.nio.file.Path; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.BiFunction; import java.util.function.UnaryOperator; /** @@ -107,7 +112,9 @@ public Collection> getGuiceServiceClasses() */ public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, ResourceWatcherService resourceWatcherService, ScriptService scriptService, - NamedXContentRegistry xContentRegistry) { + NamedXContentRegistry xContentRegistry, Environment environment, + NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry, + BiFunction clientNodeBuilder) { return Collections.emptyList(); } diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java b/core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java index 36625284d473b..a5c2965bb4a62 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java @@ -22,21 +22,27 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.node.Node; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.watcher.ResourceWatcherService; +import java.nio.file.Path; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiFunction; import java.util.function.UnaryOperator; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -67,11 +73,14 @@ public TestPlugin(Settings settings) { @Override public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, ResourceWatcherService resourceWatcherService, ScriptService scriptService, - NamedXContentRegistry xContentRegistry) { + NamedXContentRegistry xContentRegistry, Environment environment, + NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry, + BiFunction clientNodeBuilder) { clusterService.getClusterSettings().addSettingsUpdateConsumer(UPDATE_TEMPLATE_DUMMY_SETTING, integer -> { logger.debug("the template dummy setting was updated to {}", integer); }); - return super.createComponents(client, clusterService, threadPool, resourceWatcherService, scriptService, xContentRegistry); + return super.createComponents(client, clusterService, threadPool, resourceWatcherService, scriptService, xContentRegistry, + environment, nodeEnvironment, namedWriteableRegistry, clientNodeBuilder); } @Override diff --git a/modules/tribe/src/main/java/org/elasticsearch/tribe/TribePlugin.java b/modules/tribe/src/main/java/org/elasticsearch/tribe/TribePlugin.java index 6a7705b4a68bf..55706ce158a35 100644 --- a/modules/tribe/src/main/java/org/elasticsearch/tribe/TribePlugin.java +++ b/modules/tribe/src/main/java/org/elasticsearch/tribe/TribePlugin.java @@ -20,44 +20,52 @@ package org.elasticsearch.tribe; import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterApplier; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.MasterService; import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.discovery.Discovery; import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.discovery.zen.UnicastHostsProvider; +import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.node.Node; +import org.elasticsearch.plugins.ClusterPlugin; import org.elasticsearch.plugins.DiscoveryPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.watcher.ResourceWatcherService; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; -public class TribePlugin extends Plugin implements DiscoveryPlugin { +public class TribePlugin extends Plugin implements DiscoveryPlugin, ClusterPlugin { private final Settings settings; + private TribeService tribeService; public TribePlugin(Settings settings) { this.settings = settings; } - @Override public Map> getDiscoveryTypes(ThreadPool threadPool, TransportService transportService, NamedWriteableRegistry namedWriteableRegistry, @@ -68,8 +76,18 @@ public Map> getDiscoveryTypes(ThreadPool threadPool, } @Override - public Collection> getGuiceServiceClasses() { - return Collections.singleton(TribeService.class); + public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, ScriptService scriptService, + NamedXContentRegistry xContentRegistry, Environment environment, + NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry, + BiFunction clientNodeBuilder) { + tribeService = new TribeService(settings, environment, nodeEnvironment, clusterService, namedWriteableRegistry, clientNodeBuilder); + return Collections.singleton(tribeService); + } + + @Override + public void onNodeStarted() { + tribeService.startNodes(); } @Override diff --git a/modules/tribe/src/main/java/org/elasticsearch/tribe/TribeService.java b/modules/tribe/src/main/java/org/elasticsearch/tribe/TribeService.java index ec6fdd681c18c..acac96e4caff7 100644 --- a/modules/tribe/src/main/java/org/elasticsearch/tribe/TribeService.java +++ b/modules/tribe/src/main/java/org/elasticsearch/tribe/TribeService.java @@ -22,7 +22,6 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.logging.log4j.util.Supplier; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; @@ -45,7 +44,6 @@ import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.hash.MurmurHash3; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -67,6 +65,7 @@ import org.elasticsearch.transport.TcpTransport; import java.io.IOException; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -76,6 +75,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.Collectors; @@ -155,9 +155,8 @@ public class TribeService extends AbstractLifecycleComponent { private final NamedWriteableRegistry namedWriteableRegistry; - @Inject public TribeService(Settings settings, Environment environment, NodeEnvironment nodeEnvironment, ClusterService clusterService, - Node node, NamedWriteableRegistry namedWriteableRegistry, Node.NodeBuilder clientNodeBuilder) { + NamedWriteableRegistry namedWriteableRegistry, BiFunction clientNodeBuilder) { super(settings); this.clusterService = clusterService; this.namedWriteableRegistry = namedWriteableRegistry; @@ -167,7 +166,7 @@ public TribeService(Settings settings, Environment environment, NodeEnvironment for (Map.Entry entry : nodesSettings.entrySet()) { Settings clientSettings = buildClientSettings(entry.getKey(), nodeEnvironment.nodeId(), settings, entry.getValue()); try { - nodes.add(clientNodeBuilder.newNode(clientSettings, environment.configFile())); + nodes.add(clientNodeBuilder.apply(clientSettings, environment.configFile())); } catch (Exception e) { // calling close is safe for non started nodes, we can just iterate over all for (Node otherNode : nodes) { @@ -190,7 +189,6 @@ public TribeService(Settings settings, Environment environment, NodeEnvironment .deprecated("tribe nodes are deprecated in favor of cross-cluster search and will be removed in Elasticsearch 7.0.0"); } this.onConflict = ON_CONFLICT_SETTING.get(settings); - node.addOnStartedListener(this::startNodes); } // pkg private for testing diff --git a/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeIntegrationTests.java b/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeIntegrationTests.java index f532ee048a3ca..ab4c1d8cbde07 100644 --- a/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeIntegrationTests.java +++ b/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeIntegrationTests.java @@ -23,8 +23,8 @@ import org.elasticsearch.action.support.DestructiveOperations; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.health.ClusterHealthStatus; @@ -33,14 +33,12 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.inject.ProvisionException; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.discovery.MasterNotDiscoveredException; import org.elasticsearch.node.Node; @@ -79,7 +77,6 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.core.Is.is; @@ -298,9 +295,8 @@ public void testTribeNodeWithBadSettings() throws Exception { .put("tribe.some.setting.that.does.not.exist", true) .build(); - ProvisionException e = expectThrows(ProvisionException.class, () -> startTribeNode(ALL, brokenSettings)); // <3 Guice - assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); - assertThat(e.getCause().getMessage(), containsString("unknown setting [setting.that.does.not.exist]")); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> startTribeNode(ALL, brokenSettings)); + assertThat(e.getMessage(), containsString("unknown setting [setting.that.does.not.exist]")); } public void testGlobalReadWriteBlocks() throws Exception { diff --git a/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java b/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java index d809fd3fa885e..726c9bf49288c 100644 --- a/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java +++ b/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java @@ -22,6 +22,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.network.NetworkService; @@ -32,6 +33,8 @@ import org.elasticsearch.discovery.zen.UnicastHostsProvider; import org.elasticsearch.discovery.zen.UnicastZenPing; import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.node.Node; import org.elasticsearch.plugins.DiscoveryPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptService; @@ -40,12 +43,14 @@ import org.elasticsearch.watcher.ResourceWatcherService; import java.io.IOException; +import java.nio.file.Path; import java.util.Collection; import java.util.Collections; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; import java.util.function.Supplier; /** @@ -66,13 +71,11 @@ public FileBasedDiscoveryPlugin(Settings settings) { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry) { + public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, ScriptService scriptService, + NamedXContentRegistry xContentRegistry, Environment environment, + NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry, + BiFunction clientNodeBuilder) { final int concurrentConnects = UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_CONCURRENT_CONNECTS_SETTING.get(settings); final ThreadFactory threadFactory = EsExecutors.daemonThreadFactory(settings, "[file_based_discovery_resolve]"); fileBasedDiscoveryExecutorService = EsExecutors.newScaling( From 996d7d715e91d50f55b151041c827fe309f3193b Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 25 Jul 2017 18:02:41 +0200 Subject: [PATCH 08/11] don't allow dynamic updates to tribe settings --- .../src/main/java/org/elasticsearch/tribe/TribePlugin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/tribe/src/main/java/org/elasticsearch/tribe/TribePlugin.java b/modules/tribe/src/main/java/org/elasticsearch/tribe/TribePlugin.java index 55706ce158a35..8917ce7082415 100644 --- a/modules/tribe/src/main/java/org/elasticsearch/tribe/TribePlugin.java +++ b/modules/tribe/src/main/java/org/elasticsearch/tribe/TribePlugin.java @@ -167,7 +167,7 @@ public List> getSettings() { // create dummy setting just so that setting validation does not complain, these settings are going to be validated // again by the SettingsModule of the nested tribe node. Setting setting = Setting.prefixKeySetting(prefix, (key) -> new Setting<>(key, "", Function.identity(), - Setting.Property.Dynamic, Setting.Property.NodeScope)); + Setting.Property.NodeScope)); allSettings.add(setting); } From 5b811922d01fd30557471096881435236c29775c Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 25 Jul 2017 22:31:06 +0200 Subject: [PATCH 09/11] byebye Node.newNode(...) --- .../java/org/elasticsearch/node/Node.java | 8 +------ .../org/elasticsearch/plugins/Plugin.java | 10 ++++---- .../metadata/TemplateUpgradeServiceIT.java | 7 ++---- .../org/elasticsearch/tribe/TribePlugin.java | 16 ++++++++++--- .../org/elasticsearch/tribe/TribeService.java | 8 +++---- .../tribe/TribeIntegrationTests.java | 23 ++++++++++++++++++- .../file/FileBasedDiscoveryPlugin.java | 5 +--- .../java/org/elasticsearch/node/MockNode.java | 6 ----- 8 files changed, 48 insertions(+), 35 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java index 789e777a3c6ac..0b79d18e60946 100644 --- a/core/src/main/java/org/elasticsearch/node/Node.java +++ b/core/src/main/java/org/elasticsearch/node/Node.java @@ -393,8 +393,7 @@ protected Node(final Environment environment, Collection Collection pluginComponents = pluginsService.filterPlugins(Plugin.class).stream() .flatMap(p -> p.createComponents(client, clusterService, threadPool, resourceWatcherService, scriptModule.getScriptService(), xContentRegistry, environment, nodeEnvironment, - namedWriteableRegistry, - (settings, configPath) -> newNode(settings, classpathPlugins, configPath)).stream()) + namedWriteableRegistry, classpathPlugins).stream()) .collect(Collectors.toList()); final RestController restController = actionModule.getRestController(); final NetworkModule networkModule = new NetworkModule(settings, false, pluginsService.filterPlugins(NetworkPlugin.class), @@ -903,11 +902,6 @@ private List getCustomNameResolvers(List> classpathPlugins, Path configPath) { - return new Node(new Environment(settings, configPath), classpathPlugins); - } - /** Constructs a ClusterInfoService which may be mocked for tests. */ protected ClusterInfoService newClusterInfoService(Settings settings, ClusterService clusterService, ThreadPool threadPool, NodeClient client, Consumer listeners) { diff --git a/core/src/main/java/org/elasticsearch/plugins/Plugin.java b/core/src/main/java/org/elasticsearch/plugins/Plugin.java index b03befef561f3..d1da0f5af2a40 100644 --- a/core/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/core/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -42,7 +42,6 @@ import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexModule; import org.elasticsearch.indices.analysis.AnalysisModule; -import org.elasticsearch.node.Node; import org.elasticsearch.repositories.RepositoriesModule; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; @@ -53,12 +52,10 @@ import java.io.Closeable; import java.io.IOException; -import java.nio.file.Path; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.BiFunction; import java.util.function.UnaryOperator; /** @@ -109,12 +106,17 @@ public Collection> getGuiceServiceClasses() * @param threadPool A service to allow retrieving an executor to run an async action * @param resourceWatcherService A service to watch for changes to node local files * @param scriptService A service to allow running scripts on the local node + * @param xContentRegistry the registry for extensible xContent parsing + * @param environment the environment for path and setting configurations + * @param nodeEnvironment the node environment used coordinate access to the data paths + * @param namedWriteableRegistry the registry for {@link NamedWriteable} object parsing + * @param classpathPlugins the plugins available on the classpath */ public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry, Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry, - BiFunction clientNodeBuilder) { + Collection> classpathPlugins) { return Collections.emptyList(); } diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java b/core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java index a5c2965bb4a62..f24e10665840f 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java @@ -29,20 +29,17 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.node.Node; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.watcher.ResourceWatcherService; -import java.nio.file.Path; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiFunction; import java.util.function.UnaryOperator; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -75,12 +72,12 @@ public Collection createComponents(Client client, ClusterService cluster ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry, Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry, - BiFunction clientNodeBuilder) { + Collection> classpathPlugins) { clusterService.getClusterSettings().addSettingsUpdateConsumer(UPDATE_TEMPLATE_DUMMY_SETTING, integer -> { logger.debug("the template dummy setting was updated to {}", integer); }); return super.createComponents(client, clusterService, threadPool, resourceWatcherService, scriptService, xContentRegistry, - environment, nodeEnvironment, namedWriteableRegistry, clientNodeBuilder); + environment, nodeEnvironment, namedWriteableRegistry, classpathPlugins); } @Override diff --git a/modules/tribe/src/main/java/org/elasticsearch/tribe/TribePlugin.java b/modules/tribe/src/main/java/org/elasticsearch/tribe/TribePlugin.java index 8917ce7082415..5efcaee0609e1 100644 --- a/modules/tribe/src/main/java/org/elasticsearch/tribe/TribePlugin.java +++ b/modules/tribe/src/main/java/org/elasticsearch/tribe/TribePlugin.java @@ -53,7 +53,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; @@ -80,11 +79,22 @@ public Collection createComponents(Client client, ClusterService cluster ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry, Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry, - BiFunction clientNodeBuilder) { - tribeService = new TribeService(settings, environment, nodeEnvironment, clusterService, namedWriteableRegistry, clientNodeBuilder); + Collection> classpathPlugins) { + tribeService = new TribeService(settings, nodeEnvironment, clusterService, namedWriteableRegistry, + nodeBuilder(environment.configFile(), classpathPlugins)); return Collections.singleton(tribeService); } + protected Function nodeBuilder(Path configPath, Collection> classpathPlugins) { + return settings -> new TribeNode(settings, configPath, classpathPlugins); + } + + private static class TribeNode extends Node { + private TribeNode(Settings settings, Path configPath, Collection> classpathPlugins) { + super(new Environment(settings, configPath), classpathPlugins); + } + } + @Override public void onNodeStarted() { tribeService.startNodes(); diff --git a/modules/tribe/src/main/java/org/elasticsearch/tribe/TribeService.java b/modules/tribe/src/main/java/org/elasticsearch/tribe/TribeService.java index acac96e4caff7..714749b94782c 100644 --- a/modules/tribe/src/main/java/org/elasticsearch/tribe/TribeService.java +++ b/modules/tribe/src/main/java/org/elasticsearch/tribe/TribeService.java @@ -65,7 +65,6 @@ import org.elasticsearch.transport.TcpTransport; import java.io.IOException; -import java.nio.file.Path; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -75,7 +74,6 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.Collectors; @@ -155,8 +153,8 @@ public class TribeService extends AbstractLifecycleComponent { private final NamedWriteableRegistry namedWriteableRegistry; - public TribeService(Settings settings, Environment environment, NodeEnvironment nodeEnvironment, ClusterService clusterService, - NamedWriteableRegistry namedWriteableRegistry, BiFunction clientNodeBuilder) { + public TribeService(Settings settings, NodeEnvironment nodeEnvironment, ClusterService clusterService, + NamedWriteableRegistry namedWriteableRegistry, Function clientNodeBuilder) { super(settings); this.clusterService = clusterService; this.namedWriteableRegistry = namedWriteableRegistry; @@ -166,7 +164,7 @@ public TribeService(Settings settings, Environment environment, NodeEnvironment for (Map.Entry entry : nodesSettings.entrySet()) { Settings clientSettings = buildClientSettings(entry.getKey(), nodeEnvironment.nodeId(), settings, entry.getValue()); try { - nodes.add(clientNodeBuilder.apply(clientSettings, environment.configFile())); + nodes.add(clientNodeBuilder.apply(clientSettings)); } catch (Exception e) { // calling close is safe for non started nodes, we can just iterate over all for (Node otherNode : nodes) { diff --git a/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeIntegrationTests.java b/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeIntegrationTests.java index ab4c1d8cbde07..6f21d1faa5ccf 100644 --- a/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeIntegrationTests.java +++ b/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeIntegrationTests.java @@ -41,6 +41,8 @@ import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.discovery.MasterNotDiscoveredException; +import org.elasticsearch.env.Environment; +import org.elasticsearch.node.MockNode; import org.elasticsearch.node.Node; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -56,6 +58,7 @@ import org.junit.Before; import java.io.IOException; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -146,11 +149,29 @@ public List getNamedWriteables() { } } + public static class MockTribePlugin extends TribePlugin { + + public MockTribePlugin(Settings settings) { + super(settings); + } + + protected Function nodeBuilder(Path configPath, Collection> classpathPlugins) { + return settings -> new MockTribeNode(settings, configPath, classpathPlugins); + } + + private static class MockTribeNode extends MockNode { + private MockTribeNode(Settings settings, Path configPath, Collection> classpathPlugins) { + super(new Environment(settings, configPath), classpathPlugins); + } + } + + } + @Override protected Collection> nodePlugins() { ArrayList> plugins = new ArrayList<>(); plugins.addAll(getMockPlugins()); - plugins.add(TribePlugin.class); + plugins.add(MockTribePlugin.class); plugins.add(TribeAwareTestZenDiscoveryPlugin.class); plugins.add(TestCustomMetaDataPlugin.class); return plugins; diff --git a/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java b/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java index 726c9bf49288c..9f95970047bd2 100644 --- a/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java +++ b/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java @@ -34,7 +34,6 @@ import org.elasticsearch.discovery.zen.UnicastZenPing; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.node.Node; import org.elasticsearch.plugins.DiscoveryPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptService; @@ -43,14 +42,12 @@ import org.elasticsearch.watcher.ResourceWatcherService; import java.io.IOException; -import java.nio.file.Path; import java.util.Collection; import java.util.Collections; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; -import java.util.function.BiFunction; import java.util.function.Supplier; /** @@ -75,7 +72,7 @@ public Collection createComponents(Client client, ClusterService cluster ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry, Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry, - BiFunction clientNodeBuilder) { + Collection> classpathPlugins) { final int concurrentConnects = UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_CONCURRENT_CONNECTS_SETTING.get(settings); final ThreadFactory threadFactory = EsExecutors.daemonThreadFactory(settings, "[file_based_discovery_resolve]"); fileBasedDiscoveryExecutorService = EsExecutors.newScaling( diff --git a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java index e7adcbf8d457a..4255163db7fea 100644 --- a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java +++ b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java @@ -39,7 +39,6 @@ import org.elasticsearch.search.MockSearchService; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.fetch.FetchPhase; -import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.Transport; @@ -118,11 +117,6 @@ protected TransportService newTransportService(Settings settings, Transport tran } } - @Override - protected Node newNode(Settings settings, Collection> classpathPlugins, Path configPath) { - return new MockNode(new Environment(settings, configPath), classpathPlugins); - } - @Override protected void processRecoverySettings(ClusterSettings clusterSettings, RecoverySettings recoverySettings) { if (false == getPluginsService().filterPlugins(RecoverySettingsChunkSizePlugin.class).isEmpty()) { From e9c451998490085bcede0c6823780bf9e81b179a Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Thu, 27 Jul 2017 18:06:58 +0200 Subject: [PATCH 10/11] Remove classpathPlugins parameter --- .../java/org/elasticsearch/node/Node.java | 2 +- .../org/elasticsearch/plugins/Plugin.java | 4 +-- .../metadata/TemplateUpgradeServiceIT.java | 5 ++-- .../org/elasticsearch/tribe/TribePlugin.java | 15 +++-------- .../tribe/TribeIntegrationTests.java | 10 ++------ .../file/FileBasedDiscoveryPlugin.java | 3 +-- .../elasticsearch/tribe/TribeUnitTests.java | 25 ++++++++++++++----- .../test/InternalTestCluster.java | 2 +- 8 files changed, 31 insertions(+), 35 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java index 0b79d18e60946..11dfce5d6df02 100644 --- a/core/src/main/java/org/elasticsearch/node/Node.java +++ b/core/src/main/java/org/elasticsearch/node/Node.java @@ -393,7 +393,7 @@ protected Node(final Environment environment, Collection Collection pluginComponents = pluginsService.filterPlugins(Plugin.class).stream() .flatMap(p -> p.createComponents(client, clusterService, threadPool, resourceWatcherService, scriptModule.getScriptService(), xContentRegistry, environment, nodeEnvironment, - namedWriteableRegistry, classpathPlugins).stream()) + namedWriteableRegistry).stream()) .collect(Collectors.toList()); final RestController restController = actionModule.getRestController(); final NetworkModule networkModule = new NetworkModule(settings, false, pluginsService.filterPlugins(NetworkPlugin.class), diff --git a/core/src/main/java/org/elasticsearch/plugins/Plugin.java b/core/src/main/java/org/elasticsearch/plugins/Plugin.java index d1da0f5af2a40..82c8bf1bbcb18 100644 --- a/core/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/core/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -110,13 +110,11 @@ public Collection> getGuiceServiceClasses() * @param environment the environment for path and setting configurations * @param nodeEnvironment the node environment used coordinate access to the data paths * @param namedWriteableRegistry the registry for {@link NamedWriteable} object parsing - * @param classpathPlugins the plugins available on the classpath */ public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry, Environment environment, - NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry, - Collection> classpathPlugins) { + NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) { return Collections.emptyList(); } diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java b/core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java index f24e10665840f..c8d5cdc6c86db 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java @@ -71,13 +71,12 @@ public TestPlugin(Settings settings) { public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry, Environment environment, - NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry, - Collection> classpathPlugins) { + NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) { clusterService.getClusterSettings().addSettingsUpdateConsumer(UPDATE_TEMPLATE_DUMMY_SETTING, integer -> { logger.debug("the template dummy setting was updated to {}", integer); }); return super.createComponents(client, clusterService, threadPool, resourceWatcherService, scriptService, xContentRegistry, - environment, nodeEnvironment, namedWriteableRegistry, classpathPlugins); + environment, nodeEnvironment, namedWriteableRegistry); } @Override diff --git a/modules/tribe/src/main/java/org/elasticsearch/tribe/TribePlugin.java b/modules/tribe/src/main/java/org/elasticsearch/tribe/TribePlugin.java index 5efcaee0609e1..f77dea1dd306d 100644 --- a/modules/tribe/src/main/java/org/elasticsearch/tribe/TribePlugin.java +++ b/modules/tribe/src/main/java/org/elasticsearch/tribe/TribePlugin.java @@ -78,21 +78,14 @@ public Map> getDiscoveryTypes(ThreadPool threadPool, public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry, Environment environment, - NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry, - Collection> classpathPlugins) { + NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) { tribeService = new TribeService(settings, nodeEnvironment, clusterService, namedWriteableRegistry, - nodeBuilder(environment.configFile(), classpathPlugins)); + nodeBuilder(environment.configFile())); return Collections.singleton(tribeService); } - protected Function nodeBuilder(Path configPath, Collection> classpathPlugins) { - return settings -> new TribeNode(settings, configPath, classpathPlugins); - } - - private static class TribeNode extends Node { - private TribeNode(Settings settings, Path configPath, Collection> classpathPlugins) { - super(new Environment(settings, configPath), classpathPlugins); - } + protected Function nodeBuilder(Path configPath) { + return settings -> new Node(new Environment(settings, configPath)); } @Override diff --git a/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeIntegrationTests.java b/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeIntegrationTests.java index 6f21d1faa5ccf..fb77bcd33d0fe 100644 --- a/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeIntegrationTests.java +++ b/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeIntegrationTests.java @@ -155,14 +155,8 @@ public MockTribePlugin(Settings settings) { super(settings); } - protected Function nodeBuilder(Path configPath, Collection> classpathPlugins) { - return settings -> new MockTribeNode(settings, configPath, classpathPlugins); - } - - private static class MockTribeNode extends MockNode { - private MockTribeNode(Settings settings, Path configPath, Collection> classpathPlugins) { - super(new Environment(settings, configPath), classpathPlugins); - } + protected Function nodeBuilder(Path configPath) { + return settings -> new MockNode(new Environment(settings, configPath), internalCluster().getPlugins()); } } diff --git a/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java b/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java index 9f95970047bd2..0cd8176df83f5 100644 --- a/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java +++ b/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java @@ -71,8 +71,7 @@ public FileBasedDiscoveryPlugin(Settings settings) { public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry, Environment environment, - NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry, - Collection> classpathPlugins) { + NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) { final int concurrentConnects = UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_CONCURRENT_CONNECTS_SETTING.get(settings); final ThreadFactory threadFactory = EsExecutors.daemonThreadFactory(settings, "[file_based_discovery_resolve]"); fileBasedDiscoveryExecutorService = EsExecutors.newScaling( diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java index 1a36c01fe09fb..0ae69f7304036 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java @@ -23,6 +23,7 @@ import java.nio.file.Path; import java.util.Arrays; import java.util.List; +import java.util.function.Function; import org.apache.lucene.util.IOUtils; import org.elasticsearch.client.Client; @@ -55,6 +56,7 @@ @SuppressForbidden(reason = "modifies system properties intentionally") public class TribeUnitTests extends ESTestCase { + private static List> classpathPlugins; private static Node tribe1; private static Node tribe2; @@ -67,27 +69,28 @@ public static void createTribes() throws NodeValidationException { .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), 2) .build(); - final List> mockPlugins = Arrays.asList(MockTcpTransportPlugin.class, - TribeAwareTestZenDiscoveryPlugin.class, TribePlugin.class); + classpathPlugins = Arrays.asList(TribeAwareTestZenDiscoveryPlugin.class, MockTribePlugin.class, MockTcpTransportPlugin.class); + tribe1 = new MockNode( Settings.builder() .put(baseSettings) .put("cluster.name", "tribe1") .put("node.name", "tribe1_node") .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), random().nextLong()) - .build(), mockPlugins).start(); + .build(), classpathPlugins).start(); tribe2 = new MockNode( Settings.builder() .put(baseSettings) .put("cluster.name", "tribe2") .put("node.name", "tribe2_node") .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), random().nextLong()) - .build(), mockPlugins).start(); + .build(), classpathPlugins).start(); } @AfterClass public static void closeTribes() throws IOException { IOUtils.close(tribe1, tribe2); + classpathPlugins = null; tribe1 = null; tribe2 = null; } @@ -108,6 +111,18 @@ public Settings additionalSettings() { } } + public static class MockTribePlugin extends TribePlugin { + + public MockTribePlugin(Settings settings) { + super(settings); + } + + protected Function nodeBuilder(Path configPath) { + return settings -> new MockNode(new Environment(settings, configPath), classpathPlugins); + } + + } + public void testThatTribeClientsIgnoreGlobalConfig() throws Exception { assertTribeNodeSuccessfullyCreated(getDataPath("elasticsearch.yml").getParent()); assertWarnings("tribe nodes are deprecated in favor of cross-cluster search and will be removed in Elasticsearch 7.0.0"); @@ -122,8 +137,6 @@ private static void assertTribeNodeSuccessfullyCreated(Path configPath) throws E .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) .build(); - final List> classpathPlugins = - Arrays.asList(TribeAwareTestZenDiscoveryPlugin.class, TribePlugin.class, MockTcpTransportPlugin.class); try (Node node = new MockNode(settings, classpathPlugins, configPath).start()) { try (Client client = node.client()) { assertBusy(() -> { diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 72d2a690b5e02..655934fea723d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -378,7 +378,7 @@ private Settings getSettings(int nodeOrdinal, long nodeSeed, Settings others) { return builder.build(); } - private Collection> getPlugins() { + public Collection> getPlugins() { Set> plugins = new HashSet<>(nodeConfigurationSource.nodePlugins()); plugins.addAll(mockPlugins); return plugins; From be55c18afaf384a87801b69b111e19efd8f4e73f Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Fri, 28 Jul 2017 10:22:54 +0200 Subject: [PATCH 11/11] fix testTribeNodeDeprecation --- .../tribe/TribeServiceTests.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java b/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java index c6341d1f734e5..1d63ae1e0b723 100644 --- a/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java +++ b/modules/tribe/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java @@ -27,8 +27,10 @@ import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.env.Environment; import org.elasticsearch.node.MockNode; import org.elasticsearch.node.Node; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.TestCustomMetaData; import org.elasticsearch.transport.MockTcpTransportPlugin; @@ -42,6 +44,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; import static org.hamcrest.Matchers.instanceOf; @@ -181,6 +184,20 @@ public void testMergeCustomMetaDataFromMany() { assertEquals(mergedCustom.getData(), "data2"+String.valueOf(n)); } + public static class MockTribePlugin extends TribePlugin { + + static List> classpathPlugins = Arrays.asList(MockTribePlugin.class, MockTcpTransportPlugin.class); + + public MockTribePlugin(Settings settings) { + super(settings); + } + + protected Function nodeBuilder(Path configPath) { + return settings -> new MockNode(new Environment(settings, configPath), classpathPlugins); + } + + } + public void testTribeNodeDeprecation() throws IOException { final Path tempDir = createTempDir(); Settings.Builder settings = Settings.builder() @@ -196,7 +213,7 @@ public void testTribeNodeDeprecation() throws IOException { settings.put(tribeSetting + ClusterName.CLUSTER_NAME_SETTING.getKey(), clusterName) .put(tribeSetting + NetworkModule.TRANSPORT_TYPE_SETTING.getKey(), "mock-socket-network"); } - try (Node node = new MockNode(settings.build(), Arrays.asList(TribePlugin.class, MockTcpTransportPlugin.class))) { + try (Node node = new MockNode(settings.build(), MockTribePlugin.classpathPlugins)) { if (tribeServiceEnable) { assertWarnings("tribe nodes are deprecated in favor of cross-cluster search and will be removed in Elasticsearch 7.0.0"); }