Skip to content

Commit

Permalink
Move unicast_hosts.txt to the top-level config dir
Browse files Browse the repository at this point in the history 
Support both paths for BWC reasons, but emit a deprecation warning if the old
one is used.

Also, replace the unnecessary full stack trace emitted if the file isn't there
with a single-line warning. A stack trace is still emitted if the file seems to
be there but reading it fails.
  • Loading branch information
@DaveCTurner
DaveCTurner committed Aug 29, 2018
1 parent e2ef488 commit 8f9fcef
Show file tree
Hide file tree
Showing 2 changed files with 89 additions and 50 deletions.
51 changes: 31 additions & 20 deletions server/src/main/java/org/elasticsearch/discovery/zen/FileBasedUnicastHostsProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,12 @@
package org.elasticsearch.discovery.zen;

import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.logging.log4j.util.Supplier;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.util.Collections;
import java.util.List;
Expand All @@ -52,30 +49,44 @@ public class FileBasedUnicastHostsProvider extends AbstractComponent implements
public static final String UNICAST_HOSTS_FILE = "unicast_hosts.txt";

private final Path unicastHostsFilePath;
private final Path legacyUnicastHostsFilePath;

public FileBasedUnicastHostsProvider(Settings settings, Path configFile) {
super(settings);
this.unicastHostsFilePath = configFile.resolve("discovery-file").resolve(UNICAST_HOSTS_FILE);
this.unicastHostsFilePath = configFile.resolve(UNICAST_HOSTS_FILE);
this.legacyUnicastHostsFilePath = configFile.resolve("discovery-file").resolve(UNICAST_HOSTS_FILE);
}

@Override
public List<TransportAddress> buildDynamicHosts(HostsResolver hostsResolver) {
List<String> hostsList;
try (Stream<String> lines = Files.lines(unicastHostsFilePath)) {
hostsList = lines.filter(line -> line.startsWith("#") == false) // lines starting with `#` are comments
.collect(Collectors.toList());
} catch (FileNotFoundException | NoSuchFileException e) {
logger.warn((Supplier<?>) () -> new ParameterizedMessage("[discovery-file] Failed to find unicast hosts file [{}]",
unicastHostsFilePath), e);
hostsList = Collections.emptyList();
private List<String> getHostsList() {
if (Files.exists(unicastHostsFilePath)) {
return readFileContents(unicastHostsFilePath);
}

if (Files.exists(legacyUnicastHostsFilePath)) {
deprecationLogger.deprecated("Found dynamic hosts list at [{}] but this path is deprecated. This list should be at [{}] " +
"instead. Support for the deprecated path will be removed in future.", legacyUnicastHostsFilePath, unicastHostsFilePath);
return readFileContents(legacyUnicastHostsFilePath);
}

logger.warn("expected, but did not find, a dynamic hosts list at [{}]", unicastHostsFilePath);

return Collections.emptyList();
}

private List<String> readFileContents(Path path) {
try (Stream<String> lines = Files.lines(path)) {
return lines.filter(line -> line.startsWith("#") == false) // lines starting with `#` are comments
.collect(Collectors.toList());
} catch (IOException e) {
logger.warn((Supplier<?>) () -> new ParameterizedMessage("[discovery-file] Error reading unicast hosts file [{}]",
unicastHostsFilePath), e);
hostsList = Collections.emptyList();
logger.warn(() -> new ParameterizedMessage("failed to read file [{}]", unicastHostsFilePath), e);
return Collections.emptyList();
}
}

final List<TransportAddress> dynamicHosts = hostsResolver.resolveHosts(hostsList, 1);
logger.debug("[discovery-file] Using dynamic discovery nodes {}", dynamicHosts);
return dynamicHosts;
@Override
public List<TransportAddress> buildDynamicHosts(HostsResolver hostsResolver) {
final List<TransportAddress> transportAddresses = hostsResolver.resolveHosts(getHostsList(), 1);
logger.debug("seed addresses: {}", transportAddresses);
return transportAddresses;
}
}
88 changes: 58 additions & 30 deletions server/src/test/java/org/elasticsearch/discovery/zen/FileBasedUnicastHostsProviderTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,11 @@

public class FileBasedUnicastHostsProviderTests extends ESTestCase {

private boolean legacyLocation;
private ThreadPool threadPool;
private ExecutorService executorService;
private MockTransportService transportService;
private Path configPath;

@Before
public void setUp() throws Exception {
Expand All @@ -78,23 +80,20 @@ public void tearDown() throws Exception {

@Before
public void createTransportSvc() {
MockTcpTransport transport =
new MockTcpTransport(Settings.EMPTY,
threadPool,
BigArrays.NON_RECYCLING_INSTANCE,
new NoneCircuitBreakerService(),
new NamedWriteableRegistry(Collections.emptyList()),
new NetworkService(Collections.emptyList())) {
@Override
public BoundTransportAddress boundAddress() {
return new BoundTransportAddress(
new TransportAddress[]{new TransportAddress(InetAddress.getLoopbackAddress(), 9300)},
new TransportAddress(InetAddress.getLoopbackAddress(), 9300)
);
}
};
final MockTcpTransport transport = new MockTcpTransport(Settings.EMPTY, threadPool, BigArrays.NON_RECYCLING_INSTANCE,
new NoneCircuitBreakerService(),
new NamedWriteableRegistry(Collections.emptyList()),
new NetworkService(Collections.emptyList())) {
@Override
public BoundTransportAddress boundAddress() {
return new BoundTransportAddress(
new TransportAddress[]{new TransportAddress(InetAddress.getLoopbackAddress(), 9300)},
new TransportAddress(InetAddress.getLoopbackAddress(), 9300)
);
}
};
transportService = new MockTransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
null);
null);
}

public void testBuildDynamicNodes() throws Exception {
Expand All @@ -109,16 +108,26 @@ public void testBuildDynamicNodes() throws Exception {
assertEquals(9300, nodes.get(2).getPort());
}

public void testBuildDynamicNodesLegacyLocation() throws Exception {
legacyLocation = true;
testBuildDynamicNodes();
assertDeprecatedLocationWarning();
}

public void testEmptyUnicastHostsFile() throws Exception {
final List<String> hostEntries = Collections.emptyList();
final List<TransportAddress> addresses = setupAndRunHostProvider(hostEntries);
assertEquals(0, addresses.size());
}

public void testUnicastHostsDoesNotExist() throws Exception {
final Settings settings = Settings.builder()
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
.build();
public void testEmptyUnicastHostsFileLegacyLocation() throws Exception {
legacyLocation = true;
testEmptyUnicastHostsFile();
assertDeprecatedLocationWarning();
}

public void testUnicastHostsDoesNotExist() {
final Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build();
final FileBasedUnicastHostsProvider provider = new FileBasedUnicastHostsProvider(settings, createTempDir().toAbsolutePath());
final List<TransportAddress> addresses = provider.buildDynamicHosts((hosts, limitPortCounts) ->
UnicastZenPing.resolveHostsLists(executorService, logger, hosts, limitPortCounts, transportService,
Expand All @@ -127,41 +136,60 @@ public void testUnicastHostsDoesNotExist() throws Exception {
}

public void testInvalidHostEntries() throws Exception {
List<String> hostEntries = Arrays.asList("192.168.0.1:9300:9300");
List<TransportAddress> addresses = setupAndRunHostProvider(hostEntries);
final List<String> hostEntries = Arrays.asList("192.168.0.1:9300:9300");
final List<TransportAddress> addresses = setupAndRunHostProvider(hostEntries);
assertEquals(0, addresses.size());
}

public void testInvalidHostEntriesLegacyLocation() throws Exception {
legacyLocation = true;
testInvalidHostEntries();
assertDeprecatedLocationWarning();
}

public void testSomeInvalidHostEntries() throws Exception {
List<String> hostEntries = Arrays.asList("192.168.0.1:9300:9300", "192.168.0.1:9301");
List<TransportAddress> addresses = setupAndRunHostProvider(hostEntries);
final List<String> hostEntries = Arrays.asList("192.168.0.1:9300:9300", "192.168.0.1:9301");
final List<TransportAddress> addresses = setupAndRunHostProvider(hostEntries);
assertEquals(1, addresses.size()); // only one of the two is valid and will be used
assertEquals("192.168.0.1", addresses.get(0).getAddress());
assertEquals(9301, addresses.get(0).getPort());
}

public void testSomeInvalidHostEntriesLegacyLocation() throws Exception {
legacyLocation = true;
testSomeInvalidHostEntries();
assertDeprecatedLocationWarning();
}

// sets up the config dir, writes to the unicast hosts file in the config dir,
// and then runs the file-based unicast host provider to get the list of discovery nodes
private List<TransportAddress> setupAndRunHostProvider(final List<String> hostEntries) throws IOException {
final Path homeDir = createTempDir();
final Settings settings = Settings.builder()
.put(Environment.PATH_HOME_SETTING.getKey(), homeDir)
.build();
final Path configPath;
.put(Environment.PATH_HOME_SETTING.getKey(), homeDir)
.build();
if (randomBoolean()) {
configPath = homeDir.resolve("config");
} else {
configPath = createTempDir();
}
final Path discoveryFilePath = configPath.resolve("discovery-file");
final Path discoveryFilePath = legacyLocation ? configPath.resolve("discovery-file") : configPath;
Files.createDirectories(discoveryFilePath);
final Path unicastHostsPath = discoveryFilePath.resolve(UNICAST_HOSTS_FILE);
try (BufferedWriter writer = Files.newBufferedWriter(unicastHostsPath)) {
writer.write(String.join("\n", hostEntries));
}

return new FileBasedUnicastHostsProvider(settings, configPath).buildDynamicHosts((hosts, limitPortCounts) ->
UnicastZenPing.resolveHostsLists(executorService, logger, hosts, limitPortCounts, transportService,
TimeValue.timeValueSeconds(10)));
UnicastZenPing.resolveHostsLists(executorService, logger, hosts, limitPortCounts, transportService,
TimeValue.timeValueSeconds(10)));
}

private void assertDeprecatedLocationWarning() {
assertWarnings("Found dynamic hosts list at [" +
configPath.resolve("discovery-file").resolve(UNICAST_HOSTS_FILE) +
"] but this path is deprecated. This list should be at [" +
configPath.resolve(UNICAST_HOSTS_FILE) +
"] instead. Support for the deprecated path will be removed in future.");
}
}

0 comments on commit 8f9fcef

Please sign in to comment.