diff --git a/Vagrantfile b/Vagrantfile index 6f81ba0273c9f..6761fec07dab2 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -337,6 +337,7 @@ export BATS=/project/build/bats export BATS_UTILS=/project/build/packaging/bats/utils export BATS_TESTS=/project/build/packaging/bats/tests export PACKAGING_ARCHIVES=/project/build/packaging/archives +export PACKAGING_TESTS=/project/build/packaging/tests VARS cat \<\ /etc/sudoers.d/elasticsearch_vars Defaults env_keep += "ZIP" @@ -347,6 +348,7 @@ Defaults env_keep += "BATS" Defaults env_keep += "BATS_UTILS" Defaults env_keep += "BATS_TESTS" Defaults env_keep += "PACKAGING_ARCHIVES" +Defaults env_keep += "PACKAGING_TESTS" SUDOERS_VARS chmod 0440 /etc/sudoers.d/elasticsearch_vars SHELL diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index d59d25ddc55f7..4b70950866251 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -561,8 +561,6 @@ class BuildPlugin implements Plugin { */ List html4Projects = [ ':server', - ':libs:elasticsearch-core', - ':test:framework', ':x-pack:plugin:core', ] if (false == html4Projects.contains(project.path)) { diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy index 264a1e0f8ac17..e9b664a5a31b7 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy @@ -41,6 +41,9 @@ class VagrantPropertiesExtension { @Input Boolean inheritTestUtils + @Input + String testClass + VagrantPropertiesExtension(List availableBoxes) { this.boxes = availableBoxes this.batsDir = 'src/test/resources/packaging' diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy index 7a0b9f96781df..bb85359ae3f07 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy @@ -51,6 +51,7 @@ class VagrantTestPlugin implements Plugin { static List UPGRADE_FROM_ARCHIVES = ['rpm', 'deb'] private static final PACKAGING_CONFIGURATION = 'packaging' + private static final PACKAGING_TEST_CONFIGURATION = 'packagingTest' private static final BATS = 'bats' private static final String BATS_TEST_COMMAND ="cd \$PACKAGING_ARCHIVES && sudo bats --tap \$BATS_TESTS/*.$BATS" private static final String PLATFORM_TEST_COMMAND ="rm -rf ~/elasticsearch && rsync -r /elasticsearch/ ~/elasticsearch && cd ~/elasticsearch && ./gradlew test integTest" @@ -66,6 +67,7 @@ class VagrantTestPlugin implements Plugin { // Creates custom configurations for Bats testing files (and associated scripts and archives) createPackagingConfiguration(project) + project.configurations.create(PACKAGING_TEST_CONFIGURATION) // Creates all the main Vagrant tasks createVagrantTasks(project) @@ -144,10 +146,12 @@ class VagrantTestPlugin implements Plugin { } private static void createCleanTask(Project project) { - project.tasks.create('clean', Delete.class) { - description 'Clean the project build directory' - group 'Build' - delete project.buildDir + if (project.tasks.findByName('clean') == null) { + project.tasks.create('clean', Delete.class) { + description 'Clean the project build directory' + group 'Build' + delete project.buildDir + } } } @@ -174,6 +178,18 @@ class VagrantTestPlugin implements Plugin { from project.configurations[PACKAGING_CONFIGURATION] } + File testsDir = new File(packagingDir, 'tests') + Copy copyPackagingTests = project.tasks.create('copyPackagingTests', Copy) { + into testsDir + from project.configurations[PACKAGING_TEST_CONFIGURATION] + } + + Task createTestRunnerScript = project.tasks.create('createTestRunnerScript', FileContentsTask) { + dependsOn copyPackagingTests + file "${testsDir}/run-tests.sh" + contents "java -cp \"\$PACKAGING_TESTS/*\" org.junit.runner.JUnitCore ${-> project.extensions.esvagrant.testClass}" + } + Task createVersionFile = project.tasks.create('createVersionFile', FileContentsTask) { dependsOn copyPackagingArchives file "${archivesDir}/version" @@ -234,7 +250,8 @@ class VagrantTestPlugin implements Plugin { Task vagrantSetUpTask = project.tasks.create('setupPackagingTest') vagrantSetUpTask.dependsOn 'vagrantCheckVersion' - vagrantSetUpTask.dependsOn copyPackagingArchives, createVersionFile, createUpgradeFromFile, createUpgradeIsOssFile + vagrantSetUpTask.dependsOn copyPackagingArchives, copyPackagingTests, createTestRunnerScript + vagrantSetUpTask.dependsOn createVersionFile, createUpgradeFromFile, createUpgradeIsOssFile vagrantSetUpTask.dependsOn copyBatsTests, copyBatsUtils } @@ -393,20 +410,29 @@ class VagrantTestPlugin implements Plugin { packagingTest.dependsOn(batsPackagingTest) } - // This task doesn't do anything yet. In the future it will execute a jar containing tests on the vm - Task groovyPackagingTest = project.tasks.create("vagrant${boxTask}#groovyPackagingTest") - groovyPackagingTest.dependsOn(up) - groovyPackagingTest.finalizedBy(halt) + Task javaPackagingTest = project.tasks.create("vagrant${boxTask}#javaPackagingTest", VagrantCommandTask) { + command 'ssh' + boxName box + environmentVars vagrantEnvVars + dependsOn up, setupPackagingTest + finalizedBy halt + args '--command', "bash \"\$PACKAGING_TESTS/run-tests.sh\"" + } + + // todo remove this onlyIf after all packaging tests are consolidated + javaPackagingTest.onlyIf { + project.extensions.esvagrant.testClass != null + } - TaskExecutionAdapter groovyPackagingReproListener = createReproListener(project, groovyPackagingTest.path) - groovyPackagingTest.doFirst { - project.gradle.addListener(groovyPackagingReproListener) + TaskExecutionAdapter javaPackagingReproListener = createReproListener(project, javaPackagingTest.path) + javaPackagingTest.doFirst { + project.gradle.addListener(javaPackagingReproListener) } - groovyPackagingTest.doLast { - project.gradle.removeListener(groovyPackagingReproListener) + javaPackagingTest.doLast { + project.gradle.removeListener(javaPackagingReproListener) } if (project.extensions.esvagrant.boxes.contains(box)) { - packagingTest.dependsOn(groovyPackagingTest) + packagingTest.dependsOn(javaPackagingTest) } Task platform = project.tasks.create("vagrant${boxTask}#platformTest", VagrantCommandTask) { diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle index f2fc297a9e4c8..9fa06021236a2 100644 --- a/distribution/archives/build.gradle +++ b/distribution/archives/build.gradle @@ -217,6 +217,24 @@ subprojects { } check.dependsOn checkNotice + if (project.name == 'zip' || project.name == 'tar') { + task checkMlCppNotice { + dependsOn buildDist, checkExtraction + onlyIf toolExists + doLast { + // this is just a small sample from the C++ notices, the idea being that if we've added these lines we've probably added all the required lines + final List expectedLines = Arrays.asList("Apache log4cxx", "Boost Software License - Version 1.0 - August 17th, 2003") + final Path noticePath = archiveExtractionDir.toPath().resolve("elasticsearch-${VersionProperties.elasticsearch}/modules/x-pack/x-pack-ml/NOTICE.txt") + final List actualLines = Files.readAllLines(noticePath) + for (final String expectedLine : expectedLines) { + if (actualLines.contains(expectedLine) == false) { + throw new GradleException("expected [${noticePath}] to contain [${expectedLine}] but it did not") + } + } + } + } + check.dependsOn checkMlCppNotice + } } /***************************************************************************** diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc index b46799d7a8edf..1ef6534616f18 100644 --- a/docs/CHANGELOG.asciidoc +++ b/docs/CHANGELOG.asciidoc @@ -1,7 +1,7 @@ // Use these for links to issue and pulls. Note issues and pulls redirect one to // each other on Github, so don't worry too much on using the right prefix. -// :issue: https://github.com/elastic/elasticsearch/issues/ -// :pull: https://github.com/elastic/elasticsearch/pull/ +:issue: https://github.com/elastic/elasticsearch/issues/ +:pull: https://github.com/elastic/elasticsearch/pull/ = Elasticsearch Release Notes diff --git a/docs/reference/cluster/reroute.asciidoc b/docs/reference/cluster/reroute.asciidoc index 0bc8610e0c792..f076a7b83585a 100644 --- a/docs/reference/cluster/reroute.asciidoc +++ b/docs/reference/cluster/reroute.asciidoc @@ -1,13 +1,12 @@ [[cluster-reroute]] == Cluster Reroute -The reroute command allows to explicitly execute a cluster reroute -allocation command including specific commands. For example, a shard can -be moved from one node to another explicitly, an allocation can be -canceled, or an unassigned shard can be explicitly allocated on a -specific node. +The reroute command allows for manual changes to the allocation of individual +shards in the cluster. For example, a shard can be moved from one node to +another explicitly, an allocation can be cancelled, and an unassigned shard can +be explicitly allocated to a specific node. -Here is a short example of how a simple reroute API call: +Here is a short example of a simple reroute API call: [source,js] -------------------------------------------------- @@ -32,59 +31,53 @@ POST /_cluster/reroute // CONSOLE // TEST[skip:doc tests run with only a single node] -An important aspect to remember is the fact that once when an allocation -occurs, the cluster will aim at re-balancing its state back to an even -state. For example, if the allocation includes moving a shard from -`node1` to `node2`, in an `even` state, then another shard will be moved -from `node2` to `node1` to even things out. +It is important to note that that after processing any reroute commands +Elasticsearch will perform rebalancing as normal (respecting the values of +settings such as `cluster.routing.rebalance.enable`) in order to remain in a +balanced state. For example, if the requested allocation includes moving a +shard from `node1` to `node2` then this may cause a shard to be moved from +`node2` back to `node1` to even things out. -The cluster can be set to disable allocations, which means that only the -explicitly allocations will be performed. Obviously, only once all -commands has been applied, the cluster will aim to be re-balance its -state. +The cluster can be set to disable allocations using the +`cluster.routing.allocation.enable` setting. If allocations are disabled then +the only allocations that will be performed are explicit ones given using the +`reroute` command, and consequent allocations due to rebalancing. -Another option is to run the commands in `dry_run` (as a URI flag, or in -the request body). This will cause the commands to apply to the current -cluster state, and return the resulting cluster after the commands (and -re-balancing) has been applied. +It is possible to run `reroute` commands in "dry run" mode by using the +`?dry_run` URI query parameter, or by passing `"dry_run": true` in the request +body. This will calculate the result of applying the commands to the current +cluster state, and return the resulting cluster state after the commands (and +re-balancing) has been applied, but will not actually perform the requested +changes. -If the `explain` parameter is specified, a detailed explanation of why the -commands could or could not be executed is returned. +If the `?explain` URI query parameter is included then a detailed explanation +of why the commands could or could not be executed is included in the response. The commands supported are: `move`:: Move a started shard from one node to another node. Accepts `index` and `shard` for index name and shard number, `from_node` for the - node to move the shard `from`, and `to_node` for the node to move the + node to move the shard from, and `to_node` for the node to move the shard to. `cancel`:: - Cancel allocation of a shard (or recovery). Accepts `index` - and `shard` for index name and shard number, and `node` for the node to - cancel the shard allocation on. It also accepts `allow_primary` flag to - explicitly specify that it is allowed to cancel allocation for a primary - shard. This can be used to force resynchronization of existing replicas - from the primary shard by cancelling them and allowing them to be - reinitialized through the standard reallocation process. + Cancel allocation of a shard (or recovery). Accepts `index` and `shard` for + index name and shard number, and `node` for the node to cancel the shard + allocation on. This can be used to force resynchronization of existing + replicas from the primary shard by cancelling them and allowing them to be + reinitialized through the standard recovery process. By default only + replica shard allocations can be cancelled. If it is necessary to cancel + the allocation of a primary shard then the `allow_primary` flag must also + be included in the request. `allocate_replica`:: - Allocate an unassigned replica shard to a node. Accepts the - `index` and `shard` for index name and shard number, and `node` to - allocate the shard to. Takes <> into account. - -Two more commands are available that allow the allocation of a primary shard -to a node. These commands should however be used with extreme care, as primary -shard allocation is usually fully automatically handled by Elasticsearch. -Reasons why a primary shard cannot be automatically allocated include the following: - -- A new index was created but there is no node which satisfies the allocation deciders. -- An up-to-date shard copy of the data cannot be found on the current data nodes in -the cluster. To prevent data loss, the system does not automatically promote a stale -shard copy to primary. + Allocate an unassigned replica shard to a node. Accepts `index` and `shard` + for index name and shard number, and `node` to allocate the shard to. Takes + <> into account. [float] -=== Retry failed shards +=== Retrying failed allocations The cluster will attempt to allocate a shard a maximum of `index.allocation.max_retries` times in a row (defaults to `5`), before giving @@ -93,36 +86,48 @@ structural problems such as having an analyzer which refers to a stopwords file which doesn't exist on all nodes. Once the problem has been corrected, allocation can be manually retried by -calling the <> API with `?retry_failed`, which -will attempt a single retry round for these shards. +calling the <> API with the `?retry_failed` URI +query parameter, which will attempt a single retry round for these shards. [float] === Forced allocation on unrecoverable errors +Two more commands are available that allow the allocation of a primary shard to +a node. These commands should however be used with extreme care, as primary +shard allocation is usually fully automatically handled by Elasticsearch. +Reasons why a primary shard cannot be automatically allocated include the +following: + +- A new index was created but there is no node which satisfies the allocation + deciders. +- An up-to-date shard copy of the data cannot be found on the current data + nodes in the cluster. To prevent data loss, the system does not automatically +promote a stale shard copy to primary. + The following two commands are dangerous and may result in data loss. They are -meant to be used in cases where the original data can not be recovered and the cluster -administrator accepts the loss. If you have suffered a temporary issue that has been -fixed, please see the `retry_failed` flag described above. +meant to be used in cases where the original data can not be recovered and the +cluster administrator accepts the loss. If you have suffered a temporary issue +that can be fixed, please see the `retry_failed` flag described above. To +emphasise: if these commands are performed and then a node joins the cluster +that holds a copy of the affected shard then the copy on the newly-joined node +will be deleted or overwritten. `allocate_stale_primary`:: Allocate a primary shard to a node that holds a stale copy. Accepts the - `index` and `shard` for index name and shard number, and `node` to - allocate the shard to. Using this command may lead to data loss - for the provided shard id. If a node which has the good copy of the - data rejoins the cluster later on, that data will be overwritten with - the data of the stale copy that was forcefully allocated with this - command. To ensure that these implications are well-understood, - this command requires the special field `accept_data_loss` to be - explicitly set to `true` for it to work. + `index` and `shard` for index name and shard number, and `node` to allocate + the shard to. Using this command may lead to data loss for the provided + shard id. If a node which has the good copy of the data rejoins the cluster + later on, that data will be deleted or overwritten with the data of the + stale copy that was forcefully allocated with this command. To ensure that + these implications are well-understood, this command requires the flag + `accept_data_loss` to be explicitly set to `true`. `allocate_empty_primary`:: - Allocate an empty primary shard to a node. Accepts the - `index` and `shard` for index name and shard number, and `node` to - allocate the shard to. Using this command leads to a complete loss - of all data that was indexed into this shard, if it was previously - started. If a node which has a copy of the - data rejoins the cluster later on, that data will be deleted! - To ensure that these implications are well-understood, - this command requires the special field `accept_data_loss` to be - explicitly set to `true` for it to work. + Allocate an empty primary shard to a node. Accepts the `index` and `shard` + for index name and shard number, and `node` to allocate the shard to. Using + this command leads to a complete loss of all data that was indexed into + this shard, if it was previously started. If a node which has a copy of the + data rejoins the cluster later on, that data will be deleted. To ensure + that these implications are well-understood, this command requires the flag + `accept_data_loss` to be explicitly set to `true`. diff --git a/docs/reference/cluster/state.asciidoc b/docs/reference/cluster/state.asciidoc index d0ff3290c74d3..a20ff04d83f4a 100644 --- a/docs/reference/cluster/state.asciidoc +++ b/docs/reference/cluster/state.asciidoc @@ -15,6 +15,12 @@ of the cluster state (its size when serialized for transmission over the network), and the cluster state itself, which can be filtered to only retrieve the parts of interest, as described below. +The cluster's `cluster_uuid` is also returned as part of the top-level +response, in addition to the `metadata` section. added[6.4.0] + +NOTE: While the cluster is still forming, it is possible for the `cluster_uuid` + to be `_na_` as well as the cluster state's version to be `-1`. + By default, the cluster state request is routed to the master node, to ensure that the latest cluster state is returned. For debugging purposes, you can retrieve the cluster state local to a diff --git a/docs/reference/modules/cluster/disk_allocator.asciidoc b/docs/reference/modules/cluster/disk_allocator.asciidoc index 0f43d9fcd30c9..d93453a49e8ed 100644 --- a/docs/reference/modules/cluster/disk_allocator.asciidoc +++ b/docs/reference/modules/cluster/disk_allocator.asciidoc @@ -1,9 +1,9 @@ [[disk-allocator]] === Disk-based Shard Allocation -Elasticsearch factors in the available disk space on a node before deciding -whether to allocate new shards to that node or to actively relocate shards -away from that node. +Elasticsearch considers the available disk space on a node before deciding +whether to allocate new shards to that node or to actively relocate shards away +from that node. Below are the settings that can be configured in the `elasticsearch.yml` config file or updated dynamically on a live cluster with the @@ -15,29 +15,33 @@ file or updated dynamically on a live cluster with the `cluster.routing.allocation.disk.watermark.low`:: - Controls the low watermark for disk usage. It defaults to 85%, meaning ES will - not allocate new shards to nodes once they have more than 85% disk used. It - can also be set to an absolute byte value (like 500mb) to prevent ES from - allocating shards if less than the configured amount of space is available. + Controls the low watermark for disk usage. It defaults to `85%`, meaning + that Elasticsearch will not allocate shards to nodes that have more than + 85% disk used. It can also be set to an absolute byte value (like `500mb`) + to prevent Elasticsearch from allocating shards if less than the specified + amount of space is available. This setting has no effect on the primary + shards of newly-created indices or, specifically, any shards that have + never previously been allocated. `cluster.routing.allocation.disk.watermark.high`:: - Controls the high watermark. It defaults to 90%, meaning ES will attempt to - relocate shards to another node if the node disk usage rises above 90%. It can - also be set to an absolute byte value (similar to the low watermark) to - relocate shards once less than the configured amount of space is available on - the node. + Controls the high watermark. It defaults to `90%`, meaning that + Elasticsearch will attempt to relocate shards away from a node whose disk + usage is above 90%. It can also be set to an absolute byte value (similarly + to the low watermark) to relocate shards away from a node if it has less + than the specified amount of free space. This setting affects the + allocation of all shards, whether previously allocated or not. `cluster.routing.allocation.disk.watermark.flood_stage`:: + -- -Controls the flood stage watermark. It defaults to 95%, meaning ES enforces -a read-only index block (`index.blocks.read_only_allow_delete`) on every -index that has one or more shards allocated on the node that has at least -one disk exceeding the flood stage. This is a last resort to prevent nodes -from running out of disk space. The index block must be released manually -once there is enough disk space available to allow indexing operations to -continue. +Controls the flood stage watermark. It defaults to 95%, meaning that +Elasticsearch enforces a read-only index block +(`index.blocks.read_only_allow_delete`) on every index that has one or more +shards allocated on the node that has at least one disk exceeding the flood +stage. This is a last resort to prevent nodes from running out of disk space. +The index block must be released manually once there is enough disk space +available to allow indexing operations to continue. NOTE: You can not mix the usage of percentage values and byte values within these settings. Either all are set to percentage values, or all are set to byte @@ -67,12 +71,12 @@ PUT /twitter/_settings `cluster.routing.allocation.disk.include_relocations`:: Defaults to +true+, which means that Elasticsearch will take into account - shards that are currently being relocated to the target node when computing a - node's disk usage. Taking relocating shards' sizes into account may, however, - mean that the disk usage for a node is incorrectly estimated on the high side, - since the relocation could be 90% complete and a recently retrieved disk usage - would include the total size of the relocating shard as well as the space - already used by the running relocation. + shards that are currently being relocated to the target node when computing + a node's disk usage. Taking relocating shards' sizes into account may, + however, mean that the disk usage for a node is incorrectly estimated on + the high side, since the relocation could be 90% complete and a recently + retrieved disk usage would include the total size of the relocating shard + as well as the space already used by the running relocation. NOTE: Percentage values refer to used disk space, while byte values refer to diff --git a/libs/elasticsearch-core/src/main/java/org/elasticsearch/common/Booleans.java b/libs/elasticsearch-core/src/main/java/org/elasticsearch/common/Booleans.java index 7447f0111f7e2..8687785796508 100644 --- a/libs/elasticsearch-core/src/main/java/org/elasticsearch/common/Booleans.java +++ b/libs/elasticsearch-core/src/main/java/org/elasticsearch/common/Booleans.java @@ -107,7 +107,7 @@ public static Boolean parseBoolean(String value, Boolean defaultValue) { } /** - * Returns false if text is in false, 0, off, no; else, true + * Returns {@code false} if text is in "false", "0", "off", "no"; else, {@code true}. * * @deprecated Only kept to provide automatic upgrades for pre 6.0 indices. Use {@link #parseBoolean(String, Boolean)} instead. */ @@ -119,9 +119,7 @@ public static Boolean parseBooleanLenient(String value, Boolean defaultValue) { return parseBooleanLenient(value, false); } /** - * Returns true iff the value is neither of the following: - * false, 0, off, no - * otherwise false + * Returns {@code false} if text is in "false", "0", "off", "no"; else, {@code true}. * * @deprecated Only kept to provide automatic upgrades for pre 6.0 indices. Use {@link #parseBoolean(String, boolean)} instead. */ @@ -134,21 +132,21 @@ public static boolean parseBooleanLenient(String value, boolean defaultValue) { } /** - * @return true iff the value is false, otherwise false. + * @return {@code true} iff the value is "false", otherwise {@code false}. */ public static boolean isFalse(String value) { return "false".equals(value); } /** - * @return true iff the value is true, otherwise false + * @return {@code true} iff the value is "true", otherwise {@code false}. */ public static boolean isTrue(String value) { return "true".equals(value); } /** - * Returns false if text is in false, 0, off, no; else, true + * Returns {@code false} if text is in "false", "0", "off", "no"; else, {@code true}. * * @deprecated Only kept to provide automatic upgrades for pre 6.0 indices. Use {@link #parseBoolean(char[], int, int, boolean)} instead */ diff --git a/libs/elasticsearch-core/src/main/java/org/elasticsearch/core/internal/io/IOUtils.java b/libs/elasticsearch-core/src/main/java/org/elasticsearch/core/internal/io/IOUtils.java index 4108992fb1f59..67663516167d5 100644 --- a/libs/elasticsearch-core/src/main/java/org/elasticsearch/core/internal/io/IOUtils.java +++ b/libs/elasticsearch-core/src/main/java/org/elasticsearch/core/internal/io/IOUtils.java @@ -37,11 +37,11 @@ public final class IOUtils { private IOUtils() { - + // Static utils methods } /** - * Closes all given Closeables. Some of the Closeables may be null; they are + * Closes all given {@link Closeable}s. Some of the {@linkplain Closeable}s may be null; they are * ignored. After everything is closed, the method either throws the first exception it hit * while closing with other exceptions added as suppressed, or completes normally if there were * no exceptions. @@ -53,7 +53,7 @@ public static void close(final Closeable... objects) throws IOException { } /** - * Closes all given Closeables. Some of the Closeables may be null; they are + * Closes all given {@link Closeable}s. Some of the {@linkplain Closeable}s may be null; they are * ignored. After everything is closed, the method adds any exceptions as suppressed to the * original exception, or throws the first exception it hit if {@code Exception} is null. If * no exceptions are encountered and the passed in exception is null, it completes normally. @@ -65,7 +65,7 @@ public static void close(final Exception e, final Closeable... objects) throws I } /** - * Closes all given Closeables. Some of the Closeables may be null; they are + * Closes all given {@link Closeable}s. Some of the {@linkplain Closeable}s may be null; they are * ignored. After everything is closed, the method either throws the first exception it hit * while closing with other exceptions added as suppressed, or completes normally if there were * no exceptions. diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle index 2b1ffb280819c..52a6bb1efb5f5 100644 --- a/qa/vagrant/build.gradle +++ b/qa/vagrant/build.gradle @@ -1,3 +1,5 @@ +import org.elasticsearch.gradle.precommit.PrecommitTasks + /* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with @@ -17,8 +19,23 @@ * under the License. */ -apply plugin: 'elasticsearch.vagrantsupport' -apply plugin: 'elasticsearch.vagrant' +plugins { + id 'java' + id 'elasticsearch.build' + id 'elasticsearch.vagrantsupport' + id 'elasticsearch.vagrant' +} + +dependencies { + compile "junit:junit:${versions.junit}" + compile "org.hamcrest:hamcrest-core:${versions.hamcrest}" + + // needs to be on the classpath for JarHell + testRuntime project(':libs:elasticsearch-core') + + // pulls in the jar built by this project and its dependencies + packagingTest project(path: project.path, configuration: 'runtime') +} List plugins = [] for (Project subproj : project.rootProject.subprojects) { @@ -39,3 +56,20 @@ setupPackagingTest { expectedPlugins.setText(plugins.join('\n'), 'UTF-8') } } + +esvagrant { + testClass 'org.elasticsearch.packaging.PackagingTests' +} + +forbiddenApisMain { + signaturesURLs = [ + PrecommitTasks.getResource('/forbidden/jdk-signatures.txt') + ] +} + +// we don't have additional tests for the tests themselves +tasks.test.enabled = false + +// this project doesn't get published +tasks.dependencyLicenses.enabled = false +tasks.dependenciesInfo.enabled = false diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/PackagingTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/PackagingTests.java new file mode 100644 index 0000000000000..0b5e7a3b6e0d2 --- /dev/null +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/PackagingTests.java @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.packaging; + +import org.junit.Test; + +/** + * This class doesn't have any tests yet + */ +public class PackagingTests { + + @Test + public void testDummy() {} +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.state/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.state/10_basic.yml index e3af21412ca7b..ae9637c08dd55 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.state/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.state/10_basic.yml @@ -2,7 +2,7 @@ "get cluster state": - do: cluster.state: {} - + - is_true: master_node --- @@ -18,3 +18,18 @@ - is_true: master_node - gte: { compressed_size_in_bytes: 50 } - is_true: compressed_size + +--- +"get cluster state returns cluster_uuid at the top level": + - skip: + version: " - 6.3.99" + reason: "cluster state including cluster_uuid at the top level is new in v6.4.0 and higher" + + - do: + cluster.state: + human: true + + - is_true: cluster_uuid + - is_true: master_node + - gte: { compressed_size_in_bytes: 50 } + - is_true: compressed_size diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.state/20_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.state/20_filtering.yml index 1e1d57125601c..880efaff19aa6 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.state/20_filtering.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.state/20_filtering.yml @@ -156,3 +156,19 @@ setup: - is_true: routing_table.indices.index1 - is_true: metadata.indices.index2 - is_true: routing_table.indices.index2 + +--- +"Filtering the cluster state returns cluster_uuid at the top level regardless of metric filters": + - skip: + version: " - 6.3.99" + reason: "cluster state including cluster_uuid at the top level is new in v6.4.0 and higher" + + - do: + cluster.state: + metric: [ master_node, version, metadata ] + + - is_true: cluster_uuid + - is_true: master_node + - is_true: version + - is_true: state_uuid + - is_true: metadata diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index 30c8df07ec1a5..2b991d1dc611a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -326,6 +326,9 @@ public String toString() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { EnumSet metrics = Metric.parseString(params.param("metric", "_all"), true); + // always provide the cluster_uuid as part of the top-level response (also part of the metadata response) + builder.field("cluster_uuid", metaData().clusterUUID()); + if (metrics.contains(Metric.VERSION)) { builder.field("version", version); builder.field("state_uuid", stateUUID); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java index 4ced505717a2e..7d671096514f4 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java @@ -69,6 +69,7 @@ public void testToXContent() throws IOException { assertEquals("{\n" + " \"acknowledged\" : true,\n" + " \"state\" : {\n" + + " \"cluster_uuid\" : \"_na_\",\n" + " \"version\" : 0,\n" + " \"state_uuid\" : \"" + clusterState.stateUUID() + "\",\n" + " \"master_node\" : \"node0\",\n" + @@ -136,6 +137,7 @@ public void testToXContent() throws IOException { assertEquals("{\n" + " \"acknowledged\" : true,\n" + " \"state\" : {\n" + + " \"cluster_uuid\" : \"_na_\",\n" + " \"version\" : 0,\n" + " \"state_uuid\" : \"" + clusterState.stateUUID() + "\",\n" + " \"master_node\" : \"node0\"\n" + @@ -168,6 +170,7 @@ public void testToXContent() throws IOException { assertEquals("{\n" + " \"acknowledged\" : true,\n" + " \"state\" : {\n" + + " \"cluster_uuid\" : \"_na_\",\n" + " \"metadata\" : {\n" + " \"cluster_uuid\" : \"_na_\",\n" + " \"templates\" : { },\n" + diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 2d027e8bfece5..a7fd6768064e9 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -1352,7 +1352,7 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, IndexReque * segment or if only one document is in a segment etc. This method prevents issues like this by randomizing the index * layout. * - * @param forceRefresh if true all involved indices are refreshed once the documents are indexed. Additionally if true + * @param forceRefresh if {@code true} all involved indices are refreshed once the documents are indexed. Additionally if {@code true} * some empty dummy documents are may be randomly inserted into the document list and deleted once all documents are indexed. * This is useful to produce deleted documents on the server side. * @param builders the documents to index. @@ -1369,8 +1369,8 @@ public void indexRandom(boolean forceRefresh, List builders * segment or if only one document is in a segment etc. This method prevents issues like this by randomizing the index * layout. * - * @param forceRefresh if true all involved indices are refreshed once the documents are indexed. - * @param dummyDocuments if true some empty dummy documents may be randomly inserted into the document list and deleted once + * @param forceRefresh if {@code true} all involved indices are refreshed once the documents are indexed. + * @param dummyDocuments if {@code true} some empty dummy documents may be randomly inserted into the document list and deleted once * all documents are indexed. This is useful to produce deleted documents on the server side. * @param builders the documents to index. */ @@ -1385,10 +1385,10 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, Listtrue all involved indices are refreshed once the documents are indexed. - * @param dummyDocuments if true some empty dummy documents may be randomly inserted into the document list and deleted once + * @param forceRefresh if {@code true} all involved indices are refreshed once the documents are indexed. + * @param dummyDocuments if {@code true} some empty dummy documents may be randomly inserted into the document list and deleted once * all documents are indexed. This is useful to produce deleted documents on the server side. - * @param maybeFlush if true this method may randomly execute full flushes after index operations. + * @param maybeFlush if {@code true} this method may randomly execute full flushes after index operations. * @param builders the documents to index. */ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean maybeFlush, List builders) throws InterruptedException, ExecutionException { @@ -1554,27 +1554,27 @@ public enum Scope { Scope scope() default Scope.SUITE; /** - * Returns the number of nodes in the cluster. Default is -1 which means + * Returns the number of nodes in the cluster. Default is {@code -1} which means * a random number of nodes is used, where the minimum and maximum number of nodes * are either the specified ones or the default ones if not specified. */ int numDataNodes() default -1; /** - * Returns the minimum number of data nodes in the cluster. Default is -1. + * Returns the minimum number of data nodes in the cluster. Default is {@code -1}. * Ignored when {@link ClusterScope#numDataNodes()} is set. */ int minNumDataNodes() default -1; /** - * Returns the maximum number of data nodes in the cluster. Default is -1. + * Returns the maximum number of data nodes in the cluster. Default is {@code -1}. * Ignored when {@link ClusterScope#numDataNodes()} is set. */ int maxNumDataNodes() default -1; /** - * Indicates whether the cluster can have dedicated master nodes. If false means data nodes will serve as master nodes - * and there will be no dedicated master (and data) nodes. Default is true which means + * Indicates whether the cluster can have dedicated master nodes. If {@code false} means data nodes will serve as master nodes + * and there will be no dedicated master (and data) nodes. Default is {@code false} which means * dedicated master nodes will be randomly used. */ boolean supportsDedicatedMasters() default true; @@ -1703,7 +1703,7 @@ private int getNumClientNodes() { } /** - * This method is used to obtain settings for the Nth node in the cluster. + * This method is used to obtain settings for the {@code N}th node in the cluster. * Nodes in this cluster are associated with an ordinal number such that nodes can * be started with specific configurations. This method might be called multiple * times with the same ordinal and is expected to return the same value for each invocation. @@ -1878,7 +1878,7 @@ public Collection> transportClientPlugins() { /** * Iff this returns true mock transport implementations are used for the test runs. Otherwise not mock transport impls are used. - * The default is true + * The default is {@code true}. */ protected boolean addMockTransportService() { return true; @@ -1886,7 +1886,7 @@ protected boolean addMockTransportService() { /** * Iff this returns true test zen discovery implementations is used for the test runs. - * The default is true + * The default is {@code true}. */ protected boolean addTestZenDiscovery() { return true; @@ -1957,7 +1957,7 @@ private static double transportClientRatio() { /** * Returns the transport client ratio from the class level annotation or via * {@link System#getProperty(String)} if available. If both are not available this will - * return a random ratio in the interval [0..1] + * return a random ratio in the interval {@code [0..1]}. */ protected double getPerTestTransportClientRatio() { final ClusterScope annotation = getAnnotation(this.getClass(), ClusterScope.class); diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 12acd21903ec4..5099fc0540de2 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -1978,7 +1978,7 @@ public Settings onNodeStopped(String nodeName) throws Exception { } /** - * Executed for each node before the n+1 node is restarted. The given client is + * Executed for each node before the {@code n + 1} node is restarted. The given client is * an active client to the node that will be restarted next. */ public void doAfterNodes(int n, Client client) throws Exception { diff --git a/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java b/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java index 15c650173bf87..724a99f2c9425 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java @@ -145,7 +145,7 @@ private static String differenceBetweenObjectsIgnoringArrayOrder(String path, Ob * If the xContent output contains objects that should be skipped of such treatment, an optional filtering * {@link Predicate} can be supplied that checks xContent paths that should be excluded from this treatment. * - * This predicate should check the xContent path that we want to insert to and return true if the + * This predicate should check the xContent path that we want to insert to and return {@code true} if the * path should be excluded. Paths are string concatenating field names and array indices, so e.g. in: * * 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java b/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java
index bf0b7376b8148..fc2a85b35a95b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java
@@ -19,14 +19,13 @@
 package org.elasticsearch.test.engine;
 
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.index.AssertingDirectoryReader;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.FilterDirectoryReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.AssertingIndexSearcher;
-import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.QueryCache;
 import org.apache.lucene.search.QueryCachingPolicy;
-import org.apache.lucene.search.ReferenceManager;
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.logging.Loggers;
@@ -38,6 +37,7 @@
 import org.elasticsearch.index.engine.EngineException;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.engine.MockInternalEngine;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -47,14 +47,15 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
- * Support class to build MockEngines like {@link org.elasticsearch.test.engine.MockInternalEngine}
+ * Support class to build MockEngines like {@link MockInternalEngine}
  * since they need to subclass the actual engine
  */
 public final class MockEngineSupport {
 
     /**
-     * Allows tests to wrap an index reader randomly with a given ratio. This is disabled by default ie. 0.0d since reader wrapping is insanely
-     * slow if {@link org.apache.lucene.index.AssertingDirectoryReader} is used.
+     * Allows tests to wrap an index reader randomly with a given ratio. This
+     * is disabled by default ie. {@code 0.0d} since reader wrapping is insanely
+     * slow if {@link AssertingDirectoryReader} is used.
      */
     public static final Setting WRAP_READER_RATIO =
         Setting.doubleSetting("index.engine.mock.random.wrap_reader_ratio", 0.0d, 0.0d, Property.IndexScope);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
index 1c31533c9337d..6654444066d52 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
@@ -474,7 +474,7 @@ public void clearRule() {
     /**
      * Adds a new delegate transport that is used for communication with the given transport service.
      *
-     * @return true iff no other delegate was registered for any of the addresses bound by transport service.
+     * @return {@code true} iff no other delegate was registered for any of the addresses bound by transport service.
      */
     public boolean addDelegate(TransportService transportService, DelegateTransport transport) {
         boolean noRegistered = true;
@@ -487,7 +487,7 @@ public boolean addDelegate(TransportService transportService, DelegateTransport
     /**
      * Adds a new delegate transport that is used for communication with the given transport address.
      *
-     * @return true iff no other delegate was registered for this address before.
+     * @return {@code true} iff no other delegate was registered for this address before.
      */
     public boolean addDelegate(TransportAddress transportAddress, DelegateTransport transport) {
         return transport().transports.put(transportAddress, transport) == null;
diff --git a/x-pack/docs/en/security/authentication/active-directory-realm.asciidoc b/x-pack/docs/en/security/authentication/active-directory-realm.asciidoc
index 2aaca6def915a..143156ca636a6 100644
--- a/x-pack/docs/en/security/authentication/active-directory-realm.asciidoc
+++ b/x-pack/docs/en/security/authentication/active-directory-realm.asciidoc
@@ -169,186 +169,14 @@ domain name from the NetBIOS name.
 ===== Load Balancing and Failover
 The `load_balance.type` setting can be used at the realm level to configure how
 {security} should interact with multiple Active Directory servers. Two modes of
-operation are supported: failover and load balancing
+operation are supported: failover and load balancing.
 
-.Load Balancing and Failover Types
-|=======================
-| Type              | | | Description
-| `failover`        | | | The URLs specified are used in the order that they are
-                          specified. The first server that can be connected to will
-                          be used for all subsequent connections. If a connection to
-                          that server fails then the next server that a connection
-                          can be established to will be used for subsequent connections.
-| `dns_failover`    | | | In this mode of operation, only a single URL may be specified.
-                          This URL must contain a DNS name. The system will be queried
-                          for all IP addresses that correspond to this DNS name.
-                          Connections to the Active Directory server will always be
-                          tried in the order in which they were retrieved. This differs
-                          from `failover` in that there is no reordering of the list
-                          and if a server has failed at the beginning of the list, it
-                          will still be tried for each subsequent connection.
-| `round_robin`     | | | Connections will continuously iterate through the list of
-                          provided URLs. If a server is unavailable, iterating through
-                          the list of URLs will continue until a successful connection
-                          is made.
-| `dns_round_robin` | | | In this mode of operation, only a single URL may be specified.
-                          This URL must contain a DNS name. The system will be queried
-                          for all IP addresses that correspond to this DNS name.
-                          Connections will continuously iterate through the list of
-                          addresses. If a server is unavailable, iterating through the
-                          list of URLs will continue until a successful connection is
-                          made.
-|=======================
+See {ref}/security-settings.html#load-balancing[Load Balancing and Failover Settings].
 
 [[ad-settings]]
 ===== Active Directory Realm Settings
 
-[cols="4,^3,10"]
-|=======================
-| Setting                    | Required | Description
-| `type`                     | yes      | Indicates the realm type. Must be set to `active_directory`.
-| `order`                    | no       | Indicates the priority of this realm within the realm chain.
-                                          Realms with a lower order are consulted first. Although not
-                                          required, we recommend explicitly setting this value when
-                                          you configure multiple realms. Defaults to `Integer.MAX_VALUE`.
-| `enabled`                  | no       | Indicates whether this realm is enabled or disabled. Enables
-                                          you to disable a realm without removing its configuration.
-                                          Defaults to `true`.
-| `domain_name`              | yes      | Specifies the domain name of the Active Directory. {security}
-                                          uses the domain name to derive the LDAP URL and `user_search_dn`
-                                          if those fields are not specified.
-| `url`                      | no/yes   | Specifies an LDAP URL of the form `ldap[s]://:`.
-                                          {security} attempts to authenticate against this URL. If the
-                                          URL is not specified, it is derived from the `domain_name`,
-                                          assuming an unencrypted connection to port 389. For example,
-                                          `ldap://:389`. This settings is required when
-                                          connecting using SSL/TLS or via a custom port.
-| `bind_dn`                  | no       | The DN of the user that is used to bind to Active Directory
-                                          and perform searches. Due to its potential security
-                                          impact, `bind_dn` is not exposed via the
-                                          {ref}/cluster-nodes-info.html#cluster-nodes-info[nodes info API].
-| `bind_password`            | no       | The password for the user that is used to bind to
-                                          Active Directory. Due to its potential security impact,
-                                          `bind_password` is not exposed via the
-                                          {ref}/cluster-nodes-info.html#cluster-nodes-info[nodes info API].
-                                          *Deprecated.* Use `secure_bind_password` instead. 
-| `secure_bind_password`     | no       | ({ref}/secure-settings.html[Secure])
-                                          The password for the user that is used to bind to Active Directory.
-| `load_balance.type`        | no       | The behavior to use when there are multiple LDAP URLs defined.
-                                          For supported values see <>.
-| `load_balance.cache_ttl`   | no       | When using `dns_failover` or `dns_round_robin` as the load
-                                          balancing type, this setting controls the amount of time to
-                                          cache DNS lookups. Defaults to `1h`.
-| `user_search.base_dn`      | no       | Specifies the context to search for the user. Defaults to the
-                                          root of the Active Directory domain.
-| `user_search.scope`        | no       | Specifies whether the user search should be `sub_tree` (default),
-                                          `one_level`, or `base`. `sub_tree` searches all objects contained
-                                          under `base_dn`. `one_level` only searches users directly
-                                          contained within the `base_dn`. `base` specifies that the
-                                          `base_dn` is a user object and that it is the only user considered.
-| `user_search.filter`       | no       | Specifies a filter to use to lookup a user given a username.
-                                          The default filter looks up `user` objects with either
-                                          `sAMAccountName` or `userPrincipalName`. If specified, this
-                                          must be a valid LDAP user search filter, for example
-                                          `(&(objectClass=user)(sAMAccountName={0}))`. For more
-                                          information, see https://msdn.microsoft.com/en-us/library/aa746475(v=vs.85).aspx[Search Filter Syntax].
-| `user_search.upn_filter`   | no       | Specifies a filter to use to lookup a user given a user principal name.
-                                          The default filter looks up `user` objects with
-                                          a matching `userPrincipalName`. If specified, this
-                                          must be a valid LDAP user search filter, for example
-                                          `(&(objectClass=user)(userPrincipalName={1}))`. `{1}` is
-                                          the full user principal name provided by the user. For more
-                                          information, see https://msdn.microsoft.com/en-us/library/aa746475(v=vs.85).aspx[Search Filter Syntax].
-| `user_search.down_level_filter` | no  | Specifies a filter to use to lookup a user given a down level logon name (DOMAIN\user).
-                                          The default filter looks up `user` objects with a matching
-                                          `sAMAccountName` in the domain provided. If specified, this
-                                          must be a valid LDAP user search filter, for example
-                                          `(&(objectClass=user)(sAMAccountName={0}))`. For more
-                                          information, see https://msdn.microsoft.com/en-us/library/aa746475(v=vs.85).aspx[Search Filter Syntax].
-| `user_search.pool.enabled`      | no  | Enables or disables connection pooling for user search. When
-                                          disabled a new connection is created for every search. The
-                                          default is `true` when `bind_dn` is provided.
-| `user_search.pool.size`         | no  | Specifies the maximum number of connections to Active Directory
-                                          server to allow in the connection pool. Defaults to `20`.
-| `user_search.pool.initial_size` | no  | The initial number of connections to create to Active Directory
-                                          server on startup. Defaults to `0`. Values greater than `0`
-                                          could cause startup failures if the LDAP server is down.
-| `user_search.pool.health_check.enabled` | no | Enables or disables a health check on Active Directory connections in
-                                                 the connection pool. Connections are checked in the
-                                                 background at the specified interval. Defaults to `true`.
-| `user_search.pool.health_check.dn`      | no | Specifies the distinguished name to retrieve as part of
-                                                 the health check. Defaults to the value of `bind_dn` if present, and if
-                                                 not falls back to `user_search.base_dn`.
-| `user_search.pool.health_check.interval` | no | How often to perform background checks of connections in
-                                                  the pool. Defaults to `60s`.
-| `group_search.base_dn`     | no       | Specifies the context to search for groups in which the user
-                                          has membership. Defaults to the root of the Active Directory
-                                          domain.
-| `group_search.scope`       | no       | Specifies whether the group search should be `sub_tree` (default),
-                                          `one_level` or `base`.  `sub_tree` searches all objects contained
-                                          under `base_dn`. `one_level` searches for groups directly
-                                          contained within the `base_dn`. `base` specifies that the
-                                          `base_dn` is a group object and that it is the only group considered.
-| `unmapped_groups_as_roles` | no       | Specifies whether the names of any unmapped Active Directory
-                                          groups should be used as role names and assigned to the user.
-                                          A group is considered to be _unmapped_ if it is not referenced
-                                          in any <> (API based
-                                          role-mappings are not considered).
-                                          Defaults to `false`.
-| `files.role_mapping`       | no       | Specifies the path and file name of the
-                                          <>.
-                                          Defaults to `ES_PATH_CONF/x-pack/role_mapping.yml`,
-                                          where `ES_PATH_CONF` is `ES_HOME/config` (zip/tar installations)
-                                          or `/etc/elasticsearch` (package installations).
-| `follow_referrals`         | no       | Specifies whether {security} should follow referrals returned
-                                          by the Active Directory server. Referrals are URLs returned by
-                                          the server that are to be used to continue the LDAP operation
-                                          (such as `search`). Defaults to `true`.
-| `metadata`                 | no       | Specifies the list of additional LDAP attributes that should
-                                          be stored in the `metadata` of an authenticated user.
-| `ssl.key`                  | no       | Specifies the path to the PEM encoded private key to use if the Active Directory
-                                          server requires client authentication. `ssl.key` and `ssl.keystore.path` may not be used at the
-                                          same time.
-| `ssl.key_passphrase`       | no       | Specifies the passphrase to decrypt the PEM encoded private key if it is encrypted.
-| `ssl.certificate`          | no       | Specifies the path to the PEM encoded certificate (or certificate chain) that goes with the key
-                                          if the Active Directory server requires client authentication.
-| `ssl.certificate_authorities`| no     | Specifies the paths to the PEM encoded certificate authority certificates that
-                                          should be trusted. `ssl.certificate_authorities` and `ssl.truststore.path` may not be used at
-                                          the same time.
-| `ssl.keystore.path`        | no       | The path to the Java Keystore file that contains a private key and certificate. `ssl.key` and
-                                          `ssl.keystore.path` may not be used at the same time.
-| `ssl.keystore.password`    | no       | The password to the keystore.
-| `ssl.keystore.key_password`| no       | The password for the key in the keystore. Defaults to the keystore password.
-| `ssl.truststore.path`      | no       | The path to the Java Keystore file that contains the certificates to trust.
-                                          `ssl.certificate_authorities` and `ssl.truststore.path` may not be used at the same time.
-| `ssl.truststore.password`  | no       | The password to the truststore.
-| `ssl.verification_mode`    | no       | Specifies the type of verification to be performed when
-                                          connecting to an Active Directory server using `ldaps`. When
-                                          set to `full`, the hostname or IP address used in the `url`
-                                          must match one of the names in the certificate or the
-                                          connection will not be allowed. Due to their potential security impact,
-                                          `ssl` settings are not exposed via the
-                                          {ref}/cluster-nodes-info.html#cluster-nodes-info[nodes info API].
-+
-                                          Values are `none`, `certificate`, and `full`. Defaults to `full`.
-+
-                                          See {ref}/security-settings.html#ssl-tls-settings[`xpack.ssl.verification_mode`]
-                                          for an explanation of these values.
-| `ssl.supported_protocols`  | no       | Specifies the supported protocols for TLS/SSL.
-| `ssl.cipher_suites`        | no       | Specifies the cipher suites that should be supported when communicating
-                                          with the Active Directory server.
-| `cache.ttl`                | no       | Specifies the time-to-live for cached user entries. A user's
-                                          credentials are cached for this period of time. Specify the
-                                          time period using the standard Elasticsearch
-                                          {ref}/common-options.html#time-units[time units].
-                                          Defaults to `20m`.
-| `cache.max_users`          | no       | Specifies the maximum number of user entries that can be
-                                          stored in the cache at one time. Defaults to 100,000.
-| `cache.hash_algo`          | no       | Specifies the hashing algorithm that is used for the
-                                          cached user credentials.
-                                          See <> for the
-                                          possible values. (Expert Setting).
-|=======================
+See {ref}/security-settings.html#ref-ad-settings[Active Directory Realm Settings].
 
 [[mapping-roles-ad]]
 ==== Mapping Active Directory Users and Groups to Roles
diff --git a/x-pack/docs/en/security/authentication/ldap-realm.asciidoc b/x-pack/docs/en/security/authentication/ldap-realm.asciidoc
index bd32c49622877..15b014183aa46 100644
--- a/x-pack/docs/en/security/authentication/ldap-realm.asciidoc
+++ b/x-pack/docs/en/security/authentication/ldap-realm.asciidoc
@@ -137,211 +137,13 @@ The `load_balance.type` setting can be used at the realm level to configure how
 {security} should interact with multiple LDAP servers. {security} supports both
 failover and load balancing modes of operation.
 
-.Load Balancing and Failover Types
-|=======================
-| Type              | | | Description
-| `failover`        | | | The URLs specified are used in the order that they are specified.
-                          The first server that can be connected to will be used for all
-                          subsequent connections. If a connection to that server fails then
-                          the next server that a connection can be established to will be
-                          used for subsequent connections.
-| `dns_failover`    | | | In this mode of operation, only a single URL may be specified.
-                          This URL must contain a DNS name. The system will be queried for
-                          all IP addresses that correspond to this DNS name. Connections to
-                          the LDAP server will always be tried in the order in which they
-                          were retrieved. This differs from `failover` in that there is no
-                          reordering of the list and if a server has failed at the beginning
-                          of the list, it will still be tried for each subsequent connection.
-| `round_robin`     | | | Connections will continuously iterate through the list of provided
-                          URLs. If a server is unavailable, iterating through the list of
-                          URLs will continue until a successful connection is made.
-| `dns_round_robin` | | | In this mode of operation, only a single URL may be specified. This
-                          URL must contain a DNS name. The system will be queried for all IP
-                          addresses that correspond to this DNS name. Connections will
-                          continuously iterate through the list of addresses. If a server is
-                          unavailable, iterating through the list of URLs will continue until
-                          a successful connection is made.
-|=======================
+See {ref}/security-settings.html#load-balancing[Load Balancing and Failover Settings].
 
 
 [[ldap-settings]]
 ===== LDAP Realm Settings
 
-.Common LDAP Realm Settings
-[cols="4,^3,10"]
-|=======================
-| Setting                        | Required | Description
-| `type`                         | yes      | Indicates the realm type. Must be set to `ldap`.
-| `order`                        | no       | Indicates the priority of this realm within the realm
-                                              chain. Realms with a lower order are consulted first.
-                                              Although not required, we recommend explicitly
-                                              setting this value when you configure multiple realms.
-                                              Defaults to `Integer.MAX_VALUE`.
-| `enabled`                      | no       | Indicates whether this realm is enabled or disabled.
-                                              Enables you to disable a realm without removing its
-                                              configuration. Defaults to `true`.
-| `url`                          | yes      | Specifies one or more LDAP URLs of the form of
-                                              `ldap[s]://:`. Multiple URLs can be
-                                              defined using a comma separated value or array syntax:
-                                              `[ "ldaps://server1:636", "ldaps://server2:636" ]`.
-                                              `ldaps` and `ldap` URL protocols cannot be mixed in
-                                              the same realm.
-| `load_balance.type`            | no       | The behavior to use when there are multiple LDAP URLs
-                                              defined. For supported values see
-                                              <>.
-| `load_balance.cache_ttl`       | no       | When using `dns_failover` or `dns_round_robin` as the
-                                              load balancing type, this setting controls the amount of time
-                                              to cache DNS lookups. Defaults to `1h`.
-| `user_group_attribute`         | no       | Specifies the attribute to examine on the user for group
-                                              membership. The default is `memberOf`. This setting will
-                                              be ignored if any `group_search` settings are specified.
-| `group_search.base_dn`         | no       | Specifies a container DN to search for groups in which
-                                              the user has membership. When this element is absent,
-                                              Security searches for the attribute specified by
-                                              `user_group_attribute` set on the user to determine
-                                              group membership.
-| `group_search.scope`           | no       | Specifies whether the group search should be
-                                              `sub_tree`, `one_level` or `base`.  `one_level` only
-                                              searches objects directly contained within the
-                                              `base_dn`. The default `sub_tree` searches all objects
-                                              contained under `base_dn`. `base` specifies that the
-                                              `base_dn` is a group object, and that it is the only
-                                              group considered.
-| `group_search.filter`          | no       | Specifies a filter to use to lookup a group. If not
-                                              set, the realm searches for `group`,
-                                              `groupOfNames`, `groupOfUniqueNames`, or `posixGroup` with the
-                                              attributes `member`, `memberOf`, or `memberUid`. Any instance of
-                                              `{0}` in the filter is replaced by the user
-                                              attribute defined in `group_search.user_attribute`
-| `group_search.user_attribute`  | no       | Specifies the user attribute that is fetched and
-                                              provided as a parameter to the filter.  If not set,
-                                              the user DN is passed to the filter.
-| `unmapped_groups_as_roles`     | no       | Specifies whether the names of any unmapped LDAP groups
-                                              should be used as role names and assigned to the user.
-                                              A group is considered to be _unmapped_ if it is not referenced
-                                              in any <> (API based
-                                              role-mappings are not considered).
-                                              Defaults to `false`.
-| `timeout.tcp_connect`          | no       | Specifies the TCP connect timeout period for establishing an
-                                              LDAP connection. An `s` at the end indicates seconds, or `ms`
-                                              indicates milliseconds. Defaults to `5s` (5 seconds).
-| `timeout.tcp_read`             | no       | Specifies the TCP read timeout period after establishing an LDAP connection.
-                                              An `s` at the end indicates seconds, or `ms` indicates milliseconds.
-                                              Defaults to `5s` (5 seconds).
-| `timeout.ldap_search`          | no       | Specifies the LDAP Server enforced timeout period for an LDAP search.
-                                              An `s` at the end indicates seconds, or `ms` indicates milliseconds.
-                                              Defaults to `5s` (5 seconds).
-| `files.role_mapping`           | no       | Specifies the path and file name for the
-                                              <>.
-                                              Defaults to `ES_HOME/config/x-pack/role_mapping.yml`.
-| `follow_referrals`             | no       | Specifies whether {security} should follow referrals
-                                              returned by the LDAP server. Referrals are URLs returned by
-                                              the server that are to be used to continue the LDAP operation
-                                              (e.g. search). Defaults to `true`.
-| `metadata`                     | no       | Specifies the list of additional LDAP attributes that should
-                                              be stored in the `metadata` of an authenticated user.
-| `ssl.key`                      | no       | Specifies the path to the PEM encoded private key to use if the LDAP
-                                              server requires client authentication. `ssl.key` and `ssl.keystore.path`
-                                              may not be used at the same time.
-| `ssl.key_passphrase`           | no       | Specifies the passphrase to decrypt the PEM encoded private key if it is encrypted.
-| `ssl.certificate`              | no       | Specifies the path to the PEM encoded certificate (or certificate chain) that goes with the
-                                              key if the LDAP server requires client authentication.
-| `ssl.certificate_authorities`  | no       | Specifies the paths to the PEM encoded certificate authority certificates that
-                                              should be trusted. `ssl.certificate_authorities` and `ssl.truststore.path` may not be used
-                                              at the same time.
-| `ssl.keystore.path`            | no       | The path to the Java Keystore file that contains a private key and certificate. `ssl.key` and
-                                              `ssl.keystore.path` may not be used at the same time.
-| `ssl.keystore.password`        | no       | The password to the keystore.
-| `ssl.keystore.key_password`    | no       | The password for the key in the keystore. Defaults to the keystore password.
-| `ssl.truststore.path`          | no       | The path to the Java Keystore file that contains the certificates to trust.
-                                              `ssl.certificate_authorities` and `ssl.truststore.path` may not be used at the same time.
-| `ssl.truststore.password`      | no       | The password to the truststore.
-| `ssl.verification_mode`        | no       | Specifies the type of verification to be performed when
-                                              connecting to a LDAP server using `ldaps`. When
-                                              set to `full`, the hostname or IP address used in the `url`
-                                              must match one of the names in the certificate or the
-                                              connection will not be allowed. Due to their potential security impact,
-                                              `ssl` settings are not exposed via the
-                                              {ref}/cluster-nodes-info.html#cluster-nodes-info[nodes info API].
-                                              Values are `none`, `certificate`, and `full`. Defaults to `full`.
-                                              See {ref}/security-settings.html#ssl-tls-settings[`xpack.ssl.verification_mode`]
-                                              for an explanation of these values.
-| `ssl.supported_protocols`      | no       | Specifies the supported protocols for SSL/TLS.
-| `ssl.cipher_suites`            | no       | Specifies the cipher suites that should be supported when communicating
-                                              with the LDAP server.
-| `cache.ttl`                | no           | Specifies the time-to-live for cached user entries. A
-                                              user's credentials are cached for this period of time.
-                                              Specify the time period using the standard Elasticsearch
-                                              {ref}/common-options.html#time-units[time units].
-                                              Defaults to `20m`.
-| `cache.max_users`          | no           | Specifies the maximum number of user entries that can be
-                                              stored in the cache at one time. Defaults to 100,000.
-| `cache.hash_algo`          | no           | Specifies the hashing algorithm that is used for the
-                                              cached user credentials. See
-                                              <> for the possible
-                                              values. (Expert Setting).
-|=======================
-
-.User Search Mode Settings
-|=======================
-| Setting                                  | Required | Description
-| `bind_dn`                                | no       | The DN of the user that is used to bind to the LDAP
-                                                        and perform searches. If not specified, an anonymous
-                                                        bind is attempted. Due to its potential security
-                                                        impact, `bind_dn` is not exposed via the
-                                                        {ref}/cluster-nodes-info.html#cluster-nodes-info[nodes info API].
-| `bind_password`                          | no       | The password for the user that is used to bind to the
-                                                        LDAP directory. Due to its potential security impact,
-                                                        `bind_password` is not exposed via the
-                                                        {ref}/cluster-nodes-info.html#cluster-nodes-info[nodes info API].
-                                                        *Deprecated.* Use `secure_bind_password` instead. 
-| `secure_bind_password`                   | no       | ({ref}/secure-settings.html[Secure])
-                                                        The password for the user that is used to bind to LDAP directory.
-| `user_search.base_dn`                    | yes      | Specifies a container DN to search for users.
-| `user_search.scope`                      | no       | The scope of the user search. Valid values are `sub_tree`,
-                                                        `one_level` or `base`. `one_level` only searches objects
-                                                        directly contained within the `base_dn`. `sub_tree` searches
-                                                        all objects contained under `base_dn`. `base` specifies
-                                                        that the `base_dn` is the user object, and that it is the
-                                                        only user considered. Defaults to `sub_tree`.
-| `user_search.filter`                     | no       | Specifies the filter used to search the directory in attempt to match
-                                                        an entry with the username provided by the user. Defaults to `(uid={0})`.
-                                                        `{0}` is substituted with the username provided when searching.
-| `user_search.attribute`                  | no       | This setting is deprecated; use `user_search.filter` instead.
-                                                        Specifies the attribute to match with the username presented
-                                                        to. Defaults to `uid`.
-| `user_search.pool.enabled`               | no       | Enables or disables connection pooling for user search. When
-                                                        disabled a new connection is created for every search. The
-                                                        default is `true`.
-| `user_search.pool.size`                  | no       | Specifies the maximum number of connections to the LDAP
-                                                        server to allow in the connection pool. Defaults to `20`.
-| `user_search.pool.initial_size`          | no       | The initial number of connections to create to the LDAP
-                                                        server on startup. Defaults to `0`. Values greater than `0`
-                                                        could cause startup failures if the LDAP server is down.
-| `user_search.pool.health_check.enabled`  | no       | Enables or disables a health check on LDAP connections in
-                                                        the connection pool. Connections are checked in the
-                                                        background at the specified interval. Defaults to `true`.
-| `user_search.pool.health_check.dn`       | no/yes   | Specifies the distinguished name to retrieve as part of
-                                                        the health check. Defaults to the value of `bind_dn`.
-                                                        This setting is required when `bind_dn` is not configured.
-| `user_search.pool.health_check.interval` | no       | How often to perform background checks of connections in
-                                                        the pool. Defaults to `60s`.
-|=======================
-
-.User Templates Mode Settings
-[cols="4,^3,10"]
-|=======================
-| Setting               | Required  | Description
-| `user_dn_templates`   | yes       | Specifies the DN template that replaces the
-                                      user name with the string `{0}`. This element
-                                      is multivalued, allowing for multiple user
-                                      contexts.
-|=======================
-
-
-NOTE:   If any settings starting with `user_search` are specified, the
-        `user_dn_templates` the settings are ignored.
-
+See {ref}/security-settings.html#ref-ldap-settings[LDAP Realm Settings].
 
 [[mapping-roles-ldap]]
 ==== Mapping LDAP Groups to Roles
diff --git a/x-pack/docs/en/settings/security-settings.asciidoc b/x-pack/docs/en/settings/security-settings.asciidoc
index 046d76784fbde..139e54467b7a9 100644
--- a/x-pack/docs/en/settings/security-settings.asciidoc
+++ b/x-pack/docs/en/settings/security-settings.asciidoc
@@ -150,9 +150,9 @@ For a native realm, the `type` must be set to `native`. In addition to the
 <>, you can specify  
 the following optional settings: 
 
-`cache.ttl`:: The time-to-live for cached user entries. User credentials are 
-cached for this period of time. Specify the time period using the standard 
-{es} <>. Defaults to `20m`.
+`cache.ttl`:: The time-to-live for cached user entries. A user and a hash of its 
+credentials are cached for this period of time. Specify the time period using 
+the standard {es} <>. Defaults to `20m`.
 
 `cache.max_users`:: The maximum number of user entries that can live in the 
 cache at any given time. Defaults to 100,000.
@@ -169,9 +169,9 @@ in-memory cached user credentials. For possible values, see
 ===== File realm settings
 
 `cache.ttl`::
-The time-to-live for cached user entries--user credentials are cached for
-this configured period of time. Defaults to `20m`. Specify values using the
-standard Elasticsearch {ref}/common-options.html#time-units[time units].
+The time-to-live for cached user entries. A user and a hash of its credentials 
+are cached for this configured period of time. Defaults to `20m`. Specify values 
+using the standard {es} {ref}/common-options.html#time-units[time units].
 Defaults to `20m`.
 
 `cache.max_users`::
@@ -186,12 +186,18 @@ all possible values. Defaults to `ssha256`.
 [[ref-ldap-settings]]
 [float]
 ===== LDAP realm settings
-`url`::
-An LDAP URL in the format `ldap[s]://:`. Required.
+
+The `type` setting must be set to `ldap`. In addition to the 
+<>, you can specify the following settings: 
+
+`url`:: Specifies one or more LDAP URLs in the format  
+`ldap[s]://:`. Multiple URLs can be defined using a comma 
+separated value or array syntax: `[ "ldaps://server1:636", "ldaps://server2:636" ]`. 
+`ldaps` and `ldap` URL protocols cannot be mixed in the same realm. Required.
 
 `load_balance.type`::
 The behavior to use when there are multiple LDAP URLs defined. For supported
-values see {xpack-ref}/ldap-realm.html#ldap-load-balancing[LDAP load balancing and failover types].
+values see <>.
 Defaults to `failover`.
 
 `load_balance.cache_ttl`::
@@ -200,36 +206,45 @@ this setting controls the amount of time to cache DNS lookups. Defaults
 to `1h`.
 
 `bind_dn`::
-The DN of the user that will be used to bind to the LDAP and perform searches.
-Only applicable in {xpack-ref}/ldap-realm.html#ldap-user-search[user search mode].
-If this is not specified, an anonymous bind will be attempted.
-Defaults to Empty.
+The DN of the user that is used to bind to the LDAP and perform searches.
+Only applicable in user search mode.
+If not specified, an anonymous bind is attempted.
+Defaults to Empty. Due to its potential security impact, `bind_dn` is not 
+exposed via the <>.
 
 `bind_password`::
-The password for the user that will be used to bind to the LDAP directory.
-Defaults to Empty.
-*Deprecated.* Use `secure_bind_password` instead.
+deprecated[6.3] Use `secure_bind_password` instead. The password for the user 
+that is used to bind to the LDAP directory.
+Defaults to Empty. Due to its potential security impact, `bind_password` is not 
+exposed via the <>.
+
 
 `secure_bind_password` (<>)::
-The password for the user that will be used to bind to the LDAP directory.
+The password for the user that is used to bind to the LDAP directory.
 Defaults to Empty.
 
 `user_dn_templates`::
 The DN template that replaces the user name with the string `{0}`.
-This element is multivalued; you can specify multiple user contexts.
-Required to operate in user template mode. Not valid
-if `user_search.base_dn` is specified. For more information on
+This setting is multivalued; you can specify multiple user contexts.
+Required to operate in user template mode. If `user_search.base_dn` is specified, 
+this setting is not valid. For more information on
 the different modes, see {xpack-ref}/ldap-realm.html[LDAP realms].
++
+--
+NOTE: If any settings starting with `user_search` are specified, the 
+`user_dn_templates` settings are ignored.
+
+--
 
 `user_group_attribute`::
 Specifies the attribute to examine on the user for group membership.
-The default is `memberOf`. This setting will be ignored if any
-`group_search` settings are specified. Defaults to  `memberOf`.
+If any `group_search` settings are specified, this setting is ignored. Defaults 
+to `memberOf`.
 
 `user_search.base_dn`::
 Specifies a container DN to search for users. Required
-to operated in user search mode. Not valid if
-`user_dn_templates is specified. For more information on
+to operated in user search mode. If `user_dn_templates` is specified, this 
+setting is not valid. For more information on
 the different modes, see {xpack-ref}/ldap-realm.html[LDAP realms].
 
 `user_search.scope`::
@@ -240,18 +255,18 @@ The scope of the user search. Valid values are `sub_tree`, `one_level` or
 the only user considered. Defaults to  `sub_tree`.
 
 `user_search.filter`::
-Specifies the filter used to search the directory in attempt to match
+Specifies the filter used to search the directory in attempts to match
 an entry with the username provided by the user. Defaults to `(uid={0})`.
 `{0}` is substituted with the username provided when searching.
 
 `user_search.attribute`::
-This setting is deprecated; use `user_search.filter` instead.
-The attribute to match with the username presented to. Defaults to `uid`.
+deprecated[5.6] Use `user_search.filter` instead.
+The attribute to match with the username sent with the request. Defaults to `uid`.
 
 `user_search.pool.enabled`::
-Enables or disables connection pooling for user search. When
-disabled a new connection is created for every search. The
-default is `true` when `bind_dn` is provided.
+Enables or disables connection pooling for user search. If set to `false`, a new 
+connection is created for every search. The
+default is `true` when `bind_dn` is set.
 
 `user_search.pool.size`::
 The maximum number of connections to the LDAP server to allow in the
@@ -259,17 +274,18 @@ connection pool. Defaults to `20`.
 
 `user_search.pool.initial_size`::
 The initial number of connections to create to the LDAP server on startup.
-Defaults to `0`.
+Defaults to `0`. If the LDAP server is down, values greater than `0` could cause 
+startup failures.
 
 `user_search.pool.health_check.enabled`::
-Flag to enable or disable a health check on LDAP connections in the connection
+Enables or disables a health check on LDAP connections in the connection
 pool. Connections are checked in the background at the specified interval.
 Defaults to `true`.
 
 `user_search.pool.health_check.dn`::
-The distinguished name to be retrieved as part of the health check.
-Defaults to the value of `bind_dn` if present, and if
-not falls back to `user_search.base_dn`.
+The distinguished name that is retrieved as part of the health check.
+Defaults to the value of `bind_dn` if present; if
+not, falls back to `user_search.base_dn`.
 
 `user_search.pool.health_check.interval`::
 The interval to perform background checks of connections in the pool.
@@ -277,7 +293,7 @@ Defaults to `60s`.
 
 `group_search.base_dn`::
 The container DN to search for groups in which the user has membership. When
-this element is absent, Security searches for the attribute specified by
+this element is absent, {security} searches for the attribute specified by
 `user_group_attribute` set on the user in order to determine group membership.
 
 `group_search.scope`::
@@ -287,30 +303,33 @@ Specifies whether the group search should be `sub_tree`, `one_level` or
 `base` specifies that the `base_dn` is a group object, and that it is the
 only group considered. Defaults to  `sub_tree`.
 
-`group_search.filter`::
+`group_search.filter`:: 
+Specifies a filter to use to look up a group. 
 When not set, the realm searches for `group`, `groupOfNames`, `groupOfUniqueNames`,
 or `posixGroup` with the attributes `member`, `memberOf`, or `memberUid`.  Any
 instance of `{0}` in the filter is replaced by the user attribute defined in
 `group_search.user_attribute`.
 
 `group_search.user_attribute`::
-Specifies the user attribute that will be fetched and provided as a parameter to
+Specifies the user attribute that is fetched and provided as a parameter to
 the filter.  If not set, the user DN is passed into the filter. Defaults to Empty.
 
 `unmapped_groups_as_roles`::
-Takes a boolean variable. When this element is set to `true`, the names of any
-LDAP groups that are not referenced in a role-mapping _file_ are used as role
-names and assigned to the user. Defaults to `false`.
+If set to `true`, the names of any unmapped LDAP groups are used as role names 
+and assigned to the user. A group is considered to be _unmapped_ if it is not 
+not referenced in a 
+{xpack-ref}/mapping-roles.html#mapping-roles-file[role-mapping file]. API-based 
+role mappings are not considered. Defaults to `false`.
 
 `files.role_mapping`::
 The {xpack-ref}/security-files.html[location] for the {xpack-ref}/mapping-roles.html#mapping-roles[
 YAML role mapping configuration file]. Defaults to
-`CONFIG_DIR/x-pack/role_mapping.yml`.
+`CONFIG_DIR/role_mapping.yml`.
 
 `follow_referrals`::
-Boolean value that specifies whether Securityshould follow referrals returned
+Specifies whether {security} should follow referrals returned
 by the LDAP server. Referrals are URLs returned by the server that are to be
-used to continue the LDAP operation (e.g. search). Defaults to `true`.
+used to continue the LDAP operation (for example, search). Defaults to `true`.
 
 `metadata`::
 A list of additional LDAP attributes that should be loaded from the
@@ -332,7 +351,9 @@ An `s` at the end indicates seconds, or `ms` indicates milliseconds.
 Defaults to `5s` (5 seconds ).
 
 `ssl.key`::
-Path to a PEM encoded file containing the private key.
+Path to a PEM encoded file containing the private key, which is used if the 
+LDAP server requires client authentication. `ssl.key` and `ssl.keystore.path` 
+cannot be used at the same time.
 
 `ssl.key_passphrase`::
 The passphrase that is used to decrypt the private key. This value is
@@ -346,7 +367,9 @@ Path to a PEM encoded file containing the certificate (or certificate chain)
 that will be presented to clients when they connect.
 
 `ssl.certificate_authorities`::
-List of paths to PEM encoded certificate files that should be trusted.
+List of paths to PEM encoded certificate files that should be trusted. 
+`ssl.certificate_authorities` and `ssl.truststore.path` cannot be used at the 
+same time.
 
 `ssl.keystore.path`::
 The path to the Java Keystore file that contains a private key and certificate.
@@ -370,7 +393,7 @@ The password for the key in the keystore. Defaults to the keystore password.
 
 `ssl.truststore.path`::
 The path to the Java Keystore file that contains the certificates to trust.
-`ssl.certificate_authorities` and `ssl.truststore.path` may not be used at the same time.
+`ssl.certificate_authorities` and `ssl.truststore.path` cannot be used at the same time.
 
 `ssl.truststore.password`::
 The password to the truststore.
@@ -391,18 +414,19 @@ See <> for an explanation of
 these values.
 
 `ssl.supported_protocols`::
-Supported protocols with versions. Defaults to the value of
+Supported protocols for TLS/SSL (with versions). Defaults to the value of
 `xpack.ssl.supported_protocols`.
 
-`ssl.cipher_suites`
+`ssl.cipher_suites`:: Specifies the cipher suites that should be supported when 
+communicating with the LDAP server. 
 Supported cipher suites can be found in Oracle's http://docs.oracle.com/javase/8/docs/technotes/guides/security/SunProviders.html[
 Java Cryptography Architecture documentation]. Defaults to the value of
 `xpack.ssl.cipher_suites`.
 
 `cache.ttl`::
-Specifies the time-to-live for cached user entries (a user and its credentials
-are cached for this period of time). Use the standard Elasticsearch
-{ref}/common-options.html#time-units[time units]). Defaults to  `20m`.
+Specifies the time-to-live for cached user entries. A user and a hash of its 
+credentials are cached for this period of time. Use the standard {es}
+<>. Defaults to  `20m`.
 
 `cache.max_users`::
 Specifies the maximum number of user entries that the cache can contain.
@@ -410,20 +434,28 @@ Defaults to `100000`.
 
 `cache.hash_algo`::
 (Expert Setting) Specifies the hashing algorithm that is used for the
-in-memory cached user credentials (see {xpack-ref}/controlling-user-cache.html#controlling-user-cache[Cache hash algorithms]
-table for all possible values). Defaults to `ssha256`.
+in-memory cached user credentials. See {xpack-ref}/controlling-user-cache.html#controlling-user-cache[Cache hash algorithms]
+table for all possible values. Defaults to `ssha256`.
 
 [[ref-ad-settings]]
 [float]
 ===== Active Directory realm settings
 
+The `type` setting must be set to `active_directory`. In addition to the 
+<>, you can specify  
+the following settings: 
+
 `url`::
-A URL in the format `ldap[s]://:`. Defaults to `ldap://:389`.
+An LDAP URL of the form `ldap[s]://:`. {security} attempts to 
+authenticate against this URL. If the URL is not specified, it is derived from 
+the `domain_name` setting and assumes an unencrypted connection to port 389. 
+Defaults to `ldap://:389`. This setting is required when connecting 
+using SSL/TLS or when using a custom port.
 
 `load_balance.type`::
 The behavior to use when there are multiple LDAP URLs defined. For supported
-values see {xpack-ref}/active-directory-realm.html#ad-load-balancing[load balancing and failover types].
-Defaults to  `failover`.
+values see <>.
+Defaults to `failover`.
 
 `load_balance.cache_ttl`::
 When using `dns_failover` or `dns_round_robin` as the load balancing type,
@@ -431,31 +463,34 @@ this setting controls the amount of time to cache DNS lookups. Defaults
 to `1h`.
 
 `domain_name`::
-The domain name of Active Directory. The cluster can derive the URL and
-`user_search_dn` fields from values in this element if those fields are not
-otherwise specified. Required.
+The domain name of Active Directory. If the the `url` and `user_search_dn` 
+settings are not specified, the cluster can derive those values from this 
+setting. Required.
 
 `bind_dn`::
-The DN of the user that will be used to bind to Active Directory and perform searches.
-Defaults to Empty.
+The DN of the user that is used to bind to Active Directory and perform searches.
+Defaults to Empty. Due to its potential security impact, `bind_dn` is not 
+exposed via the <>.
 
 `bind_password`::
-The password for the user that will be used to bind to Active Directory.
-Defaults to Empty.
-*Deprecated.* Use `secure_bind_password` instead.
+deprecated[6.3] Use `secure_bind_password` instead. The password for the user 
+that is used to bind to Active Directory. Defaults to Empty. Due to its 
+potential security impact, `bind_password` is not exposed via the 
+<>.
 
 `secure_bind_password` (<>)::
-The password for the user that will be used to bind to Active Directory.
+The password for the user that is used to bind to Active Directory.
 Defaults to Empty.
 
 `unmapped_groups_as_roles`::
-Takes a boolean variable. When this element is set to `true`, the names of any
-LDAP groups that are not referenced in a role-mapping _file_ are used as role
-names and assigned to the user. Defaults to `false`.
+If set to `true`, the names of any unmapped Active Directory groups are used as 
+role names and assigned to the user. A group is considered _unmapped_ when it 
+is not referenced in any role-mapping files. API-based role mappings are not 
+considered. Defaults to `false`.
 
 `files.role_mapping`::
 The {xpack-ref}/security-files.html[location] for the YAML
-role mapping configuration file. Defaults to  `CONFIG_DIR/x-pack/role_mapping.yml`.
+role mapping configuration file. Defaults to `CONFIG_DIR/role_mapping.yml`.
 
 `user_search.base_dn`::
 The context to search for a user. Defaults to the root
@@ -471,22 +506,27 @@ only user considered. Defaults to `sub_tree`.
 `user_search.filter`::
 Specifies a filter to use to lookup a user given a username.  The default
 filter looks up `user` objects with either `sAMAccountName` or
-`userPrincipalName`.
+`userPrincipalName`. If specified, this must be a valid LDAP user search filter. 
+For example `(&(objectClass=user)(sAMAccountName={0}))`. For more information, 
+see 
+https://msdn.microsoft.com/en-us/library/aa746475(v=vs.85).aspx[Search Filter Syntax].
 
 `user_search.upn_filter`::
 Specifies a filter to use to lookup a user given a user principal name.
 The default filter looks up `user` objects with
 a matching `userPrincipalName`. If specified, this
-must be a valid LDAP user search filter, for example
+must be a valid LDAP user search filter. For example,
 `(&(objectClass=user)(userPrincipalName={1}))`. `{1}` is the full user principal name
-provided by the user.
+provided by the user. For more information, see 
+https://msdn.microsoft.com/en-us/library/aa746475(v=vs.85).aspx[Search Filter Syntax].
 
 `user_search.down_level_filter`::
 Specifies a filter to use to lookup a user given a down level logon name
 (DOMAIN\user). The default filter looks up `user` objects with a matching
 `sAMAccountName` in the domain provided. If specified, this
-must be a valid LDAP user search filter, for example
-`(&(objectClass=user)(sAMAccountName={0}))`.
+must be a valid LDAP user search filter. For example,
+`(&(objectClass=user)(sAMAccountName={0}))`. For more information, see 
+https://msdn.microsoft.com/en-us/library/aa746475(v=vs.85).aspx[Search Filter Syntax]. 
 
 `user_search.pool.enabled`::
 Enables or disables connection pooling for user search. When
@@ -499,16 +539,18 @@ connection pool. Defaults to `20`.
 
 `user_search.pool.initial_size`::
 The initial number of connections to create to the Active Directory server on startup.
-Defaults to `0`.
+Defaults to `0`. If the LDAP server is down, values greater than 0 
+could cause startup failures. 
 
 `user_search.pool.health_check.enabled`::
-Flag to enable or disable a health check on Active Directory connections in the connection
+Enables or disables a health check on Active Directory connections in the connection
 pool. Connections are checked in the background at the specified interval.
 Defaults to `true`.
 
 `user_search.pool.health_check.dn`::
 The distinguished name to be retrieved as part of the health check.
-Defaults to the value of `bind_dn` if it is a distinguished name.
+Defaults to the value of `bind_dn` if that setting is present. Otherwise, it 
+defaults to the value of the `user_search.base_dn` setting. 
 
 `user_search.pool.health_check.interval`::
 The interval to perform background checks of connections in the pool.
@@ -516,7 +558,7 @@ Defaults to `60s`.
 
 `group_search.base_dn`::
 The context to search for groups in which the user has membership.  Defaults
-to the root of the  Active Directory domain.
+to the root of the Active Directory domain.
 
 `group_search.scope`::
 Specifies whether the group search should be `sub_tree`, `one_level` or
@@ -546,13 +588,18 @@ Defaults to `5s` (5 seconds ).
 
 `ssl.certificate`::
 Path to a PEM encoded file containing the certificate (or certificate chain)
-that will be presented to clients when they connect.
+that will be presented to clients when they connect. 
 
 `ssl.certificate_authorities`::
-List of paths to PEM encoded certificate files that should be trusted.
+List of paths to PEM encoded certificate files that should be trusted. 
+`ssl.certificate_authorities` and `ssl.truststore.path` cannot be used at the 
+same time.
 
 `ssl.key`::
-Path to the PEM encoded file containing the private key.
+Path to the PEM encoded file containing the private key, which is used when the 
+Active Directory server requires client authentication. `ssl.key` and 
+`ssl.keystore.path` cannot be used at the same time.
+
 
 `ssl.key_passphrase`::
 The passphrase that is used to decrypt the private key. This value is
@@ -576,6 +623,7 @@ The password to the keystore.
 
 `ssl.keystore.path`::
 The path to the Java Keystore file that contains a private key and certificate.
+`ssl.key` and `ssl.keystore.path` cannot be used at the same time.
 
 `ssl.keystore.type`::
 The format of the keystore file. Should be either `jks` to use the Java
@@ -589,6 +637,8 @@ The password to the truststore.
 
 `ssl.truststore.path`::
 The path to the Java Keystore file that contains the certificates to trust.
+`ssl.certificate_authorities` and `ssl.truststore.path` cannot be used at the 
+same time.
 
 `ssl.truststore.type`::
 The format of the truststore file. Should be either `jks` to use the Java
@@ -603,17 +653,18 @@ See <> for an explanation of
 these values.
 
 `ssl.supported_protocols`::
-Supported protocols with versions. Defaults to the value of
+Supported protocols for TLS/SSL (with versions). Defaults to the value of
 `xpack.ssl.supported_protocols`.
 
-`ssl.cipher_suites`::
+`ssl.cipher_suites`:: Specifies the cipher suites that should be supported when 
+communicating with the Active Directory server. 
 Supported cipher suites can be found in Oracle's http://docs.oracle.com/javase/8/docs/technotes/guides/security/SunProviders.html[
 Java Cryptography Architecture documentation]. Defaults to the value of
 `xpack.ssl.cipher_suites`.
 
 `cache.ttl`::
-Specifies the time-to-live for cached user entries (user
-credentials are cached for this configured period of time). Use the
+Specifies the time-to-live for cached user entries. A user and a hash of its 
+credentials are cached for this configured period of time. Use the
 standard Elasticsearch {ref}/common-options.html#time-units[time units]).
 Defaults to `20m`.
 
@@ -622,9 +673,14 @@ Specifies the maximum number of user entries that the cache can contain.
 Defaults to `100000`.
 
 `cache.hash_algo`::
-(Expert Setting) Specifies the hashing algorithm that will be used for
+(Expert Setting) Specifies the hashing algorithm that is used for
 the in-memory cached user credentials (see {xpack-ref}/controlling-user-cache.html#controlling-user-cache[Cache hash algorithms] table for all possible values). Defaults to `ssha256`.
 
+`follow_referrals`::
+If set to `true` {security} follows referrals returned by the LDAP server. 
+Referrals are URLs returned by the server that are to be used to continue the 
+LDAP operation (such as `search`). Defaults to `true`.
+
 [[ref-pki-settings]]
 [float]
 ===== PKI realm settings
@@ -660,11 +716,12 @@ documentation for more information. This setting cannot be used with
 `files.role_mapping`::
 Specifies the {xpack-ref}/security-files.html[location] of the
 {xpack-ref}/mapping-roles.html[YAML role  mapping configuration file].
-Defaults to `CONFIG_DIR/x-pack/role_mapping.yml`.
+Defaults to `CONFIG_DIR/role_mapping.yml`.
 
 `cache.ttl`::
-Specifies the time-to-live for cached user entries. Use the
-standard Elasticsearch {ref}/common-options.html#time-units[time units]).
+Specifies the time-to-live for cached user entries. A user and a hash of its 
+credentials are cached for this period of time. Use the
+standard {es} {ref}/common-options.html#time-units[time units]).
 Defaults to `20m`.
 
 `cache.max_users`::
@@ -935,6 +992,32 @@ supported protocols for TLS/SSL.
 If retrieving IDP metadata via https (see `idp.metadata.path`), specifies the
 cipher suites that should be supported.
 
+[float]
+[[load-balancing]]
+===== Load balancing and failover
+
+The `load_balance.type` setting can have the following values: 
+
+* `failover`: The URLs specified are used in the order that they are specified. 
+The first server that can be connected to will be used for all subsequent 
+connections. If a connection to that server fails then the next server that a 
+connection can be established to will be used for subsequent connections.
+* `dns_failover`: In this mode of operation, only a single URL may be specified.
+This URL must contain a DNS name. The system will be queried for all IP 
+addresses that correspond to this DNS name. Connections to the Active Directory 
+or LDAP server will always be tried in the order in which they were retrieved. 
+This differs from `failover` in that there is no reordering of the list and if a 
+server has failed at the beginning of the list, it will still be tried for each 
+subsequent connection.
+* `round_robin`: Connections will continuously iterate through the list of 
+provided URLs. If a server is unavailable, iterating through the list of URLs 
+will continue until a successful connection is made.
+* `dns_round_robin`: In this mode of operation, only a single URL may be 
+specified. This URL must contain a DNS name. The system will be queried for all 
+IP addresses that correspond to this DNS name. Connections will continuously 
+iterate through the list of addresses. If a server is unavailable, iterating 
+through the list of URLs will continue until a successful connection is made.
+
 [float]
 [[ssl-tls-settings]]
 ==== Default TLS/SSL settings
diff --git a/x-pack/docs/en/watcher/encrypting-data.asciidoc b/x-pack/docs/en/watcher/encrypting-data.asciidoc
index ca06d9666bb76..166ef6f14d760 100644
--- a/x-pack/docs/en/watcher/encrypting-data.asciidoc
+++ b/x-pack/docs/en/watcher/encrypting-data.asciidoc
@@ -8,7 +8,7 @@ cluster.
 
 To encrypt sensitive data in {watcher}:
 
-. Use the {ref}/syskeygen.html[syskeygen] command to create a system key file.
+. Use the {ref}/syskeygen.html[elasticsearch-syskeygen] command to create a system key file.
 
 . Copy the `system_key` file to all of the nodes in your cluster.
 +
diff --git a/x-pack/plugin/ml/build.gradle b/x-pack/plugin/ml/build.gradle
index af2122d43d9a7..d9d4882b00e1c 100644
--- a/x-pack/plugin/ml/build.gradle
+++ b/x-pack/plugin/ml/build.gradle
@@ -64,6 +64,23 @@ artifacts {
     testArtifacts testJar
 }
 
+task extractNativeLicenses(type: Copy) {
+    dependsOn configurations.nativeBundle
+    into "${buildDir}"
+    from {
+        project.zipTree(configurations.nativeBundle.singleFile)
+    }
+    include 'platform/licenses/**'
+}
+project.afterEvaluate {
+    // Add an extra licenses directory to the combined notices
+    project.tasks.findByName('generateNotice').dependsOn extractNativeLicenses
+    project.tasks.findByName('generateNotice').licensesDir new File("${project.buildDir}/platform/licenses")
+    project.tasks.findByName('generateNotice').outputs.upToDateWhen {
+        extractNativeLicenses.state.upToDate
+    }
+}
+
 run {
     plugin xpackModule('core')
 }
@@ -85,7 +102,7 @@ task internalClusterTest(type: RandomizedTestingTask,
   include '**/*IT.class'
   systemProperty 'es.set.netty.runtime.available.processors', 'false'
 }
-check.dependsOn internalClusterTest 
+check.dependsOn internalClusterTest
 internalClusterTest.mustRunAfter test
 
 // also add an "alias" task to make typing on the command line easier
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
index 4a096f0ca4a46..098f4190b0e88 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -188,6 +189,7 @@ public void testNodesHash() {
 
     @Override
     public void testToXContent() throws IOException {
+        final String clusterUuid = "_cluster";
         final ClusterName clusterName = new ClusterName("_cluster_name");
         final TransportAddress transportAddress = new TransportAddress(TransportAddress.META_ADDRESS, 9300);
         final DiscoveryNode discoveryNode = new DiscoveryNode("_node_name",
@@ -201,6 +203,7 @@ public void testToXContent() throws IOException {
                                                                 Version.V_6_0_0_beta1);
 
         final ClusterState clusterState = ClusterState.builder(clusterName)
+                                                        .metaData(MetaData.builder().clusterUUID(clusterUuid).build())
                                                         .stateUUID("_state_uuid")
                                                         .version(12L)
                                                         .nodes(DiscoveryNodes.builder()
@@ -500,6 +503,7 @@ public void testToXContent() throws IOException {
                   + "\"cluster_state\":{"
                     + "\"nodes_hash\":1314980060,"
                     + "\"status\":\"green\","
+                    + "\"cluster_uuid\":\"_cluster\","
                     + "\"version\":12,"
                     + "\"state_uuid\":\"_state_uuid\","
                     + "\"master_node\":\"_node\","