add RemoveCorruptedShardDataCommand

distribution/src/bin/elasticsearch-shard

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+ES_MAIN_CLASS=org.elasticsearch.index.shard.ShardToolCli \
+  "`dirname "$0"`"/elasticsearch-cli \
+  "$@"

distribution/src/bin/elasticsearch-shard.bat

@@ -0,0 +1,12 @@
+@echo off
+
+setlocal enabledelayedexpansion
+setlocal enableextensions
+
+set ES_MAIN_CLASS=org.elasticsearch.index.shard.ShardToolCli
+call "%~dp0elasticsearch-cli.bat" ^
+  %%* ^
+  || exit /b 1
+
+endlocal
+endlocal

docs/reference/commands/index.asciidoc

@@ -12,6 +12,7 @@ tasks from the command line:
 * <<migrate-tool>>
 * <<saml-metadata>>
 * <<setup-passwords>>
+* <<shard-tool>>
 * <<syskeygen>>
 * <<users-command>>
 
@@ -22,5 +23,6 @@ include::certutil.asciidoc[]
 include::migrate-tool.asciidoc[]
 include::saml-metadata.asciidoc[]
 include::setup-passwords.asciidoc[]
+include::shard-tool.asciidoc[]
 include::syskeygen.asciidoc[]
 include::users-command.asciidoc[]

docs/reference/commands/shard-tool.asciidoc

@@ -0,0 +1,107 @@
+[[shard-tool]]
+== elasticsearch-shard
+
+In some cases the Lucene index or translog of a shard copy can become
+corrupted. The `elasticsearch-shard` command enables you to remove corrupted
+parts of the shard if a good copy of the shard cannot be recovered
+automatically or restored from backup.
+
+[WARNING]
+You will lose the corrupted data when you run `elasticsearch-shard`.  This tool
+should only be used as a last resort if there is no way to recover from another
+copy of the shard or restore a snapshot.
+
+When Elasticsearch detects that a shard's data is corrupted, it fails that
+shard copy and refuses to use it. Under normal conditions, the shard is
+automatically recovered from another copy. If no good copy of the shard is
+available and you cannot restore from backup, you can use `elasticsearch-shard`
+to remove the corrupted data and restore access to any remaining data in
+unaffected segments.
+
+[WARNING]
+Stop Elasticsearch before running `elasticsearch-shard`.
+
+To remove corrupted shard data use the `remove-corrupted-data` subcommand.
+
+There are two ways to specify the path:
+
+* Specify the index name and shard name with the `--index` and `--shard-id`
+  options.
+* Use the `--dir` option to specify the full path to the corrupted index or
+  translog files.
+
+[float]
+=== Removing corrupted data
+
+`elasticsearch-shard` analyses the shard copy and provides an overview of the
+corruption found. To proceed you must then confirm that you want to remove the
+corrupted data.
+
+[WARNING]
+Back up your data before running `elasticsearch-shard`. This is a destructive
+operation that removes corrupted data from the shard.
+
+[source,txt]
+--------------------------------------------------
+$ bin/elasticsearch-shard remove-corrupted-data --index twitter --shard-id 0
+
+
+    WARNING: Elasticsearch MUST be stopped before running this tool.
+
+  Please make a complete backup of your index before using this tool.
+
+
+Opening Lucene index at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/index/
+
+ >> Lucene index is corrupted at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/index/
+
+Opening translog at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/
+
+
+ >> Translog is clean at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/
+
+
+  Corrupted Lucene index segments found - 32 documents will be lost.
+
+            WARNING:              YOU WILL LOSE DATA.
+
+Continue and remove docs from the index ? Y
+
+WARNING: 1 broken segments (containing 32 documents) detected
+Took 0.056 sec total.
+Writing...
+OK
+Wrote new segments file "segments_c"
+Marking index with the new history uuid : 0pIBd9VTSOeMfzYT6p0AsA
+Changing allocation id V8QXk-QXSZinZMT-NvEq4w to tjm9Ve6uTBewVFAlfUMWjA
+
+You should run the following command to allocate this shard:
+
+POST /_cluster/reroute
+{
+  "commands" : [
+    {
+      "allocate_stale_primary" : {
+        "index" : "index42",
+        "shard" : 0,
+        "node" : "II47uXW2QvqzHBnMcl2o_Q",
+        "accept_data_loss" : false
+      }
+    }
+  ]
+}
+
+You must accept the possibility of data loss by changing parameter `accept_data_loss` to `true`.
+
+Deleted corrupt marker corrupted_FzTSBSuxT7i3Tls_TgwEag from /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/index/
+
+--------------------------------------------------
+
+When you use `elasticsearch-shard` to drop the corrupted data, the shard's
+allocation ID changes. After restarting the node, you must use the
+<<cluster-reroute,cluster reroute API>> to tell Elasticsearch to use the new
+ID. The `elasticsearch-shard` command shows the request that
+you need to submit.
+
+You can also use the `-h` option to get a list of all options and parameters
+that the `elasticsearch-shard` tool supports.

docs/reference/index-modules/translog.asciidoc

@@ -92,6 +92,10 @@ The maximum duration for which translog files will be kept. Defaults to `12h`.
 [[corrupt-translog-truncation]]
 === What to do if the translog becomes corrupted?
 
+[WARNING]
+This tool is deprecated and will be completely removed in 7.0.
+Use the <<shard-tool,elasticsearch-shard tool>> instead of this one.
+
 In some cases (a bad drive, user error) the translog on a shard copy can become
 corrupted. When this corruption is detected by Elasticsearch due to mismatching
 checksums, Elasticsearch will fail that shard copy and refuse to use that copy

libs/cli/src/main/java/org/elasticsearch/cli/Terminal.java

@@ -85,12 +85,17 @@ public final void println(Verbosity verbosity, String msg) {
 
     /** Prints message to the terminal at {@code verbosity} level, without a newline. */
     public final void print(Verbosity verbosity, String msg) {
-        if (this.verbosity.ordinal() >= verbosity.ordinal()) {
+        if (isPrintable(verbosity)) {
             getWriter().print(msg);
             getWriter().flush();
         }
     }
 
+    /** Checks if is enough {@code verbosity} level to be printed */
+    public final boolean isPrintable(Verbosity verbosity) {
+        return this.verbosity.ordinal() >= verbosity.ordinal();
+    }
+
     /**
      * Prompt for a yes or no answer from the user. This method will loop until 'y' or 'n'
      * (or the default empty value) is entered.

qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java

@@ -325,4 +325,21 @@ public void test90SecurityCliPackaging() {
         }
     }
 
+    public void test100RepairIndexCliPackaging() {
+        assumeThat(installation, is(notNullValue()));
+
+        final Installation.Executables bin = installation.executables();
+        final Shell sh = new Shell();
+
+        Platforms.PlatformAction action = () -> {
+            final Result result = sh.run(bin.elasticsearchShard + " help");
+            assertThat(result.stdout, containsString("A CLI tool to remove corrupted parts of unrecoverable shards"));
+        };
+
+        if (distribution().equals(Distribution.DEFAULT_TAR) || distribution().equals(Distribution.DEFAULT_ZIP)) {
+            Platforms.onLinux(action);
+            Platforms.onWindows(action);
+        }
+    }
+
 }

qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java

@@ -186,6 +186,7 @@ private static void verifyOssInstallation(Installation es, Distribution distribu
             "elasticsearch-env",
             "elasticsearch-keystore",
             "elasticsearch-plugin",
+            "elasticsearch-shard",
             "elasticsearch-translog"
         ).forEach(executable -> {
 

qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java

@@ -100,8 +100,9 @@ public class Executables {
         public final Path elasticsearch = platformExecutable("elasticsearch");
         public final Path elasticsearchPlugin = platformExecutable("elasticsearch-plugin");
         public final Path elasticsearchKeystore = platformExecutable("elasticsearch-keystore");
-        public final Path elasticsearchTranslog = platformExecutable("elasticsearch-translog");
         public final Path elasticsearchCertutil = platformExecutable("elasticsearch-certutil");
+        public final Path elasticsearchShard = platformExecutable("elasticsearch-shard");
+        public final Path elasticsearchTranslog = platformExecutable("elasticsearch-translog");
 
         private Path platformExecutable(String name) {
             final String platformExecutableName = Platforms.WINDOWS

qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java

@@ -187,6 +187,7 @@ private static void verifyOssInstallation(Installation es, Distribution distribu
             "elasticsearch",
             "elasticsearch-plugin",
             "elasticsearch-keystore",
+            "elasticsearch-shard",
             "elasticsearch-translog"
         ).forEach(executable -> assertThat(es.bin(executable), file(File, "root", "root", p755)));
 

qa/vagrant/src/test/resources/packaging/utils/packages.bash

@@ -95,6 +95,7 @@ verify_package_installation() {
     assert_file "$ESHOME/bin" d root root 755
     assert_file "$ESHOME/bin/elasticsearch" f root root 755
     assert_file "$ESHOME/bin/elasticsearch-plugin" f root root 755
+    assert_file "$ESHOME/bin/elasticsearch-shard" f root root 755
     assert_file "$ESHOME/bin/elasticsearch-translog" f root root 755
     assert_file "$ESHOME/lib" d root root 755
     assert_file "$ESCONFIG" d root elasticsearch 2750

qa/vagrant/src/test/resources/packaging/utils/tar.bash

@@ -94,6 +94,7 @@ verify_archive_installation() {
     assert_file "$ESHOME/bin/elasticsearch-env" f elasticsearch elasticsearch 755
     assert_file "$ESHOME/bin/elasticsearch-keystore" f elasticsearch elasticsearch 755
     assert_file "$ESHOME/bin/elasticsearch-plugin" f elasticsearch elasticsearch 755
+    assert_file "$ESHOME/bin/elasticsearch-shard" f elasticsearch elasticsearch 755
     assert_file "$ESHOME/bin/elasticsearch-translog" f elasticsearch elasticsearch 755
     assert_file "$ESCONFIG" d elasticsearch elasticsearch 755
     assert_file "$ESCONFIG/elasticsearch.yml" f elasticsearch elasticsearch 660