add RemoveCorruptedShardDataCommand

distribution/src/bin/elasticsearch-shard

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+ES_MAIN_CLASS=org.elasticsearch.index.shard.ShardToolCli \
+  "`dirname "$0"`"/elasticsearch-cli \
+  "$@"

distribution/src/bin/elasticsearch-translog.bat → distribution/src/bin/elasticsearch-shard.bat

@@ -3,7 +3,7 @@
 setlocal enabledelayedexpansion
 setlocal enableextensions
 
-set ES_MAIN_CLASS=org.elasticsearch.index.translog.TranslogToolCli
+set ES_MAIN_CLASS=org.elasticsearch.index.shard.ShardToolCli
 call "%~dp0elasticsearch-cli.bat" ^
   %%* ^
   || exit /b 1

docs/reference/index-modules.asciidoc

@@ -63,12 +63,6 @@ corruption is detected, it will prevent the shard from being opened. Accepts:
     Check for both physical and logical corruption. This is much more
     expensive in terms of CPU and memory usage.
 
-`fix`::
-
-    Check for both physical and logical corruption.  Segments that were reported
-    as corrupted will be automatically removed. This option *may result in data loss*.
-    Use with extreme caution!
-
 WARNING: Expert only. Checking shards may take a lot of time on large indices.
 --
 
@@ -279,6 +273,17 @@ Other index settings are available in index modules:
 
     Control over the transaction log and background flush operations.
 
+<<index-modules-command-line-tools,Command-line tools>>::
+
+    Command-line tools if shard is corrupted
+
+[float]
+[[elasticsearch-shard-tool]]
+=== elasticsearch-shard tool
+
+You can use the <<index-modules-elasticsearch-shard,shard-tool>> recovery tool to remove corrupted translog or corrupted Lucene
+segments if a shard cannot be recovered automatically or restored from backup.
+
 --
 
 include::index-modules/analysis.asciidoc[]
@@ -297,4 +302,6 @@ include::index-modules/store.asciidoc[]
 
 include::index-modules/translog.asciidoc[]
 
+include::index-modules/shard-tool.asciidoc[]
+
 include::index-modules/index-sorting.asciidoc[]

docs/reference/index-modules/shard-tool.asciidoc

@@ -0,0 +1,126 @@
+[[shard-tool]]
+== elasticsearch-shard
+
+In some cases (a bad drive, user error) the Lucene index or translog on a shard copy can become corrupted.
+
+The `elasticsearch-shard` command enables you to remove a corrupted Lucene index segments or corrupted translog
+if a shard cannot be recovered automatically or restored from backup.
+
+[WARNING]
+You will lose the corrupted data when you run `elasticsearch-shard`.
+This tool should only be used as a last resort if there is no way to recover from another copy of the shard
+or restore a snapshot.
+
+When Elasticsearch detects that a shard's translog or Lucene index is corrupted, it fails that shard copy
+and quits using it. Under normal conditions, the shard is automatically recovered from another copy.
+If no good copy of the shard is available and you cannot restore from backup, you can use `elasticsearch-shard`
+to remove only the corrupted data and restore access to the data in unaffected segments.
+
+[WARNING]
+Stop Elasticsearch before running `elasticsearch-shard`.
+
+To remove corrupted Lucene index segments and/or corrupted translog files, use the `remove-corrupted-data` subcommand.
+
+There are two ways to specify the path:
+
+* Specify the index name and shard name with the `--index` and `--shard-id` options.
+* Use the `-dir` option to specify the full path to the corrupted index or translog files.
+
+=== Listing corrupted Lucene index files and/or corrupted translog files
+
+You can get an overview of the corruption with `--dry-run` option :
+
+[source,txt]
+--------------------------------------------------
+$ bin/elasticsearch-shard remove-corrupted-data --dry-run --index twitter --shard-id 0
+
+
+    WARNING: ElasticSearch MUST be stopped before running this tool.
+
+  Please make a complete backup of your index before using this tool.
+
+
+Opening Lucene index at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/index/
+
+ >> Lucene index is corrupted at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/index/
+
+Opening translog at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/
+
+ >> Translog is clean at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/
+
+
+  Corrupted Lucene index segments found - 32 documents will be lost.
+
+
+--------------------------------------------------
+
+=== Removing a corrupted Lucene index files and/or corrupted translog files
+
+You must confirm that you want to remove the corrupted segments when run `elasticsearch-shard` without `--dry-run`:
+
+[WARNING]
+Back up your data before running `elasticsearch-shard`. This is a destructive operation that removes corrupted data from the shard.
+
+[source,txt]
+--------------------------------------------------
+$ bin/elasticsearch-shard remove-corrupted-data --index twitter --shard-id 0
+
+
+    WARNING: ElasticSearch MUST be stopped before running this tool.
+
+  Please make a complete backup of your index before using this tool.
+
+
+Opening Lucene index at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/index/
+
+ >> Lucene index is corrupted at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/index/
+
+Opening translog at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/
+
+
+ >> Translog is clean at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/
+
+
+  Corrupted Lucene index segments found - 32 documents will be lost.
+
+            WARNING:              YOU WILL LOSE DATA.
+
+Continue and remove docs from the index ? Y
+
+WARNING: 1 broken segments (containing 32 documents) detected
+Took 0.056 sec total.
+Writing...
+OK
+Wrote new segments file "segments_c"
+Marking index with the new history uuid : 0pIBd9VTSOeMfzYT6p0AsA
+Changing allocation id V8QXk-QXSZinZMT-NvEq4w to tjm9Ve6uTBewVFAlfUMWjA
+You should run follow command to apply allocation id changes:
+
+POST /_cluster/reroute
+{
+  "commands" : [
+    {
+      "allocate_stale_primary" : {
+        "index" : "index42",
+        "shard" : 0,
+        "node" : "II47uXW2QvqzHBnMcl2o_Q",
+        "accept_data_loss" : false
+      }
+    }
+  ]
+}
+
+Deleted corrupt marker corrupted_FzTSBSuxT7i3Tls_TgwEag
+
+--------------------------------------------------
+
+
+When you use `elasticsearch-shard` to drop the corrupted data, the shard's allocation ID changes.
+After you restart the node, you must use the cluster reroute API to tell Elasticsearch to use the new ID.
+When you run the `elasticsearch-shard` command, it shows the request that you need to submit.
+
+[WARNING]
+You should admin data loss changing parameter `accept_data_loss` to `true`.
+
+You can also use the `-h` option to get a list of all options and parameters
+that the `elasticsearch-shard` tool supports.

docs/reference/index-modules/translog.asciidoc

@@ -88,59 +88,4 @@ file based sync. Defaults to `512mb`
 The maximum duration for which translog files will be kept. Defaults to `12h`.
 
 
-[float]
-[[corrupt-translog-truncation]]
-=== What to do if the translog becomes corrupted?
-
-In some cases (a bad drive, user error) the translog on a shard copy can become
-corrupted. When this corruption is detected by Elasticsearch due to mismatching
-checksums, Elasticsearch will fail that shard copy and refuse to use that copy
-of the data.  If there are other copies of the shard available then
-Elasticsearch will automatically recover from one of them using the normal
-shard allocation and recovery mechanism.  In particular, if the corrupt shard
-copy was the primary when the corruption was detected then one of its replicas
-will be promoted in its place.
-
-If there is no copy of the data from which Elasticsearch can recover
-successfully, a user may want to recover the data that is part of the shard at
-the cost of losing the data that is currently contained in the translog. We
-provide a command-line tool for this, `elasticsearch-translog`.
-
-[WARNING]
-The `elasticsearch-translog` tool should *not* be run while Elasticsearch is
-running. If you attempt to run this tool while Elasticsearch is running, you 
-will permanently lose the documents that were contained only in the translog!
-
-In order to run the `elasticsearch-translog` tool, specify the `truncate`
-subcommand as well as the directory for the corrupted translog with the `-d`
-option:
-
-[source,txt]
---------------------------------------------------
-$ bin/elasticsearch-translog truncate -d /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/
-Checking existing translog files
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-!   WARNING: Elasticsearch MUST be stopped before running this tool   !
-!                                                                     !
-!   WARNING:    Documents inside of translog files will be lost       !
-!                                                                     !
-!   WARNING:          The following files will be DELETED!            !
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
---> data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/translog-41.ckp
---> data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/translog-6.ckp
---> data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/translog-37.ckp
---> data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/translog-24.ckp
---> data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/translog-11.ckp
-
-Continue and DELETE files? [y/N] y
-Reading translog UUID information from Lucene commit from shard at [data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/index]
-Translog Generation: 3
-Translog UUID      : AxqC4rocTC6e0fwsljAh-Q
-Removing existing translog files
-Creating new empty checkpoint at [data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/translog.ckp]
-Creating new empty translog at [data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/translog-3.tlog]
-Done.
---------------------------------------------------
-
-You can also use the `-h` option to get a list of all options and parameters
-that the `elasticsearch-translog` tool supports.
+[float]

docs/reference/migration/migrate_7_0/indices.asciidoc

@@ -78,3 +78,7 @@ The parent circuit breaker defines a new setting `indices.breaker.total.use_real
 heap memory instead of only considering the reserved memory by child circuit breakers. When this
 setting is `true`, the default parent breaker limit also changes from 70% to 95% of the JVM heap size.
 The previous behavior can be restored by setting `indices.breaker.total.use_real_memory` to `false`.
+
+==== `elasticsearch-translog` tool merged into `elasticsearch-shard`
+
+Use the `elasticsearch-shard` tool to remove corrupted translog data.

libs/cli/src/main/java/org/elasticsearch/cli/Terminal.java

@@ -85,12 +85,17 @@ public final void println(Verbosity verbosity, String msg) {
 
     /** Prints message to the terminal at {@code verbosity} level, without a newline. */
     public final void print(Verbosity verbosity, String msg) {
-        if (this.verbosity.ordinal() >= verbosity.ordinal()) {
+        if (isPrintable(verbosity)) {
             getWriter().print(msg);
             getWriter().flush();
         }
     }
 
+    /** Checks if is enough {@code verbosity} level to be printed */
+    public final boolean isPrintable(Verbosity verbosity) {
+        return this.verbosity.ordinal() >= verbosity.ordinal();
+    }
+
     /**
      * Prompt for a yes or no answer from the user. This method will loop until 'y' or 'n'
      * (or the default empty value) is entered.

qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java

@@ -325,4 +325,21 @@ public void test90SecurityCliPackaging() {
         }
     }
 
+    public void test100RepairIndexCliPackaging() {
+        assumeThat(installation, is(notNullValue()));
+
+        final Installation.Executables bin = installation.executables();
+        final Shell sh = new Shell();
+
+        Platforms.PlatformAction action = () -> {
+            final Result result = sh.run(bin.elasticsearchShard + " help");
+            assertThat(result.stdout, containsString("A CLI tool to remove corrupted parts of unrecoverable shards"));
+        };
+
+        if (distribution().equals(Distribution.DEFAULT_TAR) || distribution().equals(Distribution.DEFAULT_ZIP)) {
+            Platforms.onLinux(action);
+            Platforms.onWindows(action);
+        }
+    }
+
 }

qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java

@@ -186,7 +186,7 @@ private static void verifyOssInstallation(Installation es, Distribution distribu
             "elasticsearch-env",
             "elasticsearch-keystore",
             "elasticsearch-plugin",
-            "elasticsearch-translog"
+            "elasticsearch-shard"
         ).forEach(executable -> {
 
             assertThat(es.bin(executable), file(File, owner, owner, p755));

qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java

@@ -100,8 +100,8 @@ public class Executables {
         public final Path elasticsearch = platformExecutable("elasticsearch");
         public final Path elasticsearchPlugin = platformExecutable("elasticsearch-plugin");
         public final Path elasticsearchKeystore = platformExecutable("elasticsearch-keystore");
-        public final Path elasticsearchTranslog = platformExecutable("elasticsearch-translog");
         public final Path elasticsearchCertutil = platformExecutable("elasticsearch-certutil");
+        public final Path elasticsearchShard = platformExecutable("elasticsearch-shard");
 
         private Path platformExecutable(String name) {
             final String platformExecutableName = Platforms.WINDOWS

qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java

@@ -187,7 +187,7 @@ private static void verifyOssInstallation(Installation es, Distribution distribu
             "elasticsearch",
             "elasticsearch-plugin",
             "elasticsearch-keystore",
-            "elasticsearch-translog"
+            "elasticsearch-shard"
         ).forEach(executable -> assertThat(es.bin(executable), file(File, "root", "root", p755)));
 
         Stream.of(

qa/vagrant/src/test/resources/packaging/utils/packages.bash

@@ -95,7 +95,7 @@ verify_package_installation() {
     assert_file "$ESHOME/bin" d root root 755
     assert_file "$ESHOME/bin/elasticsearch" f root root 755
     assert_file "$ESHOME/bin/elasticsearch-plugin" f root root 755
-    assert_file "$ESHOME/bin/elasticsearch-translog" f root root 755
+    assert_file "$ESHOME/bin/elasticsearch-shard" f root root 755
     assert_file "$ESHOME/lib" d root root 755
     assert_file "$ESCONFIG" d root elasticsearch 2750
     assert_file "$ESCONFIG/elasticsearch.keystore" f root elasticsearch 660

qa/vagrant/src/test/resources/packaging/utils/tar.bash

@@ -94,7 +94,7 @@ verify_archive_installation() {
     assert_file "$ESHOME/bin/elasticsearch-env" f elasticsearch elasticsearch 755
     assert_file "$ESHOME/bin/elasticsearch-keystore" f elasticsearch elasticsearch 755
     assert_file "$ESHOME/bin/elasticsearch-plugin" f elasticsearch elasticsearch 755
-    assert_file "$ESHOME/bin/elasticsearch-translog" f elasticsearch elasticsearch 755
+    assert_file "$ESHOME/bin/elasticsearch-shard" f elasticsearch elasticsearch 755
     assert_file "$ESCONFIG" d elasticsearch elasticsearch 755
     assert_file "$ESCONFIG/elasticsearch.yml" f elasticsearch elasticsearch 660
     assert_file "$ESCONFIG/jvm.options" f elasticsearch elasticsearch 660