Add setting to bypass Rollover action (#36235)

Adds a setting that indicates that an index is done indexing, set by ILM
when the Rollover action completes. This indicates that the Rollover
action should be skipped in any future invocations, as long as the index
is no longer the write index for its alias.

This enables 1) an index with a policy that involves the Rollover action
to have the policy removed and switched to another one without use of
the move-to-step API, and 2) integrations with Beats and CCR.

docs/reference/ilm/using-policies-rollover.asciidoc

@@ -124,3 +124,36 @@ suffix number for each subsequent index.
 When the rollover is performed, the newly-created index is set as the write
 index for the rolled over alias. Documents sent to the alias are indexed into
 the new index, enabling indexing to continue uninterrupted.
+
+=== Skipping Rollover
+
+beta[]
+
+After an index has been rolled over by {ilm}, the
+`index.lifecycle.indexing_complete` setting will be set to `true` on the index.
+This indicates to {ilm} that this index has already been rolled over, and does
+not need to be rolled over again. If you <<ilm-remove-policy,remove the policy>>
+from an index and set it to use another policy, this setting indicates that the
+new policy should skip execution of the Rollover action.
+
+You can also set this setting to `true` manually if you want to indicate that
+{ilm} should not roll over a particular index. This is useful if you need to
+make an exception to your normal Lifecycle Policy and switching the alias to a
+different index by hand, but do not want to remove the index from {ilm}
+completely.
+
+IMPORTANT: If `index.lifecycle.indexing_complete` is set to `true` on an index,
+it will not be rolled over by {ilm}, but {ilm} will verify that this index is no
+longer the write index for the alias specified by
+`index.lifecycle.rollover_alias`. If that setting is missing, or if the index is
+still the write index for that alias, this index will be moved to the
+<<index-lifecycle-error-handling,error step>>.
+
+For example, if you wish to change the name of new indices while retaining
+previous data in accordance with your configured policy, you can create the
+template for the new index name pattern and the first index with the new name
+manually, change the write index of the alias using the <<indices-aliases, Index
+Aliases API>>, and set `index.lifecycle.indexing_complete` to `true` on the old
+index to indicate that it does not need to be rolled over. This way, {ilm} will
+continue to manage the old index in accordance with its existing policy, as well
+as the new one, with no interruption.

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleSettings.java

@@ -14,9 +14,12 @@
 public class LifecycleSettings {
     public static final String LIFECYCLE_POLL_INTERVAL = "indices.lifecycle.poll_interval";
     public static final String LIFECYCLE_NAME = "index.lifecycle.name";
+    public static final String LIFECYCLE_INDEXING_COMPLETE = "index.lifecycle.indexing_complete";
 
     public static final Setting<TimeValue> LIFECYCLE_POLL_INTERVAL_SETTING = Setting.positiveTimeSetting(LIFECYCLE_POLL_INTERVAL,
         TimeValue.timeValueMinutes(10), Setting.Property.Dynamic, Setting.Property.NodeScope);
     public static final Setting<String> LIFECYCLE_NAME_SETTING = Setting.simpleString(LIFECYCLE_NAME,
         Setting.Property.Dynamic, Setting.Property.IndexScope);
+    public static final Setting<Boolean> LIFECYCLE_INDEXING_COMPLETE_SETTING = Setting.boolSetting(LIFECYCLE_INDEXING_COMPLETE, false,
+        Setting.Property.Dynamic, Setting.Property.IndexScope);
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverAction.java

@@ -11,6 +11,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
@@ -29,6 +30,7 @@
  */
 public class RolloverAction implements LifecycleAction {
     public static final String NAME = "rollover";
+    public static final String INDEXING_COMPLETE_STEP_NAME = "set-indexing-complete";
     public static final ParseField MAX_SIZE_FIELD = new ParseField("max_size");
     public static final ParseField MAX_DOCS_FIELD = new ParseField("max_docs");
     public static final ParseField MAX_AGE_FIELD = new ParseField("max_age");
@@ -132,23 +134,30 @@ public boolean isSafeAction() {
 
     @Override
     public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey) {
+        Settings indexingComplete = Settings.builder().put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, true).build();
+
         StepKey waitForRolloverReadyStepKey = new StepKey(phase, NAME, WaitForRolloverReadyStep.NAME);
         StepKey rolloverStepKey = new StepKey(phase, NAME, RolloverStep.NAME);
         StepKey updateDateStepKey = new StepKey(phase, NAME, UpdateRolloverLifecycleDateStep.NAME);
+        StepKey setIndexingCompleteStepKey = new StepKey(phase, NAME, INDEXING_COMPLETE_STEP_NAME);
 
         WaitForRolloverReadyStep waitForRolloverReadyStep = new WaitForRolloverReadyStep(waitForRolloverReadyStepKey, rolloverStepKey,
             client, maxSize, maxAge, maxDocs);
         RolloverStep rolloverStep = new RolloverStep(rolloverStepKey, updateDateStepKey, client);
-        UpdateRolloverLifecycleDateStep updateDateStep = new UpdateRolloverLifecycleDateStep(updateDateStepKey, nextStepKey);
-        return Arrays.asList(waitForRolloverReadyStep, rolloverStep, updateDateStep);
+        UpdateRolloverLifecycleDateStep updateDateStep = new UpdateRolloverLifecycleDateStep(updateDateStepKey, setIndexingCompleteStepKey,
+            System::currentTimeMillis);
+        UpdateSettingsStep setIndexingCompleteStep = new UpdateSettingsStep(setIndexingCompleteStepKey, nextStepKey,
+            client, indexingComplete);
+        return Arrays.asList(waitForRolloverReadyStep, rolloverStep, updateDateStep, setIndexingCompleteStep);
     }
 
     @Override
     public List<StepKey> toStepKeys(String phase) {
         StepKey rolloverReadyStepKey = new StepKey(phase, NAME, WaitForRolloverReadyStep.NAME);
         StepKey rolloverStepKey = new StepKey(phase, NAME, RolloverStep.NAME);
         StepKey updateDateStepKey = new StepKey(phase, NAME, UpdateRolloverLifecycleDateStep.NAME);
-        return Arrays.asList(rolloverReadyStepKey, rolloverStepKey, updateDateStepKey);
+        StepKey setIndexingCompleteStepKey = new StepKey(phase, NAME, INDEXING_COMPLETE_STEP_NAME);
+        return Arrays.asList(rolloverReadyStepKey, rolloverStepKey, updateDateStepKey, setIndexingCompleteStepKey);
     }
 
     @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverStep.java

@@ -5,6 +5,8 @@
  */
 package org.elasticsearch.xpack.core.indexlifecycle;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
 import org.elasticsearch.client.Client;
@@ -19,6 +21,8 @@
  * Unconditionally rolls over an index using the Rollover API.
  */
 public class RolloverStep extends AsyncActionStep {
+    private static final Logger logger = LogManager.getLogger(RolloverStep.class);
+
     public static final String NAME = "attempt-rollover";
 
     public RolloverStep(StepKey key, StepKey nextStepKey, Client client) {
@@ -27,6 +31,13 @@ public RolloverStep(StepKey key, StepKey nextStepKey, Client client) {
 
     @Override
     public void performAction(IndexMetaData indexMetaData, ClusterState currentClusterState, Listener listener) {
+        boolean indexingComplete = LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE_SETTING.get(indexMetaData.getSettings());
+        if (indexingComplete) {
+            logger.trace(indexMetaData.getIndex() + " has lifecycle complete set, skipping " + RolloverStep.NAME);
+            listener.onResponse(true);
+            return;
+        }
+
         String rolloverAlias = RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING.get(indexMetaData.getSettings());
 
         if (Strings.isNullOrEmpty(rolloverAlias)) {

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStep.java

@@ -5,44 +5,65 @@
  */
 package org.elasticsearch.xpack.core.indexlifecycle;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.admin.indices.rollover.RolloverInfo;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.index.Index;
 
+import java.util.function.LongSupplier;
+
 import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
 
 /**
  * Copies the lifecycle reference date to a new index created by rolling over an alias.
  * Used so that the "age" of an index doesn't get reset on rollover.
  */
 public class UpdateRolloverLifecycleDateStep extends ClusterStateActionStep {
+    private static final Logger logger = LogManager.getLogger(UpdateRolloverLifecycleDateStep.class);
     public static final String NAME = "update-rollover-lifecycle-date";
 
-    public UpdateRolloverLifecycleDateStep(StepKey key, StepKey nextStepKey) {
+    private final LongSupplier fallbackTimeSupplier;
+
+    public UpdateRolloverLifecycleDateStep(StepKey key, StepKey nextStepKey, LongSupplier fallbackTimeSupplier) {
         super(key, nextStepKey);
+        this.fallbackTimeSupplier = fallbackTimeSupplier;
     }
 
     @Override
     public ClusterState performAction(Index index, ClusterState currentState) {
         IndexMetaData indexMetaData = currentState.metaData().getIndexSafe(index);
-        // find the newly created index from the rollover and fetch its index.creation_date
-        String rolloverAlias = RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING.get(indexMetaData.getSettings());
-        if (Strings.isNullOrEmpty(rolloverAlias)) {
-            throw new IllegalStateException("setting [" + RolloverAction.LIFECYCLE_ROLLOVER_ALIAS
-                + "] is not set on index [" + indexMetaData.getIndex().getName() + "]");
-        }
-        RolloverInfo rolloverInfo = indexMetaData.getRolloverInfos().get(rolloverAlias);
-        if (rolloverInfo == null) {
-            throw new IllegalStateException("no rollover info found for [" + indexMetaData.getIndex().getName() + "] with alias [" +
-                rolloverAlias + "], the index has not yet rolled over with that alias");
+
+        long newIndexTime;
+
+        boolean indexingComplete = LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE_SETTING.get(indexMetaData.getSettings());
+        if (indexingComplete) {
+            logger.trace(indexMetaData.getIndex() + " has lifecycle complete set, skipping " + UpdateRolloverLifecycleDateStep.NAME);
+
+            // The index won't have RolloverInfo if this is a Following index and indexing_complete was set by CCR,
+            // so just use the current time.
+            newIndexTime = fallbackTimeSupplier.getAsLong();
+        } else {
+            // find the newly created index from the rollover and fetch its index.creation_date
+            String rolloverAlias = RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING.get(indexMetaData.getSettings());
+            if (Strings.isNullOrEmpty(rolloverAlias)) {
+                throw new IllegalStateException("setting [" + RolloverAction.LIFECYCLE_ROLLOVER_ALIAS
+                    + "] is not set on index [" + indexMetaData.getIndex().getName() + "]");
+            }
+            RolloverInfo rolloverInfo = indexMetaData.getRolloverInfos().get(rolloverAlias);
+            if (rolloverInfo == null) {
+                throw new IllegalStateException("no rollover info found for [" + indexMetaData.getIndex().getName() + "] with alias [" +
+                    rolloverAlias + "], the index has not yet rolled over with that alias");
+            }
+            newIndexTime = rolloverInfo.getTime();
         }
 
         LifecycleExecutionState.Builder newLifecycleState = LifecycleExecutionState
             .builder(LifecycleExecutionState.fromIndexMetadata(indexMetaData));
-        newLifecycleState.setIndexCreationDate(rolloverInfo.getTime());
+        newLifecycleState.setIndexCreationDate(newIndexTime);
 
         IndexMetaData.Builder newIndexMetadata = IndexMetaData.builder(indexMetaData);
         newIndexMetadata.putCustom(ILM_CUSTOM_METADATA_KEY, newLifecycleState.build().asMap());

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForRolloverReadyStep.java

@@ -6,6 +6,8 @@
 
 package org.elasticsearch.xpack.core.indexlifecycle;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
 import org.elasticsearch.client.Client;
@@ -24,6 +26,7 @@
  * Waits for at least one rollover condition to be satisfied, using the Rollover API's dry_run option.
  */
 public class WaitForRolloverReadyStep extends AsyncWaitStep {
+    private static final Logger logger = LogManager.getLogger(WaitForRolloverReadyStep.class);
 
     public static final String NAME = "check-rollover-ready";
 
@@ -57,6 +60,21 @@ public void evaluateCondition(IndexMetaData indexMetaData, Listener listener) {
             return;
         }
 
+        boolean indexingComplete = LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE_SETTING.get(indexMetaData.getSettings());
+        if (indexingComplete) {
+            logger.trace(indexMetaData.getIndex() + " has lifecycle complete set, skipping " + WaitForRolloverReadyStep.NAME);
+            Boolean isWriteIndex = indexMetaData.getAliases().get(rolloverAlias).writeIndex();
+            if (Boolean.TRUE.equals(isWriteIndex)) {
+                listener.onFailure(new IllegalStateException(String.format(Locale.ROOT,
+                    "index [%s] has [%s] set to [true], but is still the write index for alias [%s]",
+                    indexMetaData.getIndex().getName(), LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, rolloverAlias)));
+                return;
+            }
+
+            listener.onResponse(true, new WaitForRolloverReadyStep.EmptyInfo());
+            return;
+        }
+
         RolloverRequest rolloverRequest = new RolloverRequest(rolloverAlias, null);
         rolloverRequest.dryRun(true);
         if (maxAge != null) {

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverActionTests.java

@@ -80,21 +80,25 @@ public void testToSteps() {
                 randomAlphaOfLengthBetween(1, 10));
         List<Step> steps = action.toSteps(null, phase, nextStepKey);
         assertNotNull(steps);
-        assertEquals(3, steps.size());
+        assertEquals(4, steps.size());
         StepKey expectedFirstStepKey = new StepKey(phase, RolloverAction.NAME, WaitForRolloverReadyStep.NAME);
         StepKey expectedSecondStepKey = new StepKey(phase, RolloverAction.NAME, RolloverStep.NAME);
         StepKey expectedThirdStepKey = new StepKey(phase, RolloverAction.NAME, UpdateRolloverLifecycleDateStep.NAME);
+        StepKey expectedFourthStepKey = new StepKey(phase, RolloverAction.NAME, RolloverAction.INDEXING_COMPLETE_STEP_NAME);
         WaitForRolloverReadyStep firstStep = (WaitForRolloverReadyStep) steps.get(0);
         RolloverStep secondStep = (RolloverStep) steps.get(1);
         UpdateRolloverLifecycleDateStep thirdStep = (UpdateRolloverLifecycleDateStep) steps.get(2);
+        UpdateSettingsStep fourthStep = (UpdateSettingsStep) steps.get(3);
         assertEquals(expectedFirstStepKey, firstStep.getKey());
         assertEquals(expectedSecondStepKey, secondStep.getKey());
         assertEquals(expectedThirdStepKey, thirdStep.getKey());
+        assertEquals(expectedFourthStepKey, fourthStep.getKey());
         assertEquals(secondStep.getKey(), firstStep.getNextStepKey());
         assertEquals(thirdStep.getKey(), secondStep.getNextStepKey());
+        assertEquals(fourthStep.getKey(), thirdStep.getNextStepKey());
         assertEquals(action.getMaxSize(), firstStep.getMaxSize());
         assertEquals(action.getMaxAge(), firstStep.getMaxAge());
         assertEquals(action.getMaxDocs(), firstStep.getMaxDocs());
-        assertEquals(nextStepKey, thirdStep.getNextStepKey());
+        assertEquals(nextStepKey, fourthStep.getNextStepKey());
     }
 }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverStepTests.java

@@ -126,6 +126,34 @@ public void onFailure(Exception e) {
         Mockito.verify(indicesClient, Mockito.only()).rolloverIndex(Mockito.any(), Mockito.any());
     }
 
+    public void testPerformActionWithIndexingComplete() {
+        String alias = randomAlphaOfLength(5);
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10))
+            .putAlias(AliasMetaData.builder(alias))
+            .settings(settings(Version.CURRENT)
+                .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias)
+                .put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, true))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+
+        RolloverStep step = createRandomInstance();
+
+        SetOnce<Boolean> actionCompleted = new SetOnce<>();
+        step.performAction(indexMetaData, null, new AsyncActionStep.Listener() {
+
+            @Override
+            public void onResponse(boolean complete) {
+                actionCompleted.set(complete);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError("Unexpected method call", e);
+            }
+        });
+
+        assertEquals(true, actionCompleted.get());
+    }
+
     public void testPerformActionFailure() {
         String alias = randomAlphaOfLength(5);
         IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10))

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStepTests.java

@@ -16,16 +16,21 @@
 import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
 
 import java.util.Collections;
+import java.util.function.LongSupplier;
 
 import static org.hamcrest.Matchers.equalTo;
 
 public class UpdateRolloverLifecycleDateStepTests extends AbstractStepTestCase<UpdateRolloverLifecycleDateStep> {
 
     @Override
     public UpdateRolloverLifecycleDateStep createRandomInstance() {
+        return createRandomInstanceWithFallbackTime(null);
+    }
+
+    public UpdateRolloverLifecycleDateStep createRandomInstanceWithFallbackTime(LongSupplier fallbackTimeSupplier) {
         StepKey stepKey = randomStepKey();
         StepKey nextStepKey = randomStepKey();
-        return new UpdateRolloverLifecycleDateStep(stepKey, nextStepKey);
+        return new UpdateRolloverLifecycleDateStep(stepKey, nextStepKey, fallbackTimeSupplier);
     }
 
     @Override
@@ -39,12 +44,12 @@ public UpdateRolloverLifecycleDateStep mutateInstance(UpdateRolloverLifecycleDat
             nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
         }
 
-        return new UpdateRolloverLifecycleDateStep(key, nextKey);
+        return new UpdateRolloverLifecycleDateStep(key, nextKey, null);
     }
 
     @Override
     public UpdateRolloverLifecycleDateStep copyInstance(UpdateRolloverLifecycleDateStep instance) {
-        return new UpdateRolloverLifecycleDateStep(instance.getKey(), instance.getNextStepKey());
+        return new UpdateRolloverLifecycleDateStep(instance.getKey(), instance.getNextStepKey(), null);
     }
 
     @SuppressWarnings("unchecked")
@@ -105,4 +110,26 @@ public void testPerformActionWithNoRolloverAliasSetting() {
         assertThat(exceptionThrown.getMessage(),
             equalTo("setting [index.lifecycle.rollover_alias] is not set on index [" + indexMetaData.getIndex().getName() +"]"));
     }
+
+    public void testPerformActionWithIndexingComplete() {
+        String alias = randomAlphaOfLength(3);
+        long creationDate = randomLongBetween(0, 1000000);
+        long rolloverTime = randomValueOtherThan(creationDate, () -> randomNonNegativeLong());
+
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10))
+            .settings(settings(Version.CURRENT)
+                .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias)
+                .put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, true))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metaData(MetaData.builder()
+                .put(indexMetaData, false)).build();
+
+        UpdateRolloverLifecycleDateStep step = createRandomInstanceWithFallbackTime(() -> rolloverTime);
+        ClusterState newState = step.performAction(indexMetaData.getIndex(), clusterState);
+        long actualRolloverTime = LifecycleExecutionState
+            .fromIndexMetadata(newState.metaData().index(indexMetaData.getIndex()))
+            .getLifecycleDate();
+        assertThat(actualRolloverTime, equalTo(rolloverTime));
+    }
 }