Template creation using context (#14811)

* Template creation using context Signed-off-by: Mohit Godwani <mgodwan@amazon.com>
opensearch-project · Jul 24, 2024 · c76bfeb · c76bfeb
1 parent 1fe58b5
commit c76bfeb
Show file tree

Hide file tree

Showing 13 changed files with 862 additions and 43 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -31,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Create listener to refresh search thread resource usage ([#14832](https://github.com/opensearch-project/OpenSearch/pull/14832))
 - Add rest, transport layer changes for hot to warm tiering - dedicated setup (([#13980](https://github.com/opensearch-project/OpenSearch/pull/13980))
 - Optimize Cluster Stats Indices to precomute node level stats ([#14426](https://github.com/opensearch-project/OpenSearch/pull/14426))
+- Add logic to create index templates (v2) using context field ([#14811](https://github.com/opensearch-project/OpenSearch/pull/14811))
 
 ### Dependencies
 - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442))

diff --git a/...g/opensearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/...g/opensearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java
@@ -140,7 +140,8 @@ protected void clusterManagerOperation(
             MetadataIndexTemplateService.validateV2TemplateRequest(
                 state.metadata(),
                 simulateTemplateToAdd,
-                request.getIndexTemplateRequest().indexTemplate()
+                request.getIndexTemplateRequest().indexTemplate(),
+                clusterService.getClusterSettings()
             );
             stateWithTemplate = indexTemplateService.addIndexTemplateV2(
                 state,

diff --git a/...va/org/opensearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java b/...va/org/opensearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java
@@ -134,7 +134,8 @@ protected void clusterManagerOperation(
             MetadataIndexTemplateService.validateV2TemplateRequest(
                 state.metadata(),
                 simulateTemplateToAdd,
-                request.getIndexTemplateRequest().indexTemplate()
+                request.getIndexTemplateRequest().indexTemplate(),
+                clusterService.getClusterSettings()
             );
             stateWithTemplate = indexTemplateService.addIndexTemplateV2(
                 state,

diff --git a/server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplateMetadata.java b/server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplateMetadata.java
@@ -10,6 +10,8 @@
 
 import org.opensearch.common.annotation.ExperimentalApi;
 
+import java.util.Objects;
+
 /**
  * Metadata information about a template available in a template repository.
  */
@@ -48,13 +50,14 @@ public long version() {
      * @return Metadata object based on name
      */
     public static SystemTemplateMetadata fromComponentTemplate(String fullyQualifiedName) {
-        assert fullyQualifiedName.length() > 1 : "System template name must have at least one component";
-        assert fullyQualifiedName.substring(1, fullyQualifiedName.indexOf(DELIMITER, 1)).equals(COMPONENT_TEMPLATE_TYPE);
+        assert fullyQualifiedName.length() > DELIMITER.length() * 3 + 2 + COMPONENT_TEMPLATE_TYPE.length()
+            : "System template name must have all defined components";
+        assert (DELIMITER + fullyQualifiedName.substring(1, fullyQualifiedName.indexOf(DELIMITER, 1))).equals(COMPONENT_TEMPLATE_TYPE);
 
         return new SystemTemplateMetadata(
-            Long.parseLong(fullyQualifiedName.substring(fullyQualifiedName.lastIndexOf(DELIMITER))),
+            Long.parseLong(fullyQualifiedName.substring(fullyQualifiedName.lastIndexOf(DELIMITER) + 1)),
             COMPONENT_TEMPLATE_TYPE,
-            fullyQualifiedName.substring(0, fullyQualifiedName.lastIndexOf(DELIMITER))
+            fullyQualifiedName.substring(fullyQualifiedName.indexOf(DELIMITER, 2) + 1, fullyQualifiedName.lastIndexOf(DELIMITER))
         );
     }
 
@@ -65,4 +68,22 @@ public static SystemTemplateMetadata fromComponentTemplateInfo(String name, long
     public final String fullyQualifiedName() {
         return type + DELIMITER + name + DELIMITER + version;
     }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        SystemTemplateMetadata that = (SystemTemplateMetadata) o;
+        return version == that.version && Objects.equals(type, that.type) && Objects.equals(name, that.name);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(version, type, name);
+    }
+
+    @Override
+    public String toString() {
+        return "SystemTemplateMetadata{" + "version=" + version + ", type='" + type + '\'' + ", name='" + name + '\'' + '}';
+    }
 }
diff --git a/server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplatesService.java b/server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplatesService.java
@@ -85,7 +85,7 @@ void refreshTemplates(boolean verification) {
         int failedLoadingRepositories = 0;
         List<Exception> exceptions = new ArrayList<>();
 
-        if (loaded.compareAndSet(false, true) && enabledTemplates) {
+        if ((verification || loaded.compareAndSet(false, true)) && enabledTemplates) {
             for (SystemTemplatesPlugin plugin : systemTemplatesPluginList) {
                 try (SystemTemplateRepository repository = plugin.loadRepository()) {
 

diff --git a/...src/main/java/org/opensearch/cluster/applicationtemplates/TemplateRepositoryMetadata.java b/...src/main/java/org/opensearch/cluster/applicationtemplates/TemplateRepositoryMetadata.java
@@ -10,6 +10,8 @@
 
 import org.opensearch.common.annotation.ExperimentalApi;
 
+import java.util.Objects;
+
 /**
  * The information to uniquely identify a template repository.
  */
@@ -31,4 +33,22 @@ public String id() {
     public long version() {
         return version;
     }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        TemplateRepositoryMetadata that = (TemplateRepositoryMetadata) o;
+        return version == that.version && Objects.equals(id, that.id);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, version);
+    }
+
+    @Override
+    public String toString() {
+        return "TemplateRepositoryMetadata{" + "id='" + id + '\'' + ", version=" + version + '}';
+    }
 }
diff --git a/server/src/main/java/org/opensearch/cluster/coordination/OpenSearchNodeCommand.java b/server/src/main/java/org/opensearch/cluster/coordination/OpenSearchNodeCommand.java
@@ -47,6 +47,7 @@
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.Diff;
+import org.opensearch.cluster.metadata.ComponentTemplateMetadata;
 import org.opensearch.cluster.metadata.DataStreamMetadata;
 import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.common.collect.Tuple;
@@ -94,9 +95,10 @@ public abstract class OpenSearchNodeCommand extends EnvironmentAwareCommand {
         public <T, C> T parseNamedObject(Class<T> categoryClass, String name, XContentParser parser, C context) throws IOException {
             // Currently, two unknown top-level objects are present
             if (Metadata.Custom.class.isAssignableFrom(categoryClass)) {
-                if (DataStreamMetadata.TYPE.equals(name)) {
+                if (DataStreamMetadata.TYPE.equals(name) || ComponentTemplateMetadata.TYPE.equals(name)) {
                     // DataStreamMetadata is used inside Metadata class for validation purposes and building the indicesLookup,
-                    // therefor even es node commands need to be able to parse it.
+                    // ComponentTemplateMetadata is used inside Metadata class for building the systemTemplatesLookup,
+                    // therefor even OpenSearch node commands need to be able to parse it.
                     return super.parseNamedObject(categoryClass, name, parser, context);
                     // TODO: Try to parse other named objects (e.g. stored scripts, ingest pipelines) that are part of core es as well?
                     // Note that supporting PersistentTasksCustomMetadata is trickier, because PersistentTaskParams is a named object too.

diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/opensearch/cluster/metadata/ComposableIndexTemplate.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.cluster.metadata;
 
+import org.opensearch.Version;
 import org.opensearch.cluster.AbstractDiffable;
 import org.opensearch.cluster.Diff;
 import org.opensearch.cluster.metadata.DataStream.TimestampField;
@@ -75,6 +76,7 @@ public class ComposableIndexTemplate extends AbstractDiffable<ComposableIndexTem
     private static final ParseField VERSION = new ParseField("version");
     private static final ParseField METADATA = new ParseField("_meta");
     private static final ParseField DATA_STREAM = new ParseField("data_stream");
+    private static final ParseField CONTEXT = new ParseField("context");
 
     @SuppressWarnings("unchecked")
     public static final ConstructingObjectParser<ComposableIndexTemplate, Void> PARSER = new ConstructingObjectParser<>(
@@ -87,7 +89,8 @@ public class ComposableIndexTemplate extends AbstractDiffable<ComposableIndexTem
             (Long) a[3],
             (Long) a[4],
             (Map<String, Object>) a[5],
-            (DataStreamTemplate) a[6]
+            (DataStreamTemplate) a[6],
+            (Context) a[7]
         )
     );
 
@@ -99,6 +102,7 @@ public class ComposableIndexTemplate extends AbstractDiffable<ComposableIndexTem
         PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), VERSION);
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA);
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), DataStreamTemplate.PARSER, DATA_STREAM);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), Context.PARSER, CONTEXT);
     }
 
     private final List<String> indexPatterns;
@@ -114,6 +118,8 @@ public class ComposableIndexTemplate extends AbstractDiffable<ComposableIndexTem
     private final Map<String, Object> metadata;
     @Nullable
     private final DataStreamTemplate dataStreamTemplate;
+    @Nullable
+    private final Context context;
 
     static Diff<ComposableIndexTemplate> readITV2DiffFrom(StreamInput in) throws IOException {
         return AbstractDiffable.readDiffFrom(ComposableIndexTemplate::new, in);
@@ -131,7 +137,7 @@ public ComposableIndexTemplate(
         @Nullable Long version,
         @Nullable Map<String, Object> metadata
     ) {
-        this(indexPatterns, template, componentTemplates, priority, version, metadata, null);
+        this(indexPatterns, template, componentTemplates, priority, version, metadata, null, null);
     }
 
     public ComposableIndexTemplate(
@@ -142,6 +148,19 @@ public ComposableIndexTemplate(
         @Nullable Long version,
         @Nullable Map<String, Object> metadata,
         @Nullable DataStreamTemplate dataStreamTemplate
+    ) {
+        this(indexPatterns, template, componentTemplates, priority, version, metadata, dataStreamTemplate, null);
+    }
+
+    public ComposableIndexTemplate(
+        List<String> indexPatterns,
+        @Nullable Template template,
+        @Nullable List<String> componentTemplates,
+        @Nullable Long priority,
+        @Nullable Long version,
+        @Nullable Map<String, Object> metadata,
+        @Nullable DataStreamTemplate dataStreamTemplate,
+        @Nullable Context context
     ) {
         this.indexPatterns = indexPatterns;
         this.template = template;
@@ -150,6 +169,7 @@ public ComposableIndexTemplate(
         this.version = version;
         this.metadata = metadata;
         this.dataStreamTemplate = dataStreamTemplate;
+        this.context = context;
     }
 
     public ComposableIndexTemplate(StreamInput in) throws IOException {
@@ -164,6 +184,11 @@ public ComposableIndexTemplate(StreamInput in) throws IOException {
         this.version = in.readOptionalVLong();
         this.metadata = in.readMap();
         this.dataStreamTemplate = in.readOptionalWriteable(DataStreamTemplate::new);
+        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+            this.context = in.readOptionalWriteable(Context::new);
+        } else {
+            this.context = null;
+        }
     }
 
     public List<String> indexPatterns() {
@@ -205,6 +230,10 @@ public DataStreamTemplate getDataStreamTemplate() {
         return dataStreamTemplate;
     }
 
+    public Context context() {
+        return context;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeStringCollection(this.indexPatterns);
@@ -219,6 +248,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalVLong(this.version);
         out.writeMap(this.metadata);
         out.writeOptionalWriteable(dataStreamTemplate);
+        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+            out.writeOptionalWriteable(context);
+        }
     }
 
     @Override
@@ -243,6 +275,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (this.dataStreamTemplate != null) {
             builder.field(DATA_STREAM.getPreferredName(), dataStreamTemplate);
         }
+        if (this.context != null) {
+            builder.field(CONTEXT.getPreferredName(), context);
+        }
         builder.endObject();
         return builder;
     }
@@ -256,7 +291,8 @@ public int hashCode() {
             this.priority,
             this.version,
             this.metadata,
-            this.dataStreamTemplate
+            this.dataStreamTemplate,
+            this.context
         );
     }
 
@@ -275,7 +311,8 @@ public boolean equals(Object obj) {
             && Objects.equals(this.priority, other.priority)
             && Objects.equals(this.version, other.version)
             && Objects.equals(this.metadata, other.metadata)
-            && Objects.equals(this.dataStreamTemplate, other.dataStreamTemplate);
+            && Objects.equals(this.dataStreamTemplate, other.dataStreamTemplate)
+            && Objects.equals(this.context, other.context);
     }
 
     @Override