Add index_types for OTEL logs and metrics opensearch-project#3148

Signed-off-by: Jürgen Walter <juergen.walter@sap.com>
juergen-walter · Jan 9, 2024 · 3a0ca1f · 3a0ca1f
1 parent 774fa21
commit 3a0ca1f
Show file tree

Hide file tree

Showing 4 changed files with 53 additions and 1 deletion.
diff --git a/...ain/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java b/...ain/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java
@@ -401,6 +401,10 @@ private Map<String, Object> readIndexTemplate(final String templateFile, final I
                 templateURL = loadExistingTemplate(templateType, IndexConstants.RAW_DEFAULT_TEMPLATE_FILE);
             } else if (indexType.equals(IndexType.TRACE_ANALYTICS_SERVICE_MAP)) {
                 templateURL = loadExistingTemplate(templateType, IndexConstants.SERVICE_MAP_DEFAULT_TEMPLATE_FILE);
+            } else if (indexType.equals(IndexType.LOG_ANALYTICS)) {
+                templateURL = loadExistingTemplate(templateType, IndexConstants.LOGS_DEFAULT_TEMPLATE_FILE);
+            } else if (indexType.equals(IndexType.METRIC_ANALYTICS)) {
+                templateURL = loadExistingTemplate(templateType, IndexConstants.METRICS_DEFAULT_TEMPLATE_FILE);
             } else if (templateFile != null) {
                 if (templateFile.toLowerCase().startsWith(S3_PREFIX)) {
                     FileReader s3FileReader = new S3FileReader(s3Client);

diff --git a/...rc/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConstants.java b/...rc/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConstants.java
@@ -26,7 +26,9 @@ public class IndexConstants {
 
   static {
     // TODO: extract out version number into version enum
-    TYPE_TO_DEFAULT_ALIAS.put(IndexType.TRACE_ANALYTICS_RAW, "otel-v1-apm-span");
     TYPE_TO_DEFAULT_ALIAS.put(IndexType.TRACE_ANALYTICS_SERVICE_MAP, "otel-v1-apm-service-map");
+    TYPE_TO_DEFAULT_ALIAS.put(IndexType.TRACE_ANALYTICS_RAW, "otel-v1-apm-span");
+    TYPE_TO_DEFAULT_ALIAS.put(IndexType.LOG_ANALYTICS, "logs-otel-v1");
+    TYPE_TO_DEFAULT_ALIAS.put(IndexType.METRIC_ANALYTICS, "metrics-otel-v1");
   }
 }
diff --git a/...in/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexManagerFactory.java b/...in/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexManagerFactory.java
@@ -58,6 +58,14 @@ public final IndexManager getIndexManager(final IndexType indexType,
                 indexManager = new TraceAnalyticsServiceMapIndexManager(
                         restHighLevelClient, openSearchClient, openSearchSinkConfiguration, clusterSettingsParser, templateStrategy, indexAlias);
                 break;
+            case LOG_ANALYTICS:
+                indexManager = new LogAnalyticsIndexManager(
+                        restHighLevelClient, openSearchClient, openSearchSinkConfiguration, clusterSettingsParser, templateStrategy, indexAlias);
+                break;
+            case METRIC_ANALYTICS:
+                indexManager = new MetricAnalyticsIndexManager(
+                        restHighLevelClient, openSearchClient, openSearchSinkConfiguration, clusterSettingsParser, templateStrategy, indexAlias);
+                break;
             case MANAGEMENT_DISABLED:
                 indexManager = new ManagementDisabledIndexManager(
                         restHighLevelClient, openSearchClient, openSearchSinkConfiguration, clusterSettingsParser, templateStrategy, indexAlias);
@@ -141,6 +149,42 @@ public TraceAnalyticsServiceMapIndexManager(final RestHighLevelClient restHighLe
         }
     }
 
+    private static class LogAnalyticsIndexManager extends AbstractIndexManager {
+
+        public LogsIndexManager(final RestHighLevelClient restHighLevelClient,
+                                                    final OpenSearchClient openSearchClient,
+                                                    final OpenSearchSinkConfiguration openSearchSinkConfiguration,
+                                                    final ClusterSettingsParser clusterSettingsParser,
+                                                    final TemplateStrategy templateStrategy,
+                                                    final String indexAlias) {
+            super(restHighLevelClient, openSearchClient, openSearchSinkConfiguration, clusterSettingsParser, templateStrategy, indexAlias);
+            this.ismPolicyManagementStrategy = new IsmPolicyManagement(
+                    openSearchClient,
+                    restHighLevelClient,
+                    IndexConstants.RAW_ISM_POLICY,
+                    IndexConstants.RAW_ISM_FILE_WITH_ISM_TEMPLATE,
+                    IndexConstants.RAW_ISM_FILE_NO_ISM_TEMPLATE);
+        }
+    }
+
+    private static class MetricAnalyticsIndexManager extends AbstractIndexManager {
+
+        public MetricsIndexManager(final RestHighLevelClient restHighLevelClient,
+                                                    final OpenSearchClient openSearchClient,
+                                                    final OpenSearchSinkConfiguration openSearchSinkConfiguration,
+                                                    final ClusterSettingsParser clusterSettingsParser,
+                                                    final TemplateStrategy templateStrategy,
+                                                    final String indexAlias) {
+            super(restHighLevelClient, openSearchClient, openSearchSinkConfiguration, clusterSettingsParser, templateStrategy, indexAlias);
+            this.ismPolicyManagementStrategy = new IsmPolicyManagement(
+                    openSearchClient,
+                    restHighLevelClient,
+                    IndexConstants.RAW_ISM_POLICY,
+                    IndexConstants.RAW_ISM_FILE_WITH_ISM_TEMPLATE,
+                    IndexConstants.RAW_ISM_FILE_NO_ISM_TEMPLATE);
+        }
+    }
+
     private class ManagementDisabledIndexManager extends AbstractIndexManager {
         protected ManagementDisabledIndexManager(final RestHighLevelClient restHighLevelClient,
                                                  final OpenSearchClient openSearchClient,

diff --git a/...rch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexType.java b/...rch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexType.java
@@ -14,6 +14,8 @@
 public enum IndexType {
     TRACE_ANALYTICS_RAW("trace-analytics-raw"),
     TRACE_ANALYTICS_SERVICE_MAP("trace-analytics-service-map"),
+    LOG_ANALYTICS("log-analytics"),
+    METRIC_ANALYTICS("metric-analytics"),
     CUSTOM("custom"),
     MANAGEMENT_DISABLED("management_disabled");