Canonicalized big5 mapping files.

Signed-off-by: Govind Kamat <govkamat@amazon.com>
vpehkone · Mar 4, 2024 · 63cb6b2 · 63cb6b2
1 parent fffd061
commit 63cb6b2
Show file tree

Hide file tree

Showing 5 changed files with 354 additions and 168 deletions.
diff --git a/big5/README.md b/big5/README.md
@@ -51,6 +51,7 @@ This workload allows the following parameters to be specified using `--workload-
 * `document_uncompressed_size_in_bytes`: If specifying an alternate data corpus, the uncompressed size of the corpus.
 * `document_url`:  If specifying an alternate data corpus, the full path to the corpus file (optional).
 * `error_level` (default: "non-fatal"): Available for bulk operations only to specify ignore-response-error-level.
+* `index_body` (default: "index.json"): The name of the file containing the index settings and mappings.
 * `index_name` (default: "big5"): The name of the index the workload should create and use for its operations.
 * `index_settings`: A list of index settings. Index settings defined elsewhere (e.g. `number_of_replicas`) need to be overridden explicitly.
 * `ingest_percentage` (default: 100): A number between 0 and 100 that defines how much of the document corpus should be ingested.

diff --git a/big5/index.json.ORIG → big5/index-alt-1.json b/big5/index.json.ORIG → big5/index-alt-1.json
@@ -1,8 +1,12 @@
 {
   "settings": {
-    "index.number_of_shards": 1,
-    "index.number_of_replicas": 1,
-    "index.requests.cache.enable": false
+    "index.number_of_shards": {{number_of_shards | default(1)}},
+    "index.number_of_replicas": {{number_of_replicas | default(1)}},
+    "index.queries.cache.enabled": {{query_cache_enabled | default(false) | tojson}},
+    "index.requests.cache.enable": {{requests_cache_enabled | default(false) | tojson}},
+    "index.codec": "best_compression",
+    "index.translog.sync_interval": "30s",
+    "index.translog.durability": "async"
   },
   "mappings": {
     "_data_stream_timestamp": {

diff --git a/big5/index-alt-2.json b/big5/index-alt-2.json
@@ -0,0 +1,182 @@
+{
+  "settings": {
+    "index.number_of_shards": {{number_of_shards | default(1)}},
+    "index.number_of_replicas": {{number_of_replicas | default(1)}},
+    "index.queries.cache.enabled": {{query_cache_enabled | default(false) | tojson}},
+    "index.requests.cache.enable": {{requests_cache_enabled | default(false) | tojson}},
+    "index.codec": "best_compression",
+    "index.translog.sync_interval": "30s",
+    "index.translog.durability": "async",
+    "index.query.default_field": [ "message" ]
+  },
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "match_ip": {
+          "match": "ip",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "ip"
+          }
+        }
+      },
+      {
+        "match_message": {
+          "match": "message",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "match_only_text"
+          }
+        }
+      },
+      {
+        "strings_as_keyword": {
+          "match_mapping_type": "string",
+          "mapping": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          }
+        }
+      }
+    ],
+    "date_detection": false,
+    "properties": {
+      "@timestamp": {
+        "type": "date"
+      },
+      "agent": {
+        "properties": {
+          "ephemeral_id": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "id": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "name": {
+            "type": "keyword",
+            "time_series_dimension": true
+          },
+          "type": {
+            "type": "keyword",
+            "time_series_dimension": true
+          },
+          "version": {
+            "type": "keyword",
+            "ignore_above": 1024
+          }
+        }
+      },
+      "aws": {
+        "properties": {
+          "cloudwatch": {
+            "properties": {
+              "ingestion_time": {
+                "type": "keyword",
+                "ignore_above": 1024
+              },
+              "log_group": {
+                "type": "keyword",
+                "time_series_dimension": true
+              },
+              "log_stream": {
+                "type": "keyword",
+                "time_series_dimension": true
+              }
+            }
+          }
+        }
+      },
+      "cloud": {
+        "properties": {
+          "region": {
+            "type": "keyword",
+            "time_series_dimension": true
+          }
+        }
+      },
+      "ecs": {
+        "properties": {
+          "version": {
+            "type": "keyword",
+            "ignore_above": 1024
+          }
+        }
+      },
+      "event": {
+        "properties": {
+          "dataset": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "id": {
+            "type": "keyword",
+            "time_series_dimension": true
+          },
+          "ingested": {
+            "type": "date"
+          }
+        }
+      },
+      "host": {
+        "type": "object"
+      },
+      "input": {
+        "properties": {
+          "type": {
+            "type": "keyword",
+            "time_series_dimension": true
+          }
+        }
+      },
+      "log": {
+        "properties": {
+          "file": {
+            "properties": {
+              "path": {
+                "type": "keyword",
+                "ignore_above": 1024
+              }
+            }
+          }
+        }
+      },
+      "message": {
+        "type": "match_only_text"
+      },
+      "meta": {
+        "properties": {
+          "file": {
+            "type": "keyword",
+            "time_series_dimension": true
+          }
+        }
+      },
+      "metrics": {
+        "properties": {
+          "size": {
+            "type": "long",
+            "time_series_metric": "gauge"
+          },
+          "tmin": {
+            "type": "long",
+            "time_series_metric": "gauge"
+          }
+        }
+      },
+      "process": {
+        "properties": {
+          "name": {
+            "type": "keyword",
+            "time_series_dimension": true
+          }
+        }
+      },
+      "tags": {
+        "type": "keyword",
+        "ignore_above": 1024
+      }
+    }
+  }
+}