system models, CONVENTION: system is an account than ends with an und…

…erscore

docker-compose.yml

@@ -27,6 +27,8 @@ services:
     networks:
       - esnet
 
+  # use Kibana to visualize and debug data
+  # http://localhost:5601
   kibana:
     image: docker.elastic.co/kibana/kibana-oss:6.5.2
     container_name: kibana
@@ -40,6 +42,8 @@ services:
     ports:
       - "5601:5601"
 
+  # use Cerebro to visualize and debug Elasticsearch
+  # http://localhost:9900/#/overview?host=http:%2F%2Felasticsearch:9200
   cerebro:
     image: yannart/cerebro:0.8.1
     container_name: cerebro

model.go

@@ -11,7 +11,11 @@
 
 package tm
 
-import "github.com/txn2/es"
+import (
+	"strings"
+
+	"github.com/txn2/es"
+)
 
 const IdxModel = "models"
 
@@ -89,22 +93,33 @@ func fieldProps(fields []Model) map[string]interface{} {
 func MakeModelTemplateMapping(account string, model *Model) es.IndexTemplate {
 
 	name := account + "-data-" + model.MachineName
+	idxPattern := account + "-data-" + model.MachineName + "-*"
+
+	// CONVENTION: if the account ends in an underscore "_" then
+	// it is a system model (SYSTEM_IdxModel)
+	if strings.HasSuffix(account, "_") {
+		name = account + "system-data-" + model.MachineName
+		idxPattern = "*-data-" + model.MachineName + "-*"
+	}
 
 	payloadProps := fieldProps(model.Fields)
 
 	template := es.Obj{
-		"index_patterns": []string{account + "-data-" + model.MachineName + "-*"},
+		"index_patterns": []string{idxPattern},
 		"settings": es.Obj{
 			"index": es.Obj{
-				"number_of_shards": 3,
+				"number_of_shards": 1, // @TODO allow this to be configured
 			},
 		},
 		"mappings": es.Obj{
-			"doc": es.Obj{ // _doc is the standard until deprecated, logstash uses "doc"
+			// _doc is the standard until deprecated, logstash uses "doc"
+			// messages come into elasticsearch via txn2/rxtx->txn2/rtBeat->logstash
+			"doc": es.Obj{
 				"_source": es.Obj{
 					"enabled": true,
 				},
 				"properties": es.Obj{
+					// txn2/rtbeat sends txn2/rxtx messages as rxtxMsg
 					"rxtxMsg": es.Obj{
 						"properties": es.Obj{
 							"seq":      es.Obj{"type": "long"},
@@ -173,10 +188,10 @@ func GetModelsTemplateMapping() es.IndexTemplate {
 	}
 
 	template := es.Obj{
-		"index_patterns": []string{"*-" + IdxModel},
+		"index_patterns": []string{"*-" + IdxModel, "*_" + IdxModel},
 		"settings": es.Obj{
 			"index": es.Obj{
-				"number_of_shards": 2,
+				"number_of_shards": 1,
 			},
 		},
 		"mappings": es.Obj{

tm.go

@@ -93,7 +93,15 @@ func NewApi(cfg *Config) (*Api, error) {
 // GetModel
 func (a *Api) GetModel(account string, id string) (int, *ModelResult, error) {
 
-	code, ret, err := a.Elastic.Get(fmt.Sprintf("%s-%s/_doc/%s", account, IdxModel, id))
+	locFmt := "%s-%s/_doc/%s"
+
+	// CONVENTION: if the account ends in an underscore "_" then
+	// it is a system model (SYSTEM_IdxModel)
+	if strings.HasSuffix(account, "_") {
+		locFmt = "%s%s/_doc/%s"
+	}
+
+	code, ret, err := a.Elastic.Get(fmt.Sprintf(locFmt, account, IdxModel, id))
 	if err != nil {
 		a.Logger.Error("EsError", zap.Error(err))
 		return code, nil, err
@@ -145,7 +153,15 @@ func (a *Api) UpsertModel(account string, model *Model) (int, es.Result, error)
 		return code, templateMappingResult, err
 	}
 
-	return a.Elastic.PutObj(fmt.Sprintf("%s-%s/_doc/%s", account, IdxModel, model.MachineName), model)
+	locFmt := "%s-%s/_doc/%s"
+
+	// CONVENTION: if the account ends in an underscore "_" then
+	// it is a system model (SYSTEM_IdxModel)
+	if strings.HasSuffix(account, "_") {
+		locFmt = "%s%s/_doc/%s"
+	}
+
+	return a.Elastic.PutObj(fmt.Sprintf(locFmt, account, IdxModel, model.MachineName), model)
 }
 
 // UpsertModelHandler