Extract and expose executed state

README.md

@@ -15,8 +15,12 @@ You just need to run the exporter on the same server than the Salt Master.
 ## Features
 
 Supported tags:
-* salt/job/<jid>/new
-* salt/job/<jid>/ret/<*>
+* `salt/job/<jid>/new`
+* `salt/job/<jid>/ret/<*>`
+
+It extracts and exposes:
+* the execution module, to `function` label
+* the states when using state.sls/state.apply/state.highstate, to `state` label
 
 ## Exposed metrics
 
@@ -70,7 +74,6 @@ It can be joined on function label to have details per executed module.
 
 ## Upcoming features
 
-* details per state when state.sls/state.apply is used
 * details per state module when state.single is used
 * metric regarding IPC connectivity
 * support the runners

internal/metrics/metrics.go

@@ -12,12 +12,12 @@ import (
 )
 
 func ExposeMetrics(ctx context.Context, eventChan chan events.SaltEvent) {
-	newJobsCounter := promauto.NewCounterVec(
+	newJobCounter := promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "salt_new_job_total",
 			Help: "Total number of new job processed",
 		},
-		[]string{"function", "success"},
+		[]string{"function", "state", "success"},
 	)
 	responsesCounter := promauto.NewCounterVec(
 		prometheus.CounterOpts{
@@ -31,21 +31,22 @@ func ExposeMetrics(ctx context.Context, eventChan chan events.SaltEvent) {
 			Name: "salt_function_responses_total",
 			Help: "Total number of response per function processed",
 		},
-		[]string{"function", "success"},
+		[]string{"function", "state", "success"},
 	)
+
 	scheduledJobReturnCounter := promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "salt_scheduled_job_return_total",
 			Help: "Total number of scheduled job response",
 		},
-		[]string{"function", "success"},
+		[]string{"function", "state", "success"},
 	)
 	expectedResponsesNumber := promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "salt_expected_responses_total",
 			Help: "Total number of expected minions responses",
 		},
-		[]string{"function"},
+		[]string{"function", "state"},
 	)
 
 	for {
@@ -55,14 +56,18 @@ func ExposeMetrics(ctx context.Context, eventChan chan events.SaltEvent) {
 			return
 		case event := <-eventChan:
 			start := time.Now()
+
 			switch event.Type {
 			case "new":
-				newJobsCounter.WithLabelValues(event.Data.Fun, strconv.FormatBool(event.Data.Success)).Inc()
-				expectedResponsesNumber.WithLabelValues(event.Data.Fun).Add(float64(event.TargetNumber))
+				state := event.ExtractState()
+				newJobCounter.WithLabelValues(event.Data.Fun, state, strconv.FormatBool(event.Data.Success)).Inc()
+				expectedResponsesNumber.WithLabelValues(event.Data.Fun, state).Add(float64(event.TargetNumber))
 			case "ret":
+				state := event.ExtractState()
 				if event.IsScheduleJob {
 					scheduledJobReturnCounter.WithLabelValues(
 						event.Data.Fun,
+						state,
 						strconv.FormatBool(event.Data.Success),
 					).Inc()
 				} else {
@@ -73,10 +78,12 @@ func ExposeMetrics(ctx context.Context, eventChan chan events.SaltEvent) {
 					).Inc()
 					functionResponsesCounter.WithLabelValues(
 						event.Data.Fun,
+						state,
 						sucess,
 					).Inc()
 				}
 			}
+
 			elapsed := time.Since(start)
 			log.Debug().Str("metric conversion took", elapsed.String()).Send()
 		}

pkg/events/events.go

@@ -36,6 +36,38 @@ type SaltEvent struct {
 	IsScheduleJob bool
 }
 
+func extractStateFromArgs(args interface{}) string {
+	// args only
+	if v, ok := args.(string); ok {
+		return v
+	}
+	// kwargs
+	if v, ok := args.(map[string]interface{}); ok {
+		if _, ok := v["mods"]; ok {
+			return v["mods"].(string)
+		}
+	}
+
+	return ""
+}
+
+// Extract state info from event
+func (e *SaltEvent) ExtractState() string {
+	switch e.Data.Fun {
+	case "state.sls", "state.apply":
+		if len(e.Data.Arg) > 0 {
+			return extractStateFromArgs(e.Data.Arg[0])
+		} else if len(e.Data.FunArgs) > 0 {
+			return extractStateFromArgs(e.Data.FunArgs[0])
+		} else if e.Data.Fun == "state.apply" {
+			return "highstate"
+		}
+	case "state.highstate":
+		return "highstate"
+	}
+	return ""
+}
+
 func ParseEvent(message map[string]interface{}, eventChan chan SaltEvent) {
 	body := string(message["body"].([]byte))
 	lines := strings.SplitN(body, "\n\n", 2)

pkg/events/events_test.go

@@ -39,6 +39,26 @@ func TestParseEvent(t *testing.T) {
 			args: fakeEventAsMap(fakeScheduleJobReturnEvent()),
 			want: expectedScheduleJobReturn,
 		},
+		{
+			name: "new state.sls",
+			args: fakeEventAsMap(fakeNewStateSlsJobEvent()),
+			want: expectedNewStateSlsJob,
+		},
+		{
+			name: "return state.sls",
+			args: fakeEventAsMap(fakeStateSlsReturnEvent()),
+			want: expectedStateSlsReturn,
+		},
+		{
+			name: "new state.single",
+			args: fakeEventAsMap(fakeNewStateSingleEvent()),
+			want: expectedNewStateSingle,
+		},
+		{
+			name: "return state.single",
+			args: fakeEventAsMap(fakeStateSingleReturnEvent()),
+			want: expectedStateSingleReturn,
+		},
 	}
 
 	for _, test := range tests {
@@ -55,3 +75,75 @@ func TestParseEvent(t *testing.T) {
 		}
 	}
 }
+
+func TestExtractState(t *testing.T) {
+	stateSls := getNewStateEvent()
+
+	stateSlsFunArg := getNewStateEvent()
+	stateSlsFunArg.Data.Arg = nil
+	stateSlsFunArg.Data.FunArgs = []interface{}{"test", map[string]bool{"dry_run": true}}
+
+	stateSlsFunArgMap := getNewStateEvent()
+	stateSlsFunArgMap.Data.Arg = nil
+	stateSlsFunArgMap.Data.FunArgs = []interface{}{map[string]interface{}{"mods": "test", "dry_run": true}}
+
+	stateApplyArg := getNewStateEvent()
+	stateApplyArg.Data.Fun = "state.apply"
+
+	stateApplyHighstate := getNewStateEvent()
+	stateApplyHighstate.Data.Fun = "state.apply"
+	stateApplyHighstate.Data.Arg = nil
+
+	stateHighstate := getNewStateEvent()
+	stateHighstate.Data.Fun = "state.highstate"
+	stateHighstate.Data.Arg = nil
+
+	tests := []struct {
+		name  string
+		event SaltEvent
+		want  string
+	}{
+		{
+			name:  "state via state.sls",
+			event: stateSls,
+			want:  "test",
+		},
+		{
+			name:  "state via state.sls args + kwargs",
+			event: stateSlsFunArg,
+			want:  "test",
+		},
+		{
+			name:  "state via state.sls kwargs only",
+			event: stateSlsFunArgMap,
+			want:  "test",
+		},
+		{
+			name:  "state via state.apply args only",
+			event: stateApplyArg,
+			want:  "test",
+		},
+		{
+			name:  "state via state.apply",
+			event: stateApplyArg,
+			want:  "test",
+		},
+		{
+			name:  "highstate via state.apply",
+			event: stateApplyHighstate,
+			want:  "highstate",
+		},
+		{
+			name:  "state.highstate",
+			event: stateHighstate,
+			want:  "highstate",
+		},
+	}
+
+	for _, test := range tests {
+		if res := test.event.ExtractState(); res != test.want {
+			t.Errorf("Mismatch for '%s', wants '%s' got '%s' ", test.name, test.want, res)
+		}
+	}
+
+}