Skip to content

Commit

Permalink
NETOBSERV-1200 UI: DNSTracking feedback improvments (#361)
Browse files Browse the repository at this point in the history
* DNS latency & rcodes in overview + filters

* group DNS columns

* more than filter compare

* removed unecessary check

* lint

* fix moreThanRegex + testing

* remove moreThanRegex start match
  • Loading branch information
jpinsonneau authored Aug 3, 2023
1 parent 6e00208 commit 13e24e4
Show file tree
Hide file tree
Showing 52 changed files with 1,071 additions and 188 deletions.
9 changes: 9 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,15 @@ start: build-backend install-frontend ## Run backend and frontend
bash -c "trap 'fuser -k 9002/tcp' EXIT; \
./plugin-backend -port 9002 -metrics-port 9003 $(CMDLINE_ARGS) & cd web && npm run start"

.PHONY: start-backend
start-backend: build-backend
bash -c "trap 'fuser -k 9002/tcp' EXIT; \
./plugin-backend -port 9002 -metrics-port 9003 $(CMDLINE_ARGS)"

.PHONY: start-frontend-standalone
start-frontend-standalone: install-frontend ## Run frontend as standalone
cd web && npm run start:standalone

.PHONY: start-standalone
start-standalone: build-backend install-frontend ## Run backend and frontend as standalone
@echo "### Starting backend on http://localhost:9002"
Expand Down
3 changes: 3 additions & 0 deletions config/sample-frontend-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ quickFilters:
- name: Services network
filter:
dst_kind: 'Service'
- name: DNS
filter:
dns_id!: '0'
alertNamespaces:
- netobserv
sampling: 50
Expand Down
98 changes: 93 additions & 5 deletions pkg/loki/filter.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package loki

import (
"fmt"
"math"
"strconv"
"strings"
)

Expand All @@ -11,10 +13,11 @@ var valueReplacer = strings.NewReplacer(`*`, `.*`, `"`, "")
type labelMatcher string

const (
labelEqual = labelMatcher("=")
labelMatches = labelMatcher("=~")
labelNotEqual = labelMatcher("!=")
labelNoMatches = labelMatcher("!~")
labelEqual = labelMatcher("=")
labelMatches = labelMatcher("=~")
labelNotEqual = labelMatcher("!=")
labelMoreThanOrEqual = labelMatcher(">=")
labelNoMatches = labelMatcher("!~")
)

type valueType int
Expand Down Expand Up @@ -42,6 +45,7 @@ type lineFilter struct {
strictKey bool
values []lineMatch
not bool
moreThan bool
}

type lineMatch struct {
Expand Down Expand Up @@ -76,6 +80,15 @@ func notStringLabelFilter(labelKey string, value string) labelFilter {
}
}

func moreThanNumberLabelFilter(labelKey string, value string) labelFilter {
return labelFilter{
key: labelKey,
matcher: labelMoreThanOrEqual,
value: value,
valueType: typeNumber,
}
}

func stringNotMatchLabelFilter(labelKey string, value string) labelFilter {
return labelFilter{
key: labelKey,
Expand Down Expand Up @@ -136,6 +149,8 @@ func (f *lineFilter) asLabelFilters() []labelFilter {
} else {
if f.not {
lf.matcher = labelNotEqual
} else if f.moreThan {
lf.matcher = labelMoreThanOrEqual
} else {
lf.matcher = labelEqual
}
Expand Down Expand Up @@ -172,13 +187,82 @@ func notContainsKeyLineFilter(key string) lineFilter {
key: key,
strictKey: true,
not: true,
moreThan: false,
}
}

func moreThanRegex(sb *strings.Builder, value string) {
// match each number greater than specified value using regex
// example for 123:
// ( 12[3-9] | 1[3-9][0-9]+ | [2-9][0-9]+ | [1-9][0-9]{3,} )
// | | | |
// | | | ↪ match number more than 1000
// | | |
// | | ↪ match number from 200 to 999
// | |
// | ↪ match numbers from 130 to 199
// |
// ↪ match any number from 123 to 129

sb.WriteString("(")
for i, r := range value {
if i < len(value)-1 {
sb.WriteRune(r)
} else {
sb.WriteRune('[')
sb.WriteRune(r)
sb.WriteString("-9]")
}
}

intVal, _ := strconv.Atoi(value)
for i := 1; i < len(value); i++ {
nextMin := int((intVal / int(math.Pow10(i))) + 1)
nextMinStr := fmt.Sprintf("%d", nextMin)

sb.WriteRune('|')
if nextMin >= 10 {
sb.WriteString(nextMinStr[0 : len(nextMinStr)-1])
} else if i < len(value)-1 {
sb.WriteString(value[0 : len(value)-1-i])
}

nextMinRune := nextMinStr[len(nextMinStr)-1:]
if nextMin%9 != 0 {
sb.WriteRune('[')
sb.WriteString(nextMinRune)
sb.WriteString("-9]")
} else {
sb.WriteString(nextMinRune)
}

sb.WriteString("[0-9]")

if i > 1 {
sb.WriteString("{")
sb.WriteString(fmt.Sprintf("%d", i))
sb.WriteString(",}")
}
}

sb.WriteString("|[1-9][0-9]{")
sb.WriteString(fmt.Sprintf("%d", len(value)))
sb.WriteString(",})")
}

// writeInto transforms a lineFilter to its corresponding part of a LogQL query
// under construction (contained in the provided strings.Builder)
func (f *lineFilter) writeInto(sb *strings.Builder) {
if f.not {
// the record must contains the field if values are specified
// since FLP skip empty fields / zeros values
if len(f.values) > 0 {
sb.WriteString("|~`\"")
sb.WriteString(f.key)
sb.WriteString("\"`")
}

// then we exclude match results
sb.WriteString("!~`")
} else {
sb.WriteString("|~`")
Expand Down Expand Up @@ -214,7 +298,11 @@ func (f *lineFilter) writeInto(sb *strings.Builder) {
sb.WriteString(`":`)
switch v.valueType {
case typeNumber, typeRegex:
sb.WriteString(v.value)
if f.moreThan {
moreThanRegex(sb, v.value)
} else {
sb.WriteString(v.value)
}
// a number or regex can be followed by } if it's the last property of a JSON document
sb.WriteString("[,}]")
case typeString, typeIP:
Expand Down
76 changes: 76 additions & 0 deletions pkg/loki/filter_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package loki

import (
"fmt"
"regexp"
"strconv"
"strings"
"testing"

"github.com/stretchr/testify/assert"
)

func TestWriteInto_1(t *testing.T) {
sb := strings.Builder{}
moreThanRegex(&sb, "1")
result := sb.String()
assert.Equal(t, result, "([1-9]|[1-9][0-9]{1,})")
reg := regexp.MustCompile(result)
fmt.Printf("\nRegex: %s", result)
for _, i := range []int{0, 1, 2, 3, 10, 100} {
val := strconv.Itoa(i)
assert.Equal(t, reg.MatchString(val), i >= 1, fmt.Sprintf("Value: %d", i))
}
}

func TestWriteInto_7(t *testing.T) {
sb := strings.Builder{}
moreThanRegex(&sb, "7")
result := sb.String()
assert.Equal(t, result, "([7-9]|[1-9][0-9]{1,})")
reg := regexp.MustCompile(result)
fmt.Printf("\nRegex: %s", result)
for _, i := range []int{0, 1, 3, 7, 10, 11, 100} {
val := strconv.Itoa(i)
assert.Equal(t, reg.MatchString(val), i >= 7, fmt.Sprintf("Value: %d", i))
}
}

func TestWriteInto_15(t *testing.T) {
sb := strings.Builder{}
moreThanRegex(&sb, "15")
result := sb.String()
assert.Equal(t, result, "(1[5-9]|[2-9][0-9]|[1-9][0-9]{2,})")
reg := regexp.MustCompile(result)
fmt.Printf("\nRegex: %s", result)
for _, i := range []int{0, 1, 5, 14, 15, 16, 150, 1050} {
val := strconv.Itoa(i)
assert.Equal(t, reg.MatchString(val), i >= 15, fmt.Sprintf("Value: %d", i))
}
}

func TestWriteInto_123(t *testing.T) {
sb := strings.Builder{}
moreThanRegex(&sb, "123")
result := sb.String()
assert.Equal(t, result, "(12[3-9]|1[3-9][0-9]|[2-9][0-9]{2,}|[1-9][0-9]{3,})")
reg := regexp.MustCompile(result)
fmt.Printf("\nRegex: %s", result)
for _, i := range []int{0, 1, 10, 100, 115, 123, 124, 150, 200, 1230} {
val := strconv.Itoa(i)
assert.Equal(t, reg.MatchString(val), i >= 123, fmt.Sprintf("Value: %d", i))
}
}

func TestWriteInto_7654(t *testing.T) {
sb := strings.Builder{}
moreThanRegex(&sb, "7654")
result := sb.String()
assert.Equal(t, result, "(765[4-9]|76[6-9][0-9]|7[7-9][0-9]{2,}|[8-9][0-9]{3,}|[1-9][0-9]{4,})")
reg := regexp.MustCompile(result)
fmt.Printf("\nRegex: %s", result)
for _, i := range []int{0, 1, 1000, 7654, 7655, 10000} {
val := strconv.Itoa(i)
assert.Equal(t, reg.MatchString(val), i >= 7654, fmt.Sprintf("Value: %d", i))
}
}
46 changes: 35 additions & 11 deletions pkg/loki/flow_query.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,8 @@ func (q *FlowQueryBuilder) addFilter(filter filters.Match) error {
if len(values) == 1 && isExactMatch(values[0]) {
if filter.Not {
q.labelFilters = append(q.labelFilters, notStringLabelFilter(filter.Key, trimExactMatch(values[0])))
} else if filter.MoreThanOrEqual {
q.labelFilters = append(q.labelFilters, moreThanNumberLabelFilter(filter.Key, trimExactMatch(values[0])))
} else {
q.labelFilters = append(q.labelFilters, stringEqualLabelFilter(filter.Key, trimExactMatch(values[0])))
}
Expand All @@ -131,7 +133,7 @@ func (q *FlowQueryBuilder) addFilter(filter filters.Match) error {
}
q.addIPFilters(filter.Key, values)
} else {
q.addLineFilters(filter.Key, values, filter.Not)
q.addLineFilters(filter.Key, values, filter.Not, filter.MoreThanOrEqual)
}

return nil
Expand Down Expand Up @@ -169,13 +171,14 @@ func (q *FlowQueryBuilder) addLabelRegex(key string, values []string, not bool)
}
}

func (q *FlowQueryBuilder) addLineFilters(key string, values []string, not bool) {
func (q *FlowQueryBuilder) addLineFilters(key string, values []string, not bool, moreThan bool) {
if len(values) == 0 {
return
}
lf := lineFilter{
key: key,
not: not,
key: key,
not: not,
moreThan: moreThan,
}
isNumeric := fields.IsNumeric(key)
emptyMatches := false
Expand Down Expand Up @@ -245,23 +248,44 @@ func (q *FlowQueryBuilder) appendLineFilters(sb *strings.Builder) {
}

func (q *FlowQueryBuilder) appendDeduplicateFilter(sb *strings.Builder) {
// |~`Duplicate":false`
// |~`"Duplicate":false`
sb.WriteString("|~`")
sb.WriteString(`Duplicate":false`)
sb.WriteString(`"Duplicate":false`)
sb.WriteString("`")
}

func (q *FlowQueryBuilder) appendPktDropStateFilter(sb *strings.Builder) {
// !~`PktDropLatestState":0`
sb.WriteString("!~`")
sb.WriteString(`PktDropLatestState":0`)
// ensure PktDropLatestState is specified
// |~`"PktDropLatestState"`
sb.WriteString("|~`")
sb.WriteString(`"PktDropLatestState"`)
sb.WriteString("`")
}

func (q *FlowQueryBuilder) appendPktDropCauseFilter(sb *strings.Builder) {
// !~`PktDropLatestDropCause":0`
// ensure PktDropLatestDropCause is specified
// |~`"PktDropLatestDropCause"`
sb.WriteString("|~`")
sb.WriteString(`"PktDropLatestDropCause"`)
sb.WriteString("`")
}

func (q *FlowQueryBuilder) appendDNSFilter(sb *strings.Builder) {
// ensure at least one Dns field is specified
// |~`"Dns`
sb.WriteString("|~`")
sb.WriteString(`"Dns`)
sb.WriteString("`")
}

func (q *FlowQueryBuilder) appendDNSRCodeFilter(sb *strings.Builder) {
// ensure DnsFlagsResponseCode field is specified with valid error
// |~`"DnsFlagsResponseCode"`!~`"DnsFlagsResponseCode":"NoError"`
sb.WriteString("|~`")
sb.WriteString(`"DnsFlagsResponseCode"`)
sb.WriteString("`")
sb.WriteString("!~`")
sb.WriteString(`PktDropLatestDropCause":0`)
sb.WriteString(`"DnsFlagsResponseCode":"NoError"`)
sb.WriteString("`")
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/loki/query_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ func TestFlowQuery_AddNotLineFilters(t *testing.T) {
err = query.addFilter(filters.NewNotMatch("flis", `"flas"`))
require.NoError(t, err)
urlQuery := query.Build()
assert.Equal(t, `/loki/api/v1/query_range?query={app="netobserv-flowcollector"}|~`+backtick(`foo":"bar"`)+`!~`+backtick(`flis":"flas"`), urlQuery)
assert.Equal(t, `/loki/api/v1/query_range?query={app="netobserv-flowcollector"}|~`+backtick(`foo":"bar"`)+`|~`+backtick(`"flis"`)+`!~`+backtick(`flis":"flas"`), urlQuery)
}

func TestFlowQuery_AddLineFiltersWithEmpty(t *testing.T) {
Expand Down
Loading

0 comments on commit 13e24e4

Please sign in to comment.