Skip to content

Commit

Permalink
chore: some more cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
raj-k-singh committed Nov 29, 2023
1 parent 2b2cda7 commit 34c52b6
Show file tree
Hide file tree
Showing 5 changed files with 14 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,6 @@ func getOperators(ops []PipelineOperator) ([]PipelineOperator, error) {

}
// TODO(Raj): Maybe add support for gotime too eventually
operator.OnError = "send"
}

filteredOp = append(filteredOp, operator)
Expand Down
4 changes: 4 additions & 0 deletions pkg/query-service/app/logparsingpipeline/postablePipeline.go
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ func isValidOperator(op PipelineOperator) error {
return fmt.Errorf(fmt.Sprintf("parse from of time parsing processor %s cannot be empty", op.ID))
}
if op.LayoutType != "epoch" && op.LayoutType != "strptime" {
// TODO(Raj): Maybe add support for gotime format
return fmt.Errorf(fmt.Sprintf(
"invalid format type '%s' of time parsing processor %s", op.LayoutType, op.ID,
))
Expand All @@ -186,6 +187,9 @@ func isValidOperator(op PipelineOperator) error {
))
}

// TODO(Raj): Add validation for strptime layouts via
// collector simulator maybe.

default:
return fmt.Errorf(fmt.Sprintf("operator type %s not supported for %s, use one of (grok_parser, regex_parser, copy, move, add, remove, trace_parser, retain)", op.Type, op.ID))
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -309,10 +309,9 @@ var operatorTest = []struct {
}, {
Name: "Timestamp Parser - invalid epoch layout",
Operator: PipelineOperator{
ID: "time",
Type: "time_parser",
ParseFrom: "attributes.test_timestamp",
// TODO(Raj): Maybe add support for gotime format
ID: "time",
Type: "time_parser",
ParseFrom: "attributes.test_timestamp",
LayoutType: "epoch",
Layout: "%Y-%m-%d",
},
Expand Down
9 changes: 7 additions & 2 deletions pkg/query-service/app/logparsingpipeline/time_parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,12 @@ import (
)

// Regex for strptime format placeholders supported by the time parser.
// Used for defining if conditions on time parsing operators so they do not
// spam collector logs when encountering values that can't be parsed.
//
// Based on ctimeSubstitutes defined in https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/internal/coreinternal/timeutils/internal/ctimefmt/ctimefmt.go#L22
//
// TODO(Raj): Maybe make the expressions tighter.
var ctimeRegex = map[string]string{
// %Y - Year, zero-padded (0001, 0002, ..., 2019, 2020, ..., 9999)
"%Y": "[0-9]{4}",
Expand Down Expand Up @@ -105,8 +110,8 @@ func RegexForStrptimeLayout(layout string) (string, error) {
return ""
}

ctimeRegexp := regexp.MustCompile(`%.`)
layoutRegex = ctimeRegexp.ReplaceAllStringFunc(layoutRegex, replaceStrptimeDirectiveWithRegex)
strptimeDirectiveRegexp := regexp.MustCompile(`%.`)
layoutRegex = strptimeDirectiveRegexp.ReplaceAllStringFunc(layoutRegex, replaceStrptimeDirectiveWithRegex)
if len(errs) != 0 {
return "", fmt.Errorf("couldn't generate regex for ctime format: %v", errs)
}
Expand Down
2 changes: 0 additions & 2 deletions pkg/query-service/app/logparsingpipeline/time_parser_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,6 @@ func TestTimestampParsingProcessor(t *testing.T) {
},
}

// Start with JSON serialized timestamp parser to validate deserialization too
// TODO(Raj): Is this needed?
var timestampParserOp PipelineOperator
err := json.Unmarshal([]byte(`
{
Expand Down

0 comments on commit 34c52b6

Please sign in to comment.