-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix(forwarder): improve CSV handling (#324)
A few simple performance fixes for converting CSV to JSON: - avoid unmarshalling if target is json.RawMessage, since that calls `json.Valid`. - avoid calling `strconv.Quote` where possible. Most of the inputs we handle in practice do not require special handling. - keep track of max record length and use it to initialize our buffer, which avoids resizing. In profiling, these changes reduced processing time for a VPC flow logs file by ~33%.
- Loading branch information
Showing
3 changed files
with
47 additions
and
19 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
20 changes: 10 additions & 10 deletions
20
pkg/handler/forwarder/s3http/internal/decoders/testdata/cloudwatchlogs.json.golden
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,10 +1,10 @@ | ||
{"id":"1","timestamp":1620678200,"message":"Error: Unable to connect to database server","owner":"123456789012","logGroup":"example-log-group","logStream":"example-log-stream","subscriptionFilters":["example-filter"],"messageType":"DATA_MESSAGE"} | ||
{"id":"2","timestamp":1620678300,"message":"Warning: High CPU usage detected","owner":"123456789012","logGroup":"example-log-group","logStream":"example-log-stream","subscriptionFilters":["example-filter"],"messageType":"DATA_MESSAGE"} | ||
{"id":"3","timestamp":1620678400,"message":"Info: Application started successfully","owner":"123456789012","logGroup":"example-log-group","logStream":"example-log-stream","subscriptionFilters":["example-filter"],"messageType":"DATA_MESSAGE"} | ||
{"id":"4","timestamp":1620678500,"message":"Error: Disk space running low","owner":"123456789012","logGroup":"example-log-group","logStream":"example-log-stream","subscriptionFilters":["example-filter"],"messageType":"DATA_MESSAGE"} | ||
{"id":"5","timestamp":1620678600,"message":"Info: User 'admin' logged in","owner":"123456789012","logGroup":"example-log-group","logStream":"example-log-stream","subscriptionFilters":["example-filter"],"messageType":"DATA_MESSAGE"} | ||
{"id":"1","timestamp":1620679200,"message":"Info: Server restarted","owner":"987654321098","logGroup":"another-log-group","logStream":"another-log-stream","subscriptionFilters":["filter-1","filter-2"],"messageType":"DATA_MESSAGE"} | ||
{"id":"2","timestamp":1620679300,"message":"Error: Service crashed unexpectedly","owner":"987654321098","logGroup":"another-log-group","logStream":"another-log-stream","subscriptionFilters":["filter-1","filter-2"],"messageType":"DATA_MESSAGE"} | ||
{"id":"3","timestamp":1620679400,"message":"Warning: Memory usage exceeding 90%","owner":"987654321098","logGroup":"another-log-group","logStream":"another-log-stream","subscriptionFilters":["filter-1","filter-2"],"messageType":"DATA_MESSAGE"} | ||
{"id":"4","timestamp":1620679500,"message":"Info: New user registered","owner":"987654321098","logGroup":"another-log-group","logStream":"another-log-stream","subscriptionFilters":["filter-1","filter-2"],"messageType":"DATA_MESSAGE"} | ||
{"id":"5","timestamp":1620679600,"message":"Error: Network connection lost","owner":"987654321098","logGroup":"another-log-group","logStream":"another-log-stream","subscriptionFilters":["filter-1","filter-2"],"messageType":"DATA_MESSAGE"} | ||
{"id":"1","logGroup":"example-log-group","logStream":"example-log-stream","message":"Error: Unable to connect to database server","messageType":"DATA_MESSAGE","owner":"123456789012","subscriptionFilters":["example-filter"],"timestamp":1620678200} | ||
{"id":"2","logGroup":"example-log-group","logStream":"example-log-stream","message":"Warning: High CPU usage detected","messageType":"DATA_MESSAGE","owner":"123456789012","subscriptionFilters":["example-filter"],"timestamp":1620678300} | ||
{"id":"3","logGroup":"example-log-group","logStream":"example-log-stream","message":"Info: Application started successfully","messageType":"DATA_MESSAGE","owner":"123456789012","subscriptionFilters":["example-filter"],"timestamp":1620678400} | ||
{"id":"4","logGroup":"example-log-group","logStream":"example-log-stream","message":"Error: Disk space running low","messageType":"DATA_MESSAGE","owner":"123456789012","subscriptionFilters":["example-filter"],"timestamp":1620678500} | ||
{"id":"5","logGroup":"example-log-group","logStream":"example-log-stream","message":"Info: User 'admin' logged in","messageType":"DATA_MESSAGE","owner":"123456789012","subscriptionFilters":["example-filter"],"timestamp":1620678600} | ||
{"id":"1","logGroup":"another-log-group","logStream":"another-log-stream","message":"Info: Server restarted","messageType":"DATA_MESSAGE","owner":"987654321098","subscriptionFilters":["filter-1","filter-2"],"timestamp":1620679200} | ||
{"id":"2","logGroup":"another-log-group","logStream":"another-log-stream","message":"Error: Service crashed unexpectedly","messageType":"DATA_MESSAGE","owner":"987654321098","subscriptionFilters":["filter-1","filter-2"],"timestamp":1620679300} | ||
{"id":"3","logGroup":"another-log-group","logStream":"another-log-stream","message":"Warning: Memory usage exceeding 90%","messageType":"DATA_MESSAGE","owner":"987654321098","subscriptionFilters":["filter-1","filter-2"],"timestamp":1620679400} | ||
{"id":"4","logGroup":"another-log-group","logStream":"another-log-stream","message":"Info: New user registered","messageType":"DATA_MESSAGE","owner":"987654321098","subscriptionFilters":["filter-1","filter-2"],"timestamp":1620679500} | ||
{"id":"5","logGroup":"another-log-group","logStream":"another-log-stream","message":"Error: Network connection lost","messageType":"DATA_MESSAGE","owner":"987654321098","subscriptionFilters":["filter-1","filter-2"],"timestamp":1620679600} |