Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: do not replace influxdb acceptable chars when FieldsExtend enabled #565

Merged
merged 3 commits into from
Dec 12, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions helper/decoder/influxdb/decoder.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,15 +119,21 @@ func (d *Decoder) parsePointsToLogs(points []models.Point, req *http.Request) []
name = string(s.Name()) + ":" + field
}

helper.ReplaceInvalidChars(&name)
if !d.FieldsExtend {
helper.ReplaceInvalidChars(&name)
}
var builder strings.Builder
for index, v := range s.Tags() {
if index != 0 {
builder.WriteByte('|')
}
key := string(v.Key)
helper.ReplaceInvalidChars(&key)
builder.WriteString(key)
if !d.FieldsExtend {
key := string(v.Key)
helper.ReplaceInvalidChars(&key)
builder.WriteString(key)
} else {
builder.Write(v.Key)
}
builder.WriteString("#$#")
builder.WriteString(string(v.Value))
}
Expand Down
79 changes: 79 additions & 0 deletions helper/decoder/influxdb/decoder_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ import (
"testing"

"github.com/stretchr/testify/assert"

"github.com/alibaba/ilogtail/pkg/protocol"
)

var textFormat = `
Expand Down Expand Up @@ -59,6 +61,83 @@ mysql,host=Vm-Req-170328120400894271-tianchi113855.tc,server=rm-bp1eomqfte2vj91t
mysql,host=Vm-Req-170328120400894271-tianchi113855.tc,server=rm-bp1eomqfte2vj91tkjo.mysql.rds.aliyuncs.com:3306 innodb_buffer_pool_read_ahead_rnd=0i,innodb_data_pending_fsyncs=0i,innodb_buffer_pool_bytes_dirty=4325376i,innodb_buffer_pool_pages_flushed=21810i,innodb_buffer_pool_pages_total=40960i,innodb_buffer_pool_read_ahead_evicted=0i,innodb_buffer_pool_reads=757i,innodb_buffer_pool_load_status="Buffer pool(s) load completed at 200702 21:33:49",innodb_buffer_pool_pages_data=846i,innodb_buffer_pool_read_ahead=0i,innodb_buffer_pool_write_requests=36830857i,innodb_data_fsyncs=344588i,innodb_buffer_pool_dump_status="Dumping of buffer pool not started",innodb_buffer_pool_pages_dirty=264i,innodb_buffer_pool_pages_misc=3i,innodb_buffer_pool_read_requests=45390218i,innodb_buffer_pool_wait_free=0i,innodb_buffer_pool_bytes_data=13860864i,innodb_buffer_pool_pages_free=40111i 1595406780000000000
`

var txtWithDotNames = `
cpu.load,host=server01,region=uswest value=1 1434055562000000000
cpu.load,host.dd=server02,region=uswest value=3 1434055562000010000
`

func TestFieldsExtend(t *testing.T) {
cases := []struct {
enableFieldsExtend bool
data string
wantLogs []*protocol.Log
wantErr bool
}{
{
enableFieldsExtend: true,
data: txtWithDotNames,
wantErr: false,
wantLogs: []*protocol.Log{
{
Contents: []*protocol.Log_Content{
{Key: "__name__", Value: "cpu.load"},
{Key: "__value__", Value: "1"},
{Key: "__labels__", Value: "host#$#server01|region#$#uswest"},
{Key: "__time_nano__", Value: "1434055562000000000"},
{Key: "__type__", Value: "float"},
},
},
{
Contents: []*protocol.Log_Content{
{Key: "__name__", Value: "cpu.load"},
{Key: "__value__", Value: "3"},
{Key: "__labels__", Value: "host.dd#$#server02|region#$#uswest"},
{Key: "__time_nano__", Value: "1434055562000010000"},
{Key: "__type__", Value: "float"},
},
},
},
},
{
enableFieldsExtend: false,
data: txtWithDotNames,
wantErr: false,
wantLogs: []*protocol.Log{
{
Contents: []*protocol.Log_Content{
{Key: "__name__", Value: "cpu_load"},
{Key: "__value__", Value: "1"},
{Key: "__labels__", Value: "host#$#server01|region#$#uswest"},
{Key: "__time_nano__", Value: "1434055562000000000"},
},
},
{
Contents: []*protocol.Log_Content{
{Key: "__name__", Value: "cpu_load"},
{Key: "__value__", Value: "3"},
{Key: "__labels__", Value: "host_dd#$#server02|region#$#uswest"},
{Key: "__time_nano__", Value: "1434055562000010000"},
},
},
},
},
}

for _, testCase := range cases {
decoder := &Decoder{FieldsExtend: testCase.enableFieldsExtend}
logs, err := decoder.Decode([]byte(txtWithDotNames), &http.Request{})
if testCase.wantErr {
assert.NotNil(t, err)
continue
}
assert.Nil(t, err)
assert.Len(t, logs, len(testCase.wantLogs))
for i := 0; i < len(testCase.wantLogs); i++ {
assert.ElementsMatch(t, testCase.wantLogs[i].Contents, logs[i].Contents)
}
}
}

func TestNormal(t *testing.T) {
decoder := &Decoder{}
req := &http.Request{}
Expand Down