-
Notifications
You must be signed in to change notification settings - Fork 0
/
influxdb_sink.go
175 lines (140 loc) · 3.44 KB
/
influxdb_sink.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
package main
import (
"bytes"
"fmt"
"net/http"
"time"
"strings"
"sync"
)
// InfluxDB sink object.
type InfluxDBSink struct {
fifo []*Point
lock sync.Mutex
pushInterval time.Duration
url string
batchSize uint
bufferSize uint
client *http.Client
}
// Returns a new influx db sink.
func NewInfluxDBSink(url string, bufferSize uint, batchSize uint,
pushInterval time.Duration) (is *InfluxDBSink, err error) {
// buffer 100k points by default
if bufferSize == 0 {
bufferSize = 100000
}
// default to pushing 1k points at once
if batchSize == 0 {
batchSize = 1000
}
is = &InfluxDBSink{
fifo: make([]*Point, 0),
batchSize: batchSize,
bufferSize: bufferSize,
pushInterval: pushInterval,
url: url,
client: &http.Client{
Timeout: 1 * time.Minute,
},
}
go is.writer()
return
}
// Pushes points to the sink's internal buffer. Points are rejected when
// the buffer is full.
func (is *InfluxDBSink) Save(points []*Point) (acceptedCount uint) {
var pointCount uint
pointCount = uint(len(points))
if pointCount == 0 {
return
}
is.lock.Lock()
acceptedCount = is.bufferSize - uint(len(is.fifo))
if acceptedCount > pointCount {
acceptedCount = pointCount
}
if acceptedCount > 0 {
is.fifo = append(is.fifo, points[0:acceptedCount]...)
}
is.lock.Unlock()
if acceptedCount != pointCount {
fmt.Printf("influxdb sink: dropped %v points out of %v\n",
pointCount - acceptedCount, pointCount)
}
return
}
// Periodically reads points from the internal buffer, serializes them and pushes
// them to influxdb API.
func (is *InfluxDBSink) writer() {
var batch []*Point
var batchSize uint
var ticker *time.Ticker
var buf bytes.Buffer
var res *http.Response
var err error
ticker = time.NewTicker(is.pushInterval)
for {
// clear the send buffer
buf.Reset()
<-ticker.C
is.lock.Lock()
// determine how many points to grab
batchSize = uint(len(is.fifo))
if batchSize > is.batchSize {
batchSize = is.batchSize
}
// grab batchSize points
batch = is.fifo[0:batchSize]
is.lock.Unlock()
if batchSize == 0 {
continue
}
// serialize the batch
is.serialize(&buf, batch)
// post the serialized payload to influxdb
res, err = is.client.Post(
fmt.Sprintf("%s&precision=ms", is.url), "text", &buf)
if err != nil {
fmt.Printf("failed to perform influxdb POST request: %v\n", err)
continue
}
res.Body.Close()
if res.StatusCode != http.StatusNoContent {
fmt.Printf("influxdb POST request failed with status code: %d\n",
res.StatusCode)
continue
}
// free points which have been successfully pushed
is.lock.Lock()
is.fifo = is.fifo[batchSize:]
is.lock.Unlock()
}
return
}
// Turns points into influxdb line protocol entries.
func (is *InfluxDBSink) serialize(buf *bytes.Buffer, points []*Point) {
var idx int
var fieldName string
for _, p := range points {
if p == nil {
continue
}
// locate the last dot
idx = strings.LastIndex(p.Label, ".")
// if there is no dot or if the dot is either at the beginning or at the end
// of the label, we can't split on it: drop the point.
if idx < 1 || idx == len(p.Label) - 1 {
fmt.Printf("discarding point with label '%s': cannot split on '.'\n",
p.Label)
continue
}
// use the last token as field name
fieldName = p.Label[idx+1:len(p.Label)]
buf.WriteString(
fmt.Sprintf("%s %s=%v %v\n",
p.Label[0:idx], fieldName,
p.Value, p.Timestamp.UnixNano() / 1e6))
}
return
}