Skip to content

Commit

Permalink
refactor: removed concurrent retry for failed log messages
Browse files Browse the repository at this point in the history
We've commented the code for concurrent retries for failed log messages.
This was needed since we're flooding the NIC with too many requests and
it causes unexpected behaviours

Signed-off-by: jay-dee7 <jasdeepsingh.uppal@gmail.com>
  • Loading branch information
jay-dee7 committed Oct 28, 2021
1 parent c6ef926 commit d66cdcc
Showing 1 changed file with 1 addition and 13 deletions.
14 changes: 1 addition & 13 deletions telemetry/fluent-bit/fluent_bit.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"bytes"
"context"
"net/http"
"sync"
"time"

"github.com/containerish/OpenRegistry/config"
Expand All @@ -17,10 +16,8 @@ type (
}

fluentBit struct {
wg sync.WaitGroup
client *http.Client
retryMessages map[string]retryLogMsg
gate chan struct{}
config *config.RegistryConfig
}

Expand All @@ -39,8 +36,6 @@ func New(config *config.RegistryConfig) (FluentBit, error) {
fbClient := &fluentBit{
client: httpClient,
config: config,
wg: sync.WaitGroup{},
gate: make(chan struct{}, 5),
retryMessages: make(map[string]retryLogMsg),
}

Expand Down Expand Up @@ -101,19 +96,12 @@ func (fb *fluentBit) retry() {
// lets not do more than 5 req/second just to not flood our free instance of grafana cloud
for range ticker.C {
for id, logMsg := range fb.retryMessages {
fb.gate <- struct{}{}
fb.wg.Add(1)
go fb.retrier(logMsg.content, id)
fb.retrier(logMsg.content, id)
}
}
}

func (fb *fluentBit) retrier(logBytes []byte, id string) {
defer func() {
fb.wg.Done()
<-fb.gate
}()

// TODO - (@jay-dee7) what to do then? maybe have a different way to ship these logs? like via promtail?
if msg, ok := fb.retryMessages[id]; ok && msg.count > 3 {
delete(fb.retryMessages, id)
Expand Down

0 comments on commit d66cdcc

Please sign in to comment.