Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add HTTP datasource #3294

Open
wants to merge 28 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
791901f
add HTTP datasource
he2ss Oct 23, 2024
4fc9e2d
remove unwanted comments
he2ss Oct 23, 2024
8315cdd
remove unused code
he2ss Oct 23, 2024
066bda9
add server close on dying tomb + acquis mode
he2ss Oct 23, 2024
5577e10
remove unwanted trace
he2ss Oct 23, 2024
2cc5c4e
Merge branch 'master' into feat/acquisition/add-http-datasource
he2ss Oct 23, 2024
1784d54
fix branch
he2ss Oct 23, 2024
329a957
fix lint
he2ss Oct 24, 2024
fc29db9
make transform unmarshal works + fix lint
he2ss Oct 24, 2024
9bc0f74
handle gzip encoding
he2ss Oct 24, 2024
4713184
fix lint
he2ss Oct 24, 2024
0e6031c
assert line.src in tests
he2ss Oct 24, 2024
e9bf32e
validate path + better naming
he2ss Oct 24, 2024
075bb92
comment for now chunk_size param
he2ss Oct 25, 2024
bdb0a40
handle ndjson + adapt tests to code and not the inverse
he2ss Oct 25, 2024
da457ee
remove unused param in func
he2ss Oct 25, 2024
5de442a
forgotten file
he2ss Oct 25, 2024
a53e2a9
fix comments
he2ss Oct 25, 2024
6004557
fix lint
he2ss Oct 28, 2024
8141609
update metrics + fix tests
he2ss Oct 28, 2024
168b452
fix laurence comments
he2ss Oct 28, 2024
e39cf00
Merge branch 'master' into feat/acquisition/add-http-datasource
he2ss Oct 28, 2024
d5a485c
fix typo
he2ss Oct 28, 2024
750a6b6
constructor for types.Event
blotus Oct 28, 2024
6d5df68
more parameters for MakeEvent
blotus Oct 28, 2024
bfd32a1
fix laurence comments
he2ss Oct 29, 2024
85f74d0
fix laurence comments
he2ss Oct 29, 2024
99a988e
properly copy event in transform
blotus Oct 29, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 17 additions & 7 deletions pkg/acquisition/acquisition.go
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,20 @@
return nil
}

// There's no need for an actual deep copy
// The event is almost empty, we are mostly interested in allocating new maps for Parsed/Meta/...
func copyEvent(evt types.Event, line string) types.Event {
evtCopy := types.MakeEvent(evt.ExpectMode == types.TIMEMACHINE, evt.Type, evt.Process)
evtCopy.Line = evt.Line
evtCopy.Line.Raw = line
evtCopy.Line.Labels = make(map[string]string)
for k, v := range evt.Line.Labels {
evtCopy.Line.Labels[k] = v
}

Check warning on line 349 in pkg/acquisition/acquisition.go

View check run for this annotation

Codecov / codecov/patch

pkg/acquisition/acquisition.go#L342-L349

Added lines #L342 - L349 were not covered by tests

return evtCopy

Check warning on line 351 in pkg/acquisition/acquisition.go

View check run for this annotation

Codecov / codecov/patch

pkg/acquisition/acquisition.go#L351

Added line #L351 was not covered by tests
}

func transform(transformChan chan types.Event, output chan types.Event, AcquisTomb *tomb.Tomb, transformRuntime *vm.Program, logger *log.Entry) {
defer trace.CatchPanic("crowdsec/acquis")
logger.Infof("transformer started")
Expand All @@ -363,8 +377,7 @@
switch v := out.(type) {
case string:
logger.Tracef("transform expression returned %s", v)
evt.Line.Raw = v
output <- evt
output <- copyEvent(evt, v)

Check warning on line 380 in pkg/acquisition/acquisition.go

View check run for this annotation

Codecov / codecov/patch

pkg/acquisition/acquisition.go#L380

Added line #L380 was not covered by tests
case []interface{}:
logger.Tracef("transform expression returned %v", v) //nolint:asasalint // We actually want to log the slice content

Expand All @@ -373,19 +386,16 @@
if !ok {
logger.Errorf("transform expression returned []interface{}, but cannot assert an element to string")
output <- evt

continue
}

evt.Line.Raw = l
output <- evt
output <- copyEvent(evt, l)

Check warning on line 392 in pkg/acquisition/acquisition.go

View check run for this annotation

Codecov / codecov/patch

pkg/acquisition/acquisition.go#L392

Added line #L392 was not covered by tests
}
case []string:
logger.Tracef("transform expression returned %v", v)

for _, line := range v {
evt.Line.Raw = line
output <- evt
output <- copyEvent(evt, line)

Check warning on line 398 in pkg/acquisition/acquisition.go

View check run for this annotation

Codecov / codecov/patch

pkg/acquisition/acquisition.go#L398

Added line #L398 was not covered by tests
}
default:
logger.Errorf("transform expression returned an invalid type %T, sending event as-is", out)
Expand Down
12 changes: 12 additions & 0 deletions pkg/acquisition/http.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
//go:build !no_datasource_http

package acquisition

import (
httpacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/http"
)

//nolint:gochecknoinits
func init() {
registerDataSource("http", func() DataSource { return &httpacquisition.HTTPSource{} })
}
5 changes: 1 addition & 4 deletions pkg/acquisition/modules/appsec/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -195,10 +195,7 @@ func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) {
}

func EventFromRequest(r *appsec.ParsedRequest, labels map[string]string) (types.Event, error) {
evt := types.Event{}
// we might want to change this based on in-band vs out-of-band ?
evt.Type = types.LOG
evt.ExpectMode = types.LIVE
evt := types.MakeEvent(false, types.LOG, true)
// def needs fixing
evt.Stage = "s00-raw"
evt.Parsed = map[string]string{
Expand Down
5 changes: 1 addition & 4 deletions pkg/acquisition/modules/cloudwatch/cloudwatch.go
Original file line number Diff line number Diff line change
Expand Up @@ -710,7 +710,7 @@ func (cw *CloudwatchSource) CatLogStream(ctx context.Context, cfg *LogStreamTail

func cwLogToEvent(log *cloudwatchlogs.OutputLogEvent, cfg *LogStreamTailConfig) (types.Event, error) {
l := types.Line{}
evt := types.Event{}
evt := types.MakeEvent(cfg.ExpectMode == types.TIMEMACHINE, types.LOG, true)
if log.Message == nil {
return evt, errors.New("nil message")
}
Expand All @@ -726,9 +726,6 @@ func cwLogToEvent(log *cloudwatchlogs.OutputLogEvent, cfg *LogStreamTailConfig)
l.Process = true
l.Module = "cloudwatch"
evt.Line = l
evt.Process = true
evt.Type = types.LOG
evt.ExpectMode = cfg.ExpectMode
cfg.logger.Debugf("returned event labels : %+v", evt.Line.Labels)
return evt, nil
}
13 changes: 6 additions & 7 deletions pkg/acquisition/modules/docker/docker.go
Original file line number Diff line number Diff line change
Expand Up @@ -334,7 +334,10 @@ func (d *DockerSource) OneShotAcquisition(ctx context.Context, out chan types.Ev
if d.metricsLevel != configuration.METRICS_NONE {
linesRead.With(prometheus.Labels{"source": containerConfig.Name}).Inc()
}
evt := types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE}
evt := types.MakeEvent(true, types.LOG, true)
evt.Line = l
evt.Process = true
evt.Type = types.LOG
out <- evt
d.logger.Debugf("Sent line to parsing: %+v", evt.Line.Raw)
}
Expand Down Expand Up @@ -579,12 +582,8 @@ func (d *DockerSource) TailDocker(ctx context.Context, container *ContainerConfi
l.Src = container.Name
l.Process = true
l.Module = d.GetName()
var evt types.Event
if !d.Config.UseTimeMachine {
evt = types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.LIVE}
} else {
evt = types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE}
}
evt := types.MakeEvent(d.Config.UseTimeMachine, types.LOG, true)
evt.Line = l
linesRead.With(prometheus.Labels{"source": container.Name}).Inc()
outChan <- evt
d.logger.Debugf("Sent line to parsing: %+v", evt.Line.Raw)
Expand Down
10 changes: 4 additions & 6 deletions pkg/acquisition/modules/file/file.go
Original file line number Diff line number Diff line change
Expand Up @@ -621,11 +621,9 @@ func (f *FileSource) tailFile(out chan types.Event, t *tomb.Tomb, tail *tail.Tai
// we're tailing, it must be real time logs
logger.Debugf("pushing %+v", l)

expectMode := types.LIVE
if f.config.UseTimeMachine {
expectMode = types.TIMEMACHINE
}
out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: expectMode}
evt := types.MakeEvent(f.config.UseTimeMachine, types.LOG, true)
evt.Line = l
out <- evt
}
}
}
Expand Down Expand Up @@ -684,7 +682,7 @@ func (f *FileSource) readFile(filename string, out chan types.Event, t *tomb.Tom
linesRead.With(prometheus.Labels{"source": filename}).Inc()

// we're reading logs at once, it must be time-machine buckets
out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE}
out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE, Unmarshaled: make(map[string]interface{})}
}
}

Expand Down
Loading