From 391cd9b27ee4b192876c43774f8a71b4dd599191 Mon Sep 17 00:00:00 2001 From: Thomas Labarussias Date: Fri, 21 Oct 2022 12:56:57 +0200 Subject: [PATCH] AWS Security Lake output Signed-off-by: Thomas Labarussias --- README.md | 21 +++ config.go | 17 +++ config_example.yaml | 23 ++- go.mod | 7 + go.sum | 69 +++++++++ handlers.go | 10 +- main.go | 24 ++- outputs/aws.go | 10 +- outputs/awssecuritylake.go | 304 +++++++++++++++++++++++++++++++++++++ outputs/webui.go | 3 - stats.go | 1 + types/types.go | 19 +++ 12 files changed, 491 insertions(+), 17 deletions(-) create mode 100644 outputs/awssecuritylake.go diff --git a/README.md b/README.md index cc5f9ebeb..363ea504c 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,10 @@ It works as a single endpoint for as many as you want `Falco` instances : - [**Node-RED**](https://nodered.org/) - [**WebUI**](https://github.com/falcosecurity/falcosidekick-ui) (a Web UI for displaying latest events in real time) +### SIEM + +- [**AWS Security Lake**](https://aws.amazon.com/security-lake/) + ### Other - [**Policy Report**](https://github.com/kubernetes-sigs/wg-policy-prototypes/tree/master/policy-report/falco-adapter) @@ -288,6 +292,7 @@ aws: # accesskeyid: "" # aws access key (optional if you use EC2 Instance Profile) # secretaccesskey: "" # aws secret access key (optional if you use EC2 Instance Profile) # region : "" # aws region (by default, the metadata are used to get it) + # checkidentity: true # check the identity credentials, set to false for locale developments (default: true) lambda: # functionname : "" # Lambda function name, if not empty, AWS Lambda output is enabled # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) @@ -306,6 +311,14 @@ aws: # bucket: "falcosidekick" # AWS S3, bucket name # prefix : "" # name of prefix, keys will have format: s3:////YYYY-MM-DD/YYYY-MM-DDTHH:mm:ss.s+01:00.json # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) + securitylake.: + # bucket: "" # Bucket for AWS SecurityLake data, if not empty, AWS SecurityLake output is enabled + # region: "" # Bucket Region + # prefix: "" # Prefix for keys + # accountid: "" # Account ID + interval: 5 # Time in minutes between two puts to S3 (must be between 5 and 60min) (default: 5min) + batchsize: 1000 # Max number of events by parquet file (default: 1000) + # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) kinesis: # streamname: "" # AWS Kinesis Stream Name, if not empty, Kinesis output is enabled # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) @@ -758,6 +771,14 @@ care of lower/uppercases**) : `yaml: a.b --> envvar: A_B` : _enabled_ - **AWS_S3_PREFIX** : Prefix name of the object, keys will have format: s3:////YYYY-MM-DD/YYYY-MM-DDTHH:mm:ss.s+01:00.json - **AWS_S3_MINIMUMPRIORITY** : minimum priority of event for using this output, +- **AWS_SECURITYLAKE_BUCKET** : Bucket for AWS SecurityLake data, if not empty, AWS SecurityLake. output is _enabled_ +- **AWS_SECURITYLAKE_REGION** : Bucket Region (mandatory) +- **AWS_SECURITYLAKE_PREFIX** : Prefix for keys (mandatory) +- **AWS_SECURITYLAKE_ACCOUNTID** : Account ID (mandatory) +- **AWS_SECURITYLAKE_INTERVAL** : Time in minutes between two puts to S3 (must be between 5 and 60min) (default: 5min) +- **AWS_SECURITYLAKE_BATCHSIZE** : Max number of events by parquet file (default: 1000) +- **AWS_SECURITYLAKE_PREFIX** : Prefix name of the object, keys will have format: s3:////YYYY-MM-DD/YYYY-MM-DDTHH:mm:ss.s+01:00.json +- **AWS_SECURITYLAKE_MINIMUMPRIORITY** : minimum priority of event for using this output, order is - **AWS_KINESIS_STREAMNAME** : AWS Kinesis Stream Name, if not empty, Kinesis output is enabled - **AWS_KINESIS_MINIMUMPRIORITY** : minimum priority of event for using diff --git a/config.go b/config.go index ee12262a7..5bc6d394e 100644 --- a/config.go +++ b/config.go @@ -133,6 +133,7 @@ func getConfig() *types.Configuration { v.SetDefault("AWS.AccessKeyID", "") v.SetDefault("AWS.SecretAccessKey", "") v.SetDefault("AWS.Region", "") + v.SetDefault("AWS.CheckIdentity", true) v.SetDefault("AWS.Lambda.FunctionName", "") v.SetDefault("AWS.Lambda.InvocationType", "RequestResponse") @@ -154,6 +155,14 @@ func getConfig() *types.Configuration { v.SetDefault("AWS.S3.Prefix", "falco") v.SetDefault("AWS.S3.MinimumPriority", "") + v.SetDefault("AWS.SecurityLake.Bucket", "") + v.SetDefault("AWS.SecurityLake.Region", "") + v.SetDefault("AWS.SecurityLake.Prefix", "") + v.SetDefault("AWS.SecurityLake.Interval", 5) + v.SetDefault("AWS.SecurityLake.BatchSize", 1000) + v.SetDefault("AWS.SecurityLake.AccountID", "") + v.SetDefault("AWS.SecurityLake.MinimumPriority", "") + v.SetDefault("AWS.Kinesis.StreamName", "") v.SetDefault("AWS.Kinesis.MinimumPriority", "") @@ -507,6 +516,13 @@ func getConfig() *types.Configuration { } } + if c.AWS.SecurityLake.Interval < 5 { + c.AWS.SecurityLake.Interval = 5 + } + if c.AWS.SecurityLake.Interval > 60 { + c.AWS.SecurityLake.Interval = 60 + } + if c.ListenPort == 0 || c.ListenPort > 65536 { log.Fatalf("[ERROR] : Bad port number\n") } @@ -538,6 +554,7 @@ func getConfig() *types.Configuration { c.AWS.SQS.MinimumPriority = checkPriority(c.AWS.SQS.MinimumPriority) c.AWS.SNS.MinimumPriority = checkPriority(c.AWS.SNS.MinimumPriority) c.AWS.S3.MinimumPriority = checkPriority(c.AWS.S3.MinimumPriority) + c.AWS.SecurityLake.MinimumPriority = checkPriority(c.AWS.SecurityLake.MinimumPriority) c.AWS.CloudWatchLogs.MinimumPriority = checkPriority(c.AWS.CloudWatchLogs.MinimumPriority) c.AWS.Kinesis.MinimumPriority = checkPriority(c.AWS.Kinesis.MinimumPriority) c.Opsgenie.MinimumPriority = checkPriority(c.Opsgenie.MinimumPriority) diff --git a/config_example.yaml b/config_example.yaml index 70e7c917b..03f87c0a9 100644 --- a/config_example.yaml +++ b/config_example.yaml @@ -114,12 +114,13 @@ aws: # accesskeyid: "" # aws access key (optional if you use EC2 Instance Profile) # secretaccesskey: "" # aws secret access key (optional if you use EC2 Instance Profile) # region : "" # aws region (by default, the metadata are used to get it) + # checkidentity: true # check the identity credentials, set to false for locale developments (default: true) lambda: - # functionname : "" # Lambda function name, if not empty, AWS Lambda output is enabled - # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) + # functionname : "" # Lambda function name, if not empty, AWS Lambda output is enabled + # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) sqs: - # url : "" # SQS Queue URL, if not empty, AWS SQS output is enabled - # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) + # url : "" # SQS Queue URL, if not empty, AWS SQS output is enabled + # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) sns: # topicarn : "" # SNS TopicArn, if not empty, AWS SNS output is enabled rawjson: false # Send Raw JSON or parse it (default: false) @@ -129,9 +130,17 @@ aws: # logstream : "" # AWS CloudWatch Logs Stream name, if empty, Falcosidekick will try to create a log stream # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) s3: - # bucket: "falcosidekick" # AWS S3, bucket name - # prefix : "" # name of prefix, keys will have format: s3:////YYYY-MM-DD/YYYY-MM-DDTHH:mm:ss.s+01:00.json - # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) + # bucket: "falcosidekick" # AWS S3, bucket name + # prefix : "" # name of prefix, keys will have format: s3:////YYYY-MM-DD/YYYY-MM-DDTHH:mm:ss.s+01:00.json + # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) + securitylake.: + # bucket: "" # Bucket for AWS SecurityLake data, if not empty, AWS SecurityLake output is enabled + # region: "" # Bucket Region (mandatory) + # prefix: "" # Prefix for keys (mandatory) + # accountid: "" # Account ID (mandatory) + interval: 5 # Time in minutes between two puts to S3 (must be between 5 and 60min) (default: 5min) + batchsize: 1000 # Max number of events by parquet file (default: 1000) + # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) kinesis: # streamname: "" # AWS Kinesis Stream Name, if not empty, Kinesis output is enabled # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) diff --git a/go.mod b/go.mod index 458ef18b9..67cecddd7 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/aws/aws-sdk-go v1.44.89 github.com/cloudevents/sdk-go/v2 v2.11.0 github.com/eclipse/paho.mqtt.golang v1.4.1 + github.com/embano1/memlog v0.4.3 github.com/emersion/go-sasl v0.0.0-20211008083017-0b9dcfb154ac github.com/emersion/go-smtp v0.15.0 github.com/google/uuid v1.3.0 @@ -26,6 +27,8 @@ require ( github.com/streadway/amqp v1.0.0 github.com/stretchr/testify v1.8.0 github.com/wavefronthq/wavefront-sdk-go v0.10.3 + github.com/xitongsys/parquet-go v1.6.2 + github.com/xitongsys/parquet-go-source v0.0.0-20220723234337-052319f3f36b golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 google.golang.org/api v0.94.0 google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf @@ -54,6 +57,9 @@ require ( github.com/Microsoft/go-winio v0.5.2 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect + github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect + github.com/apache/thrift v0.14.2 // indirect + github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/caio/go-tdigest v3.1.0+incompatible // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect @@ -69,6 +75,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.3 // indirect github.com/google/gnostic v0.6.9 // indirect github.com/google/go-cmp v0.5.8 // indirect github.com/google/go-querystring v1.1.0 // indirect diff --git a/go.sum b/go.sum index 1036d13f9..07f79c202 100644 --- a/go.sum +++ b/go.sum @@ -78,9 +78,16 @@ github.com/Azure/azure-event-hubs-go/v3 v3.3.18 h1:jgWDk2qmknA0UsfyzjHiW5yciOw3a github.com/Azure/azure-event-hubs-go/v3 v3.3.18/go.mod h1:R5H325+EzgxcBDkUerEwtor7ZQg77G7HiOTwpcuIVXY= github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.1.9/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= +github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= github.com/Azure/azure-sdk-for-go v51.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v66.0.0+incompatible h1:bmmC38SlE8/E81nNADlgmVGurPWMHDX2YNXVQMrBpEE= github.com/Azure/azure-sdk-for-go v66.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.1/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.0.0/go.mod h1:+6sju8gk8FRmSajX3Oz4G5Gm7P+mbqE9FVaXXFYTkCM= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1/go.mod h1:eZ4g6GUvXiGulfIbbhh1Xr4XwUYaYaWMqzGD/284wCA= github.com/Azure/azure-storage-blob-go v0.6.0/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= github.com/Azure/go-amqp v0.17.0/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fwmw9Zlg= github.com/Azure/go-amqp v0.17.5 h1:7Lsi9H9ijCAfqOaMiNmQ4c+GL9bdrpCjebNKhV/eQ+c= @@ -126,6 +133,8 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= +github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0= github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= @@ -154,7 +163,12 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 h1:byKBBF2CKWBjjA4J1ZL2JXttJULvWSl50LegTyRZ728= +github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0= +github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.14.2 h1:hY4rAyg7Eqbb27GB6gkhUKrRAuc8xRjlNtJq+LseKeY= +github.com/apache/thrift v0.14.2/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/aquasecurity/kube-bench v0.6.0/go.mod h1:6nKPtl3GryEOixjlp8uo2fIvrV9oUxkNgMqzK9BHlgU= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -164,10 +178,26 @@ github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.35.28/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= github.com/aws/aws-sdk-go v1.44.89 h1:Xf5Pp9GsNSMRinAuWNiQd0vusXXb3IgYbNlxldhWS2Q= github.com/aws/aws-sdk-go v1.44.89/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go-v2 v1.7.1/go.mod h1:L5LuPC1ZgDr2xQS7AmIec/Jlc7O/Y1u2KxJyNVab250= +github.com/aws/aws-sdk-go-v2/config v1.5.0/go.mod h1:RWlPOAW3E3tbtNAqTwvSW54Of/yP3oiZXMI0xfUdjyA= +github.com/aws/aws-sdk-go-v2/credentials v1.3.1/go.mod h1:r0n73xwsIVagq8RsxmZbGSRQFj9As3je72C2WzUIToc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.3.0/go.mod h1:2LAuqPx1I6jNfaGDucWfA2zqQCYCOMCDHiCOciALyNw= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.3.2/go.mod h1:qaqQiHSrOUVOfKe6fhgQ6UzhxjwqVW8aHNegd6Ws4w4= +github.com/aws/aws-sdk-go-v2/internal/ini v1.1.1/go.mod h1:Zy8smImhTdOETZqfyn01iNOe0CNggVbPjCajyaz6Gvg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.1/go.mod h1:v33JQ57i2nekYTA70Mb+O18KeH4KqhdqxTJZNK1zdRE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.1/go.mod h1:zceowr5Z1Nh2WVP8bf/3ikB41IZW59E4yIYbg+pC6mw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.1/go.mod h1:6EQZIwNNvHpq/2/QSJnp4+ECvqIy55w95Ofs0ze+nGQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.11.1/go.mod h1:XLAGFrEjbvMCLvAtWLLP32yTv8GpBquCApZEycDLunI= +github.com/aws/aws-sdk-go-v2/service/sso v1.3.1/go.mod h1:J3A3RGUvuCZjvSuZEcOpHDnzZP/sKbhDWV2T1EOzFIM= +github.com/aws/aws-sdk-go-v2/service/sts v1.6.0/go.mod h1:q7o0j7d7HrJk/vr9uUt3BVRASvcU7gYZB9PUgPiByXg= +github.com/aws/smithy-go v1.6.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -202,6 +232,7 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -226,6 +257,7 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -236,6 +268,8 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP github.com/eclipse/paho.mqtt.golang v1.4.1 h1:tUSpviiL5G3P9SZZJPC4ZULZJsxQKXxfENpMvdbAXAI= github.com/eclipse/paho.mqtt.golang v1.4.1/go.mod h1:JGt0RsEwEX+Xa/agj90YJ9d9DH2b7upDZMK9HRbFvCA= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/embano1/memlog v0.4.3 h1:riVTumM3e3/qBrsKYsml7sSJYvCm6z4V4+4C3hRk0aA= +github.com/embano1/memlog v0.4.3/go.mod h1:yDNCaHBS5MqgHjF5SRlCibLaGqSrcC9DjG3AomLj+bk= github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= github.com/emersion/go-sasl v0.0.0-20211008083017-0b9dcfb154ac h1:tn/OQ2PmwQ0XFVgAHfjlLyqMewry25Rz7jWnVoh4Ggs= github.com/emersion/go-sasl v0.0.0-20211008083017-0b9dcfb154ac/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= @@ -324,6 +358,7 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= @@ -347,6 +382,7 @@ github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71 github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -366,10 +402,12 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= @@ -480,6 +518,7 @@ github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v0.0.0-20180228145832-27454136f036/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= @@ -513,9 +552,11 @@ github.com/jackc/pgx/v5 v5.0.4 h1:r5O6y84qHX/z/HZV40JBdx2obsHz7/uRj5b+CcYEdeY= github.com/jackc/pgx/v5 v5.0.4/go.mod h1:U0ynklHtgg43fue9Ly30w3OCSTDPlXjig9ghrNGaguQ= github.com/jackc/puddle/v2 v2.0.0 h1:Kwk/AlLigcnZsDssc3Zun1dk1tAtQNPaBBxBHWn0Mjc= github.com/jackc/puddle/v2 v2.0.0/go.mod h1:itE7ZJY8xnoo0JqJEpSMprN0f+NQkMCuEV/N9j8h0oc= +github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jinzhu/gorm v0.0.0-20160404144928-5174cc5c242a/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= github.com/jinzhu/inflection v0.0.0-20170102125226-1c35d901db3d/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -544,6 +585,10 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM= github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= @@ -561,6 +606,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kubernetes-sigs/wg-policy-prototypes/policy-report/kube-bench-adapter v0.0.0-20210714174227-a3d56502c383 h1:J9r2W6Nko09LP8H13VloEbNi3tBG9yeGpIcp6iOcQoI= github.com/kubernetes-sigs/wg-policy-prototypes/policy-report/kube-bench-adapter v0.0.0-20210714174227-a3d56502c383/go.mod h1:a55lp29S0A2gzERE+JrRYa9w/aC+hhKZ7pmuqG6cRU4= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leesper/go_rng v0.0.0-20190531154944-a612b043e353 h1:X/79QL0b4YJVO5+OsPH9rF2u428CIrGL/jLmPsoOQQ4= github.com/lib/pq v0.0.0-20171126050459-83612a56d3dd/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= @@ -580,6 +626,7 @@ github.com/mattn/go-colorable v0.0.0-20170210172801-5411d3eea597/go.mod h1:9vuHe github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.0-20170307163044-57fdcb988a5c/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -613,6 +660,8 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= +github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -640,6 +689,7 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS github.com/nats-io/stan.go v0.7.0/go.mod h1:Ci6mUIpGQTjl++MqK2XzkWI/0vF+Bl72uScx7ejSYmU= github.com/nats-io/stan.go v0.10.3 h1:8DOyQJ0+nza3zSVJZ19/cpikkrWA4rSKB3YvckIGOTI= github.com/nats-io/stan.go v0.10.3/go.mod h1:Cgf5zk6kKpOCqqUIJeuBz6ZDz9osT791VhS6m28sSQQ= +github.com/ncw/swift v1.0.52/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -672,6 +722,7 @@ github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9 github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= @@ -783,6 +834,7 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -811,6 +863,13 @@ github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xitongsys/parquet-go v1.5.1/go.mod h1:xUxwM8ELydxh4edHGegYq1pA8NnMKDx0K/GyB0o2bww= +github.com/xitongsys/parquet-go v1.6.2 h1:MhCaXii4eqceKPu9BwrjLqyK10oX9WF+xGhwvwbw7xM= +github.com/xitongsys/parquet-go v1.6.2/go.mod h1:IulAQyalCm0rPiZVNnCgm/PCL64X2tdSVGMQ/UeKqWA= +github.com/xitongsys/parquet-go-source v0.0.0-20190524061010-2b72cbee77d5/go.mod h1:xxCx7Wpym/3QCo6JhujJX51dzSXrwmb0oH6FQb39SEA= +github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0/go.mod h1:HYhIKsdns7xz80OgkbgJYrtQY7FjHWHKH6cvN7+czGE= +github.com/xitongsys/parquet-go-source v0.0.0-20220723234337-052319f3f36b h1:tA9vmqC+hxBElCuZLUQAw4TGMs9kXjF5UJNZHLQUJ/4= +github.com/xitongsys/parquet-go-source v0.0.0-20220723234337-052319f3f36b/go.mod h1:YFoRvz/hJ2HUiZGjjk3HfzHYS/Yt/R/K27cGKokTE28= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -851,6 +910,7 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= +golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -871,6 +931,7 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM= @@ -943,6 +1004,7 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -959,6 +1021,7 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -1050,6 +1113,7 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1432,6 +1496,11 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= +gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= +gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= +gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= diff --git a/handlers.go b/handlers.go index 6e710ad39..f05ffb711 100644 --- a/handlers.go +++ b/handlers.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "io" - "io/ioutil" "log" "net/http" "strings" @@ -12,6 +11,7 @@ import ( "time" "github.com/falcosecurity/falcosidekick/types" + "github.com/google/uuid" ) const testRule string = "Test rule" @@ -70,7 +70,7 @@ func healthHandler(w http.ResponseWriter, r *http.Request) { // testHandler sends a test event to all enabled outputs. func testHandler(w http.ResponseWriter, r *http.Request) { - r.Body = ioutil.NopCloser(bytes.NewReader([]byte(`{"output":"This is a test from falcosidekick","priority":"Debug","rule":"Test rule", "time":"` + time.Now().UTC().Format(time.RFC3339) + `","output_fields": {"proc.name":"falcosidekick","user.name":"falcosidekick"}, "tags":["test","example"]}`))) + r.Body = io.NopCloser(bytes.NewReader([]byte(`{"output":"This is a test from falcosidekick","priority":"Debug","rule":"Test rule", "time":"` + time.Now().UTC().Format(time.RFC3339) + `","output_fields": {"proc.name":"falcosidekick","user.name":"falcosidekick"}, "tags":["test","example"]}`))) mainHandler(w, r) } @@ -98,6 +98,8 @@ func newFalcoPayload(payload io.Reader) (types.FalcoPayload, error) { falcopayload.Source = "syscalls" } + falcopayload.UUID = uuid.New().String() + var kn, kp string for i, j := range falcopayload.OutputFields { if i == "k8s.ns.name" { @@ -230,6 +232,10 @@ func forwardEvent(falcopayload types.FalcoPayload) { go awsClient.UploadS3(falcopayload) } + if (config.AWS.SecurityLake.Bucket != "" && config.AWS.SecurityLake.Region != "" && config.AWS.SecurityLake.AccountID != "" && config.AWS.SecurityLake.Prefix != "") && (falcopayload.Priority >= types.Priority(config.AWS.SecurityLake.MinimumPriority) || falcopayload.Rule == testRule) { + go awsClient.EnqueueSecurityLake(falcopayload) + } + if config.AWS.Kinesis.StreamName != "" && (falcopayload.Priority >= types.Priority(config.AWS.Kinesis.MinimumPriority) || falcopayload.Rule == testRule) { go awsClient.PutRecord(falcopayload) } diff --git a/main.go b/main.go index 1237d0cf1..cd9170008 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "log" "net/http" @@ -9,6 +10,7 @@ import ( "strings" "github.com/DataDog/datadog-go/statsd" + "github.com/embano1/memlog" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -263,7 +265,7 @@ func init() { if config.AWS.Lambda.FunctionName != "" || config.AWS.SQS.URL != "" || config.AWS.SNS.TopicArn != "" || config.AWS.CloudWatchLogs.LogGroup != "" || config.AWS.S3.Bucket != "" || - config.AWS.Kinesis.StreamName != "" { + config.AWS.Kinesis.StreamName != "" || (config.AWS.SecurityLake.Bucket != "" && config.AWS.SecurityLake.Region != "" && config.AWS.SecurityLake.AccountID != "") { var err error awsClient, err = outputs.NewAWSClient(config, stats, promStats, statsdClient, dogstatsdClient) if err != nil { @@ -277,6 +279,9 @@ func init() { config.AWS.CloudWatchLogs.LogGroup = "" config.AWS.CloudWatchLogs.LogStream = "" config.AWS.Kinesis.StreamName = "" + config.AWS.SecurityLake.Region = "" + config.AWS.SecurityLake.Bucket = "" + config.AWS.SecurityLake.AccountID = "" } else { if config.AWS.Lambda.FunctionName != "" { outputs.EnabledOutputs = append(outputs.EnabledOutputs, "AWSLambda") @@ -296,6 +301,23 @@ func init() { if config.AWS.Kinesis.StreamName != "" { outputs.EnabledOutputs = append(outputs.EnabledOutputs, "AWSKinesis") } + if config.AWS.SecurityLake.Bucket != "" && config.AWS.SecurityLake.Region != "" && config.AWS.SecurityLake.AccountID != "" && config.AWS.SecurityLake.Prefix != "" { + config.AWS.SecurityLake.Ctx = context.Background() + config.AWS.SecurityLake.ReadOffset, config.AWS.SecurityLake.WriteOffset = new(memlog.Offset), new(memlog.Offset) + config.AWS.SecurityLake.Memlog, err = memlog.New(config.AWS.SecurityLake.Ctx, memlog.WithMaxSegmentSize(10000)) + if config.AWS.SecurityLake.Interval < 5 { + config.AWS.SecurityLake.Interval = 5 + } + go awsClient.StartSecurityLakeWorker() + if err != nil { + config.AWS.SecurityLake.Region = "" + config.AWS.SecurityLake.Bucket = "" + config.AWS.SecurityLake.AccountID = "" + config.AWS.SecurityLake.Prefix = "" + } else { + outputs.EnabledOutputs = append(outputs.EnabledOutputs, "AWSSecurityLake") + } + } } } diff --git a/outputs/aws.go b/outputs/aws.go index 2078578d4..22454e4bc 100644 --- a/outputs/aws.go +++ b/outputs/aws.go @@ -68,10 +68,12 @@ func NewAWSClient(config *types.Configuration, stats *types.Statistics, promStat return nil, errors.New("error while creating AWS Session") } - _, err = sts.New(sess).GetCallerIdentity(&sts.GetCallerIdentityInput{}) - if err != nil { - log.Printf("[ERROR] : AWS - Error while getting AWS Token: %v\n", err.Error()) - return nil, errors.New("error while getting AWS Token") + if config.AWS.CheckIdentity { + _, err = sts.New(sess).GetCallerIdentity(&sts.GetCallerIdentityInput{}) + if err != nil { + log.Printf("[ERROR] : AWS - Error while getting AWS Token: %v\n", err.Error()) + return nil, errors.New("error while getting AWS Token") + } } var endpointURL *url.URL diff --git a/outputs/awssecuritylake.go b/outputs/awssecuritylake.go new file mode 100644 index 000000000..d27064b2b --- /dev/null +++ b/outputs/awssecuritylake.go @@ -0,0 +1,304 @@ +package outputs + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/embano1/memlog" + "github.com/falcosecurity/falcosidekick/types" + "github.com/google/uuid" + "github.com/xitongsys/parquet-go-source/mem" + "github.com/xitongsys/parquet-go/writer" +) + +const ( + sevUnknown = iota + sevInformational + sevLow + sevMedium + sevHigh + sevCritical + sevFatal +) + +const schemaVersion = "0.1.0" + +// Security Finding [2001] Class +// https://schema.ocsf.io/classes/security_finding +type OCSFSecurityFinding struct { + // Attacks []OCSFAttack `json:"attacks,omitempty" parquet:"name=attacks, type=MAP, convertedtype=LIST, valuetype=BYTE_ARRAY, valueconvertedtype=UTF8"` + ActivityID int32 `json:"activity_id" parquet:"name=activity_id, type=INT32"` + ActivityName string `json:"activity_name" parquet:"name=activity_name, type=BYTE_ARRAY, convertedtype=UTF8"` + CategoryName string `json:"category_name" parquet:"name=category_name, type=BYTE_ARRAY, convertedtype=UTF8"` + CategoryUID int32 `json:"category_uid" parquet:"name=category_uid, type=INT32"` + ClassName string `json:"class_name" parquet:"name=classname, type=BYTE_ARRAY, convertedtype=UTF8"` + ClassUID int32 `json:"class_uid" parquet:"name=class_uid, type=INT32"` + Finding OCSFFIndingDetails `json:"finding" parquet:"name=finding"` + Message string `json:"message" parquet:"name=message, type=BYTE_ARRAY, convertedtype=UTF8"` + Metadata OCSFMetadata `json:"metadata" parquet:"name=metadata"` + Observables []OCSFObservable `json:"observables" parquet:"name=observables, repetitiontype=REPEATED"` + RawData string `json:"raw_data" parquet:"name=raw_data, type=BYTE_ARRAY, convertedtype=UTF8"` + Severity string `json:"severity" parquet:"name=severity, type=BYTE_ARRAY, convertedtype=UTF8"` + SeverityID int32 `json:"severity_id" parquet:"name=severity_id, type=INT32"` + State string `json:"state" parquet:"name=state, type=BYTE_ARRAY, convertedtype=UTF8"` + StateID int32 `json:"state_id" parquet:"name=state_id, type=INT32"` + Status string `json:"status" parquet:"name=status, type=BYTE_ARRAY, convertedtype=UTF8"` + Timestamp int64 `json:"time" parquet:"name=time, type=INT64"` + TypeName string `json:"type_name" parquet:"name=type_name, type=BYTE_ARRAY, convertedtype=UTF8"` + TypeUID int32 `json:"type_uid" parquet:"name=type_uid, type=INT32"` +} + +// // https://schema.ocsf.io/objects/attack +// type OCSFAttack struct { +// Tactics []string `json:"tactics" parquet:"name=tactics, type=MAP, convertedtype=LIST, valuetype=BYTE_ARRAY, valueconvertedtype=UTF8"` +// TechniqueUID int32 `json:"technique_uid" parquet:"name=technique_uid, type=INT32"` +// TechniqueName string `json:"technique_name" parquet:"name=technique_name, type=BYTE_ARRAY, convertedtype=UTF8"` +// } + +// func (o OCSFAttack) String() string { +// return fmt.Sprintf("{tactics:[%s], technique_uid:%v, technique:%s}", strings.Join(o.Tactics, ","), o.TechniqueUID, o.TechniqueName) +// } + +// https://schema.ocsf.io/objects/finding +type OCSFFIndingDetails struct { + CreatedTime int64 `json:"created_time" parquet:"name=created_time, type=INT64"` + Desc string `json:"desc" parquet:"name=desc, type=BYTE_ARRAY, convertedtype=UTF8"` + Title string `json:"title" parquet:"name=title, type=BYTE_ARRAY, convertedtype=UTF8"` + Types []string `json:"types" parquet:"name=types, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=REPEATED"` + UID string `json:"uid" parquet:"name=uid, type=BYTE_ARRAY, convertedtype=UTF8"` +} + +// https://schema.ocsf.io/objects/observable +type OCSFObservable struct { + Name string `json:"name" parquet:"name=name, type=BYTE_ARRAY, convertedtype=UTF8"` + Type string `json:"type" parquet:"name=type, type=BYTE_ARRAY, convertedtype=UTF8"` + TypeID int32 `json:"type_id" parquet:"name=type_id, type=INT32"` + Value string `json:"value" parquet:"name=value, type=BYTE_ARRAY, convertedtype=UTF8"` +} + +// https://schema.ocsf.io/objects/metadata +type OCSFMetadata struct { + Version string `json:"version" parquet:"name=version, type=BYTE_ARRAY, convertedtype=UTF8"` + Product OCSFProduct `json:"product" parquet:"name=product"` + Labels []string `json:"labels" parquet:"name=labels, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=REPEATED"` +} + +// https://schema.ocsf.io/objects/product +type OCSFProduct struct { + VendorName string `json:"vendor_name" parquet:"name=vendor_name, type=BYTE_ARRAY, convertedtype=UTF8"` + Name string `json:"name" parquet:"name=name, type=BYTE_ARRAY, convertedtype=UTF8"` +} + +func NewOCSFSecurityFinding(falcopayload types.FalcoPayload) OCSFSecurityFinding { + ocsfsf := OCSFSecurityFinding{ + ActivityID: 1, + ActivityName: "Generate", + CategoryUID: 2, + CategoryName: "Findings", + ClassName: "Security Finding", + ClassUID: 2001, + TypeUID: 200101, + TypeName: "Security Finding: Generate", + // Attacks: getMitreAttacke(falcopayload.Tags), + Metadata: OCSFMetadata{ + Labels: falcopayload.Tags, + Product: OCSFProduct{ + Name: "Falco", + VendorName: "Falcosecurity", + }, + Version: schemaVersion, + }, + RawData: falcopayload.String(), + State: "New", + StateID: 1, + Finding: OCSFFIndingDetails{ + CreatedTime: falcopayload.Time.UnixMilli(), + Desc: falcopayload.Output, + Title: falcopayload.Rule, + Types: []string{falcopayload.Source}, + UID: falcopayload.UUID, + }, + Message: falcopayload.Rule, + Observables: getObservables(falcopayload.Hostname, falcopayload.OutputFields), + Timestamp: falcopayload.Time.UnixMilli(), + Status: falcopayload.Priority.String(), + } + + ocsfsf.SeverityID, ocsfsf.Severity = getAWSSecurityLakeSeverity(falcopayload.Priority) + return ocsfsf +} + +func getObservables(hostname string, outputFields map[string]interface{}) []OCSFObservable { + ocsfobs := []OCSFObservable{} + + if hostname != "" { + ocsfobs = append(ocsfobs, OCSFObservable{ + Name: "hostname", + Type: "Other", + TypeID: 0, + Value: hostname, + }) + } + + for i, j := range outputFields { + switch j.(type) { + case string, int, int16, int32, float32, float64: + ocsfobs = append(ocsfobs, OCSFObservable{ + Name: i, + Type: "Other", + TypeID: 0, + Value: fmt.Sprintf("%v", j), + }) + default: + continue + } + } + return ocsfobs +} + +func getAWSSecurityLakeSeverity(priority types.PriorityType) (int32, string) { + switch priority { + case types.Debug, types.Informational: + return sevInformational, "Informational" + case types.Notice: + return sevLow, "Low" + case types.Warning: + return sevMedium, "Medium" + case types.Error: + return sevHigh, "High" + case types.Critical: + return sevCritical, "Critical" + case types.Alert, types.Emergency: + return sevFatal, "Fatal" + default: + return sevUnknown, "Uknown" + } +} + +// Todo if mitre tags are becoming more precise +// func getMitreAttack(tags []string) []OCSFAttack { +// ocsfa := []OCSFAttack{} +// for _, i := range tags { +// if ok := strings.HasPrefix(strings.ToLower(i), "mitre_"); !ok { +// continue +// } +// // todo +// } +// return ocsfa +// } + +func (c *Client) EnqueueSecurityLake(falcopayload types.FalcoPayload) { + offset, err := c.Config.AWS.SecurityLake.Memlog.Write(c.Config.AWS.SecurityLake.Ctx, []byte(falcopayload.String())) + if err != nil { + if err.Error() != "future offset" { + go c.CountMetric(Outputs, 1, []string{"output:awssecuritylake.", "status:error"}) + c.Stats.AWSSecurityLake.Add(Error, 1) + c.PromStats.Outputs.With(map[string]string{"destination": "awssecuritylake.", "status": Error}).Inc() + log.Printf("[ERROR] : %v SecurityLake. - %v\n", c.OutputType, err) + return + } + } + log.Printf("[INFO] : %v SecurityLake. - Event queued (%v)\n", c.OutputType, falcopayload.UUID) + *c.Config.AWS.SecurityLake.WriteOffset = offset +} + +func (c *Client) StartSecurityLakeWorker() { + for { + time.Sleep(time.Duration(c.Config.AWS.SecurityLake.Interval) * time.Minute) + // time.Sleep(5 * time.Second) + batch := make([]memlog.Record, c.Config.AWS.SecurityLake.BatchSize) + count, err := c.Config.AWS.SecurityLake.Memlog.ReadBatch(c.Config.AWS.SecurityLake.Ctx, *c.Config.AWS.SecurityLake.ReadOffset+1, batch) + if err != nil { + if err.Error() != "future offset" { + go c.CountMetric(Outputs, 1, []string{"output:awssecuritylake.", "status:error"}) + c.Stats.AWSSecurityLake.Add(Error, 1) + c.PromStats.Outputs.With(map[string]string{"destination": "awssecuritylake.", "status": Error}).Inc() + log.Printf("[ERROR] : %v SecurityLake. - %v\n", c.OutputType, err) + continue + } + } + if count > 0 { + var err error + *c.Config.AWS.SecurityLake.ReadOffset = batch[count-1].Metadata.Offset + uid := uuid.New().String() + if count == 1 { + err = c.writeParquet(uid, []memlog.Record{batch[0]}) + } + if count > 1 { + err = c.writeParquet(uid, batch[:count-1]) + } + if err != nil { + go c.CountMetric(Outputs, 1, []string{"output:awssecuritylake.", "status:error"}) + c.Stats.AWSSecurityLake.Add(Error, 1) + c.PromStats.Outputs.With(map[string]string{"destination": "awssecuritylake.", "status": Error}).Inc() + continue + } + if err != nil { + go c.CountMetric(Outputs, 1, []string{"output:awssecuritylake.", "status:ok"}) + c.Stats.AWSSecurityLake.Add(OK, 1) + c.PromStats.Outputs.With(map[string]string{"destination": "awssecuritylake.", "status": "ok"}).Inc() + } + } + } +} + +func (c *Client) writeParquet(uid string, records []memlog.Record) error { + fw, err := mem.NewMemFileWriter(uid+".parquet", func(name string, r io.Reader) error { + t := time.Now() + key := fmt.Sprintf("/%s/region=%s/accountId=%s/eventHour=%s/%s.parquet", c.Config.AWS.SecurityLake.Prefix, c.Config.AWS.SecurityLake.Region, c.Config.AWS.SecurityLake.AccountID, t.Format("2006010215"), uid) + ctx, cancelFn := context.WithTimeout(context.Background(), 10*time.Second) + defer cancelFn() + resp, err := s3.New(c.AWSSession).PutObjectWithContext(ctx, &s3.PutObjectInput{ + Bucket: aws.String(c.Config.AWS.SecurityLake.Bucket), + Key: aws.String(key), + Body: aws.ReadSeekCloser(r), + ContentType: aws.String("Apache Parquet"), + ACL: aws.String(s3.ObjectCannedACLBucketOwnerFullControl), + }) + if err != nil { + log.Printf("[ERROR] : %v SecurityLake. - Upload parquet file %s.parquet Failed: %v\n", c.OutputType, uid, err) + return err + } + if resp.SSECustomerAlgorithm != nil { + log.Printf("[INFO] : %v SecurityLake. - Upload parquet file %s.parquet OK (%v) (%v events) \n", c.OutputType, uid, *resp.SSECustomerKeyMD5, len(records)) + } else { + log.Printf("[INFO] : %v SecurityLake. - Upload parquet file %s.parquet OK (%v events)\n", c.OutputType, uid, len(records)) + } + return nil + }) + if err != nil { + log.Printf("[ERROR] : %v SecurityLake. - Can't create the parquet file %s.parquet: %v\n", c.OutputType, uid, err) + return err + } + pw, err := writer.NewParquetWriter(fw, new(OCSFSecurityFinding), 10) + if err != nil { + log.Printf("[ERROR] : %v SecurityLake. - Can't create the parquet writer: %v\n", c.OutputType, err) + return err + } + for _, i := range records { + var f types.FalcoPayload + if err := json.Unmarshal(i.Data, &f); err != nil { + log.Printf("[ERROR] : %v SecurityLake. - Unmarshalling error: %v\n", c.OutputType, err) + continue + } + o := NewOCSFSecurityFinding(f) + if err = pw.Write(o); err != nil { + log.Printf("[ERROR] : %v SecurityLake. - Parquet writer error: %v\n", c.OutputType, err) + continue + } + } + if err = pw.WriteStop(); err != nil { + log.Printf("[ERROR] : %v SecurityLake. - Can't stop the parquet writer: %v\n", c.OutputType, err) + } + if err = fw.Close(); err != nil { + log.Printf("[ERROR] : %v SecurityLake. - Can't close the parquet file %s.parquet: %v\n", c.OutputType, uid, err) + return err + } + return nil +} diff --git a/outputs/webui.go b/outputs/webui.go index e426cef91..be10e48be 100644 --- a/outputs/webui.go +++ b/outputs/webui.go @@ -4,7 +4,6 @@ import ( "log" "github.com/falcosecurity/falcosidekick/types" - "github.com/google/uuid" ) type WebUIPayload struct { @@ -13,8 +12,6 @@ type WebUIPayload struct { } func newWebUIPayload(falcopayload types.FalcoPayload, config *types.Configuration) WebUIPayload { - falcopayload.UUID = uuid.New().String() - return WebUIPayload{ Event: falcopayload, Outputs: EnabledOutputs, diff --git a/stats.go b/stats.go index d69c14209..479791ca7 100644 --- a/stats.go +++ b/stats.go @@ -40,6 +40,7 @@ func getInitStats() *types.Statistics { AWSSNS: getOutputNewMap("awssns"), AWSCloudWatchLogs: getOutputNewMap("awscloudwatchlogs"), AWSS3: getOutputNewMap("awss3"), + AWSSecurityLake: getOutputNewMap("awssecuritylake"), AWSKinesis: getOutputNewMap("awskinesis"), SMTP: getOutputNewMap("smtp"), Opsgenie: getOutputNewMap("opsgenie"), diff --git a/types/types.go b/types/types.go index cd646f103..1f6ecae99 100644 --- a/types/types.go +++ b/types/types.go @@ -1,11 +1,13 @@ package types import ( + "context" "encoding/json" "expvar" "text/template" "time" + "github.com/embano1/memlog" "github.com/prometheus/client_golang/prometheus" ) @@ -265,10 +267,12 @@ type awsOutputConfig struct { Region string AccessKeyID string SecretAccessKey string + CheckIdentity bool Lambda awsLambdaConfig SQS awsSQSConfig SNS awsSNSConfig S3 awsS3Config + SecurityLake awsSecurityLakeConfig CloudWatchLogs awsCloudWatchLogs Kinesis awsKinesisConfig } @@ -308,6 +312,20 @@ type awsKinesisConfig struct { MinimumPriority string } +type awsSecurityLakeConfig struct { + Bucket string + Region string + Prefix string + AccountID string + Interval uint + BatchSize uint + MinimumPriority string + Ctx context.Context + Memlog *memlog.Log + ReadOffset *memlog.Offset + WriteOffset *memlog.Offset +} + type smtpOutputConfig struct { HostPort string AuthMechanism string @@ -626,6 +644,7 @@ type Statistics struct { AWSSNS *expvar.Map AWSCloudWatchLogs *expvar.Map AWSS3 *expvar.Map + AWSSecurityLake *expvar.Map AWSKinesis *expvar.Map SMTP *expvar.Map Opsgenie *expvar.Map