fix(forwarder): set content-encoding on override (#305)

We currently rely on the S3 object being written with the correct
content-encoding. There are many cases, such as `vpcflowlogs`, where the
content-type is set to `gz` and the content-encoding is left blank.

This commit ensures we always set the content-encoding correctly for
filenames that end in `.gz`

pkg/handler/forwarder/override/presets/aws/v1.yaml

@@ -10,24 +10,28 @@
     source: '/cloudwatchmetrics/[a-z\d-]+/json/\d{4}/\d{2}/\d{2}/\d{2}'
   override:
     content-type: 'application/x-aws-cloudwatchmetrics'
+    # note, cloudwatchmetrics payloads are _not_ gzipped
 
 - id: configSnapshot
   match:
     source: '\d{12}_Config_[a-z\d-]+_ConfigSnapshot_\d{8}T\d{6}Z_[a-f\d-]+\.json\.gz$'
   override:
     content-type: 'application/x-aws-config'
+    content-encoding: 'gzip'
 
 - id: configHistory
   match:
     source: '\d{12}_Config_[a-z\d-]+_ConfigHistory_[^\.]+.json\.gz$'
   override:
     content-type: 'application/x-aws-config'
+    content-encoding: 'gzip'
 
 - id: configChangeNotification
   match:
     source: '\d{12}_Config_[a-z\d-]+_ChangeNotification_AWS\S+_\d{8}T\d{6}Z_[a-f\d-]+\.json\.gz$'
   override:
     content-type: 'application/x-aws-change'
+    content-encoding: 'gzip'
 
 - id: cloudtrail
   match:
@@ -40,6 +44,7 @@
     source: '\d{12}_vpcflowlogs_[a-z\d-]+_[a-zA-Z0-9-]+_\d{8}T\d{4}Z_[a-zA-Z0-9-]+\.log\.gz$'
   override:
     content-type: 'application/x-aws-vpcflowlogs'
+    content-encoding: 'gzip'
 
 - id: awsTestObject
   match:
@@ -52,3 +57,4 @@
     source: '\d{12}_elasticloadbalancing_[a-z\d-]+_[a-zA-Z0-9-]+_\d{8}T\d{4}Z_[0-9.]+_[a-zA-Z0-9]+\.log\.gz$'
   override:
     content-type: 'application/x-aws-elasticloadbalancing'
+    content-encoding: 'gzip'

pkg/handler/forwarder/override/presets_test.go

@@ -62,8 +62,7 @@ func TestPresets(t *testing.T) {
 				},
 				{
 					Input: &s3.CopyObjectInput{
-						CopySource:      aws.String("test-bucket/AWSLogs/123456789012/vpcflowlogs/eu-central-1/2024/04/18/123456789012_vpcflowlogs_eu-central-1_fl-0d867ec290a114c9d_20240418T2155Z_9b1b75d1.log.gz"),
-						ContentEncoding: aws.String("gzip"),
+						CopySource: aws.String("test-bucket/AWSLogs/123456789012/vpcflowlogs/eu-central-1/2024/04/18/123456789012_vpcflowlogs_eu-central-1_fl-0d867ec290a114c9d_20240418T2155Z_9b1b75d1.log.gz"),
 					},
 					Expect: &s3.CopyObjectInput{
 						CopySource:        aws.String("test-bucket/AWSLogs/123456789012/vpcflowlogs/eu-central-1/2024/04/18/123456789012_vpcflowlogs_eu-central-1_fl-0d867ec290a114c9d_20240418T2155Z_9b1b75d1.log.gz"),
@@ -90,6 +89,7 @@ func TestPresets(t *testing.T) {
 					Expect: &s3.CopyObjectInput{
 						CopySource:        aws.String("test-bucket/AWSLogs/123456789012/elasticloadbalancing/us-west-2/2024/05/23/123456789012_elasticloadbalancing_us-west-2_ac5f85808922711e98f8e02481e016be_20240523T0015Z_127.69.70.85_5cpyxp74.log.gz"),
 						ContentType:       aws.String("application/x-aws-elasticloadbalancing"),
+						ContentEncoding:   aws.String("gzip"),
 						MetadataDirective: types.MetadataDirectiveReplace,
 					},
 				},