From 0a70104ddb5a11f94f5ed924a3179b680d721a89 Mon Sep 17 00:00:00 2001 From: Matt Searle Date: Tue, 10 Dec 2024 17:37:06 +0100 Subject: [PATCH 1/3] fix(#22007): add content encoding header when compression is enabled --- .../gcp_chronicle/chronicle_unstructured.rs | 47 ++++++++++++++----- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/src/sinks/gcp_chronicle/chronicle_unstructured.rs b/src/sinks/gcp_chronicle/chronicle_unstructured.rs index aba02d742452e..7ca16fe88385c 100644 --- a/src/sinks/gcp_chronicle/chronicle_unstructured.rs +++ b/src/sinks/gcp_chronicle/chronicle_unstructured.rs @@ -5,7 +5,8 @@ use bytes::{Bytes, BytesMut}; use futures_util::{future::BoxFuture, task::Poll}; use goauth::scopes::Scope; -use http::{header::HeaderValue, Request, StatusCode, Uri}; +use http::header::{self, HeaderName, HeaderValue}; +use http::{Request, StatusCode, Uri}; use hyper::Body; use indoc::indoc; use serde::Serialize; @@ -318,6 +319,7 @@ impl ChronicleUnstructuredConfig { pub struct ChronicleRequest { pub body: Bytes, pub finalizers: EventFinalizers, + pub headers: HashMap, metadata: RequestMetadata, } @@ -471,7 +473,33 @@ impl RequestBuilder<(ChroniclePartitionKey, Vec)> for ChronicleRequestBui metadata: RequestMetadata, payload: EncodeResult, ) -> Self::Request { - ChronicleRequest { + let mut headers = HashMap::new(); + headers.insert( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + ); + + match payload.compressed_byte_size { + Some(compressed_byte_size) => { + headers.insert( + header::CONTENT_LENGTH, + HeaderValue::from_str(&compressed_byte_size.to_string()).unwrap(), + ); + headers.insert( + header::CONTENT_ENCODING, + HeaderValue::from_str(&self.compression.content_encoding().unwrap()).unwrap(), + ); + } + None => { + headers.insert( + header::CONTENT_LENGTH, + HeaderValue::from_str(&payload.uncompressed_byte_size.to_string()).unwrap(), + ); + } + } + + return ChronicleRequest { + headers: headers, body: payload.into_payload().bytes, finalizers, metadata, @@ -547,18 +575,13 @@ impl Service for ChronicleService { fn call(&mut self, request: ChronicleRequest) -> Self::Future { let mut builder = Request::post(&self.base_url); - let headers = builder.headers_mut().unwrap(); - headers.insert( - "content-type", - HeaderValue::from_str("application/json").unwrap(), - ); - headers.insert( - "content-length", - HeaderValue::from_str(&request.body.len().to_string()).unwrap(), - ); - let metadata = request.get_metadata().clone(); + let headers = builder.headers_mut().unwrap(); + for (name, value) in request.headers { + headers.insert(name, value); + } + let mut http_request = builder.body(Body::from(request.body)).unwrap(); self.creds.apply(&mut http_request); From a6e3efa60e47781da9fe61beedda3ad274940b87 Mon Sep 17 00:00:00 2001 From: Matt Searle Date: Fri, 13 Dec 2024 22:29:56 +0100 Subject: [PATCH 2/3] docs(#22007): add change log file --- changelog.d/22007_chronicle_ingest_header.fix.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog.d/22007_chronicle_ingest_header.fix.md diff --git a/changelog.d/22007_chronicle_ingest_header.fix.md b/changelog.d/22007_chronicle_ingest_header.fix.md new file mode 100644 index 0000000000000..f8c184afe6fdc --- /dev/null +++ b/changelog.d/22007_chronicle_ingest_header.fix.md @@ -0,0 +1,3 @@ +Set Content Encoding headers when configuring compression for the Chronicle unstructured log sink + +authors: chocpanda From 5ef0dff56a65575f3445340273be96e7d1422693 Mon Sep 17 00:00:00 2001 From: Matt Searle Date: Fri, 13 Dec 2024 23:06:05 +0100 Subject: [PATCH 3/3] feat(chronicle-endpoints): add support for all Google SecOps regions endpoints in the chronicle unstructured log sink --- ...hronicle_regional_endpoints.enhancement.md | 3 + .../gcp_chronicle/chronicle_unstructured.rs | 62 ++++++++++++++++++- 2 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 changelog.d/add_chronicle_regional_endpoints.enhancement.md diff --git a/changelog.d/add_chronicle_regional_endpoints.enhancement.md b/changelog.d/add_chronicle_regional_endpoints.enhancement.md new file mode 100644 index 0000000000000..91e03f2fd8fba --- /dev/null +++ b/changelog.d/add_chronicle_regional_endpoints.enhancement.md @@ -0,0 +1,3 @@ +Add support for more chronicle regional endpoints as listed - https://cloud.google.com/chronicle/docs/reference/ingestion-api#regional_endpoints + +authors: chocpanda diff --git a/src/sinks/gcp_chronicle/chronicle_unstructured.rs b/src/sinks/gcp_chronicle/chronicle_unstructured.rs index 7ca16fe88385c..312778a3ff62e 100644 --- a/src/sinks/gcp_chronicle/chronicle_unstructured.rs +++ b/src/sinks/gcp_chronicle/chronicle_unstructured.rs @@ -70,14 +70,56 @@ pub enum GcsHealthcheckError { #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[serde(rename_all = "snake_case")] pub enum Region { - /// EU region. + /// European Multi region - "https://europe-malachiteingestion-pa.googleapis.com". Eu, - /// US region. + /// US Multi region - "https://malachiteingestion-pa.googleapis.com". Us, - /// APAC region. + /// APAC region (this is the same as the Singapore region endpoint retained for backwards compatibility) - "https://asia-southeast1-malachiteingestion-pa.googleapis.com". Asia, + + /// SãoPaulo Region - "https://southamerica-east1-malachiteingestion-pa.googleapis.com" + SãoPaulo, + + /// Canada Region - "https://northamerica-northeast2-malachiteingestion-pa.googleapis.com" + Canada, + + /// Dammam Region - "https://me-central2-malachiteingestion-pa.googleapis.com" + Dammam, + + /// Doha Region - "https://me-central1-malachiteingestion-pa.googleapis.com" + Doha, + + /// Frankfurt Region - "https://europe-west3-malachiteingestion-pa.googleapis.com" + Frankfurt, + + /// London Region - "https://europe-west2-malachiteingestion-pa.googleapis.com" + London, + + /// Mumbai Region - "https://asia-south1-malachiteingestion-pa.googleapis.com" + Mumbai, + + /// Paris Region - "https://europe-west9-malachiteingestion-pa.googleapis.com" + Paris, + + /// Singapore Region - "https://asia-southeast1-malachiteingestion-pa.googleapis.com" + Singapore, + + /// Sydney Region - "https://australia-southeast1-malachiteingestion-pa.googleapis.com" + Sydney, + + /// TelAviv Region - "https://me-west1-malachiteingestion-pa.googleapis.com" + TelAviv, + + /// Tokyo Region - "https://asia-northeast1-malachiteingestion-pa.googleapis.com" + Tokyo, + + /// Turin Region - "https://europe-west12-malachiteingestion-pa.googleapis.com" + Turin, + + /// Zurich Region - "https://europe-west6-malachiteingestion-pa.googleapis.com" + Zurich, } impl Region { @@ -87,6 +129,20 @@ impl Region { Region::Eu => "https://europe-malachiteingestion-pa.googleapis.com", Region::Us => "https://malachiteingestion-pa.googleapis.com", Region::Asia => "https://asia-southeast1-malachiteingestion-pa.googleapis.com", + Region::SãoPaulo => "https://southamerica-east1-malachiteingestion-pa.googleapis.com", + Region::Canada => "https://northamerica-northeast2-malachiteingestion-pa.googleapis.com", + Region::Dammam => "https://me-central2-malachiteingestion-pa.googleapis.com", + Region::Doha => "https://me-central1-malachiteingestion-pa.googleapis.com", + Region::Frankfurt => "https://europe-west3-malachiteingestion-pa.googleapis.com", + Region::London => "https://europe-west2-malachiteingestion-pa.googleapis.com", + Region::Mumbai => "https://asia-south1-malachiteingestion-pa.googleapis.com", + Region::Paris => "https://europe-west9-malachiteingestion-pa.googleapis.com", + Region::Singapore => "https://asia-southeast1-malachiteingestion-pa.googleapis.com", + Region::Sydney => "https://australia-southeast1-malachiteingestion-pa.googleapis.com", + Region::TelAviv => "https://me-west1-malachiteingestion-pa.googleapis.com", + Region::Tokyo => "https://asia-northeast1-malachiteingestion-pa.googleapis.com", + Region::Turin => "https://europe-west12-malachiteingestion-pa.googleapis.com", + Region::Zurich => "https://europe-west6-malachiteingestion-pa.googleapis.com", } } }