enhancement(Chronicle Unstructured): Add support for all Google SecOps regional endpoints

changelog.d/22007_chronicle_ingest_header.fix.md

@@ -0,0 +1,3 @@
+Set Content Encoding headers when configuring compression for the Chronicle unstructured log sink
+
+authors: chocpanda

changelog.d/add_chronicle_regional_endpoints.enhancement.md

@@ -0,0 +1,3 @@
+Add support for more chronicle regional endpoints as listed - https://cloud.google.com/chronicle/docs/reference/ingestion-api#regional_endpoints
+
+authors: chocpanda

src/sinks/gcp_chronicle/chronicle_unstructured.rs

@@ -5,7 +5,8 @@ use bytes::{Bytes, BytesMut};
 
 use futures_util::{future::BoxFuture, task::Poll};
 use goauth::scopes::Scope;
-use http::{header::HeaderValue, Request, StatusCode, Uri};
+use http::header::{self, HeaderName, HeaderValue};
+use http::{Request, StatusCode, Uri};
 use hyper::Body;
 use indoc::indoc;
 use serde::Serialize;
@@ -69,14 +70,56 @@ pub enum GcsHealthcheckError {
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 #[serde(rename_all = "snake_case")]
 pub enum Region {
-    /// EU region.
+    /// European Multi region - "https://europe-malachiteingestion-pa.googleapis.com".
     Eu,
 
-    /// US region.
+    /// US Multi region - "https://malachiteingestion-pa.googleapis.com".
     Us,
 
-    /// APAC region.
+    /// APAC region (this is the same as the Singapore region endpoint retained for backwards compatibility) - "https://asia-southeast1-malachiteingestion-pa.googleapis.com".
     Asia,
+
+    /// SãoPaulo Region - "https://southamerica-east1-malachiteingestion-pa.googleapis.com"
+    SãoPaulo,
+
+    /// Canada Region - "https://northamerica-northeast2-malachiteingestion-pa.googleapis.com"
+    Canada,
+
+    /// Dammam Region - "https://me-central2-malachiteingestion-pa.googleapis.com"
+    Dammam,
+
+    /// Doha Region - "https://me-central1-malachiteingestion-pa.googleapis.com"
+    Doha,
+
+    /// Frankfurt Region - "https://europe-west3-malachiteingestion-pa.googleapis.com"
+    Frankfurt,
+
+    /// London Region - "https://europe-west2-malachiteingestion-pa.googleapis.com"
+    London,
+
+    /// Mumbai Region - "https://asia-south1-malachiteingestion-pa.googleapis.com"
+    Mumbai,
+
+    /// Paris Region - "https://europe-west9-malachiteingestion-pa.googleapis.com"
+    Paris,
+
+    /// Singapore Region - "https://asia-southeast1-malachiteingestion-pa.googleapis.com"
+    Singapore,
+
+    /// Sydney Region - "https://australia-southeast1-malachiteingestion-pa.googleapis.com"
+    Sydney,
+
+    /// TelAviv Region - "https://me-west1-malachiteingestion-pa.googleapis.com"
+    TelAviv,
+
+    /// Tokyo Region - "https://asia-northeast1-malachiteingestion-pa.googleapis.com"
+    Tokyo,
+
+    /// Turin Region - "https://europe-west12-malachiteingestion-pa.googleapis.com"
+    Turin,
+
+    /// Zurich Region - "https://europe-west6-malachiteingestion-pa.googleapis.com"
+    Zurich,
 }
 
 impl Region {
@@ -86,6 +129,20 @@ impl Region {
             Region::Eu => "https://europe-malachiteingestion-pa.googleapis.com",
             Region::Us => "https://malachiteingestion-pa.googleapis.com",
             Region::Asia => "https://asia-southeast1-malachiteingestion-pa.googleapis.com",
+            Region::SãoPaulo => "https://southamerica-east1-malachiteingestion-pa.googleapis.com",
+            Region::Canada => "https://northamerica-northeast2-malachiteingestion-pa.googleapis.com",
+            Region::Dammam => "https://me-central2-malachiteingestion-pa.googleapis.com",
+            Region::Doha => "https://me-central1-malachiteingestion-pa.googleapis.com",
+            Region::Frankfurt => "https://europe-west3-malachiteingestion-pa.googleapis.com",
+            Region::London => "https://europe-west2-malachiteingestion-pa.googleapis.com",
+            Region::Mumbai => "https://asia-south1-malachiteingestion-pa.googleapis.com",
+            Region::Paris => "https://europe-west9-malachiteingestion-pa.googleapis.com",
+            Region::Singapore => "https://asia-southeast1-malachiteingestion-pa.googleapis.com",
+            Region::Sydney => "https://australia-southeast1-malachiteingestion-pa.googleapis.com",
+            Region::TelAviv => "https://me-west1-malachiteingestion-pa.googleapis.com",
+            Region::Tokyo => "https://asia-northeast1-malachiteingestion-pa.googleapis.com",
+            Region::Turin => "https://europe-west12-malachiteingestion-pa.googleapis.com",
+            Region::Zurich => "https://europe-west6-malachiteingestion-pa.googleapis.com",
         }
     }
 }
@@ -318,6 +375,7 @@ impl ChronicleUnstructuredConfig {
 pub struct ChronicleRequest {
     pub body: Bytes,
     pub finalizers: EventFinalizers,
+    pub headers: HashMap<HeaderName, HeaderValue>,
     metadata: RequestMetadata,
 }
 
@@ -471,7 +529,33 @@ impl RequestBuilder<(ChroniclePartitionKey, Vec<Event>)> for ChronicleRequestBui
         metadata: RequestMetadata,
         payload: EncodeResult<Self::Payload>,
     ) -> Self::Request {
-        ChronicleRequest {
+        let mut headers = HashMap::new();
+        headers.insert(
+            header::CONTENT_TYPE,
+            HeaderValue::from_static("application/json"),
+        );
+
+        match payload.compressed_byte_size {
+            Some(compressed_byte_size) => {
+                headers.insert(
+                    header::CONTENT_LENGTH,
+                    HeaderValue::from_str(&compressed_byte_size.to_string()).unwrap(),
+                );
+                headers.insert(
+                    header::CONTENT_ENCODING,
+                    HeaderValue::from_str(&self.compression.content_encoding().unwrap()).unwrap(),
+                );
+            }
+            None => {
+                headers.insert(
+                    header::CONTENT_LENGTH,
+                    HeaderValue::from_str(&payload.uncompressed_byte_size.to_string()).unwrap(),
+                );
+            }
+        }
+
+        return ChronicleRequest {
+            headers: headers,
             body: payload.into_payload().bytes,
             finalizers,
             metadata,
@@ -547,18 +631,13 @@ impl Service<ChronicleRequest> for ChronicleService {
 
     fn call(&mut self, request: ChronicleRequest) -> Self::Future {
         let mut builder = Request::post(&self.base_url);
-        let headers = builder.headers_mut().unwrap();
-        headers.insert(
-            "content-type",
-            HeaderValue::from_str("application/json").unwrap(),
-        );
-        headers.insert(
-            "content-length",
-            HeaderValue::from_str(&request.body.len().to_string()).unwrap(),
-        );
-
         let metadata = request.get_metadata().clone();
 
+        let headers = builder.headers_mut().unwrap();
+        for (name, value) in request.headers {
+            headers.insert(name, value);
+        }
+
         let mut http_request = builder.body(Body::from(request.body)).unwrap();
         self.creds.apply(&mut http_request);