enhancement: allow more missing upstream info in parse_nginx_log (#498)

* Allow more more missing upstream info in parse_nginx_log * Add an entry to changelog * Change wording of the changelog entry
vectordotdev · Oct 10, 2023 · a8469ad · a8469ad
1 parent 1be5db9
commit a8469ad
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 3 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,8 @@
 
 ## unreleased
 
+- `parse_nginx_log` no longer fails if `upstream_response_length`, `upstream_response_time`, `upstream_status` are missing (https://github.com/vectordotdev/vrl/pull/498)
+
 #### Features
 - 'from_unix_timestamp' now accepts a new unit: Microseconds.
 

diff --git a/src/stdlib/log_util.rs b/src/stdlib/log_util.rs
@@ -142,9 +142,9 @@ pub(crate) static REGEX_INGRESS_NGINX_UPSTREAMINFO_LOG: Lazy<Regex> = Lazy::new(
         \[(?P<proxy_upstream_name>[^\]]+)\]\s+              # Match all characters within square brackets
         \[(?P<proxy_alternative_upstream_name>[^\]]+)?\]\s+ # Match all characters within square brackets, optional
         (?P<upstream_addr>\S+)\s+                           # Match any non space character
-        (?P<upstream_response_length>\d+)\s+                # Match numbers
-        (?P<upstream_response_time>\d+\.\d+)\s+             # Match numbers with dot
-        (?P<upstream_status>\d+)\s+                         # Match numbers
+        (-|(?P<upstream_response_length>\d+))\s+            # Match `-` or numbers
+        (-|(?P<upstream_response_time>\d+\.\d+))\s+         # Match `-` or numbers with dot
+        (-|(?P<upstream_status>\d+))\s+                     # Match `-` or numbers
         (?P<req_id>\S+)                                     # Match any non space character
         \s*$                                                # Match any number of whitespaces (to be discarded).
     "#)

diff --git a/src/stdlib/parse_nginx_log.rs b/src/stdlib/parse_nginx_log.rs
@@ -339,6 +339,28 @@ mod tests {
             tdef: TypeDef::object(kind_ingress_upstreaminfo()).fallible(),
         }
 
+        ingress_nginx_upstreaminfo_valid_missing_upstream {
+            args: func_args![
+                value: r#"0.0.0.0 - - [18/Mar/2023:15:00:00 +0000] "GET /some/path HTTP/2.0" 200 12312 "https://10.0.0.1/some/referer" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36" 462 0.050 [some-upstream-service-9000] [] - - - - 752178adb17130b291aefd8c386279e7"#,
+                format: "ingress_upstreaminfo"
+            ],
+            want: Ok(btreemap! {
+                "remote_addr" => "0.0.0.0",
+                "timestamp" => Value::Timestamp(DateTime::parse_from_rfc3339("2023-03-18T15:00:00Z").unwrap().into()),
+                "request" => "GET /some/path HTTP/2.0",
+                "status" => 200,
+                "body_bytes_size" => 12312,
+                "http_referer" => "https://10.0.0.1/some/referer",
+                "http_user_agent" => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36",
+                "request_length" => 462,
+                "request_time" => 0.050,
+                "upstream_addr" => "-",
+                "proxy_upstream_name" => "some-upstream-service-9000",
+                "req_id" => "752178adb17130b291aefd8c386279e7",
+            }),
+            tdef: TypeDef::object(kind_ingress_upstreaminfo()).fallible(),
+        }
+
         ingress_nginx_upstreaminfo_valid_all_fields {
             args: func_args![
                 value: r#"0.0.0.0 - bob [18/Mar/2023:15:00:00 +0000] "GET /some/path HTTP/2.0" 200 12312 "https://10.0.0.1/some/referer" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36" 462 0.050 [some-upstream-service-9000] [some-other-upstream-5000] 10.0.50.80:9000 19437 0.049 200 752178adb17130b291aefd8c386279e7"#,