Loosen Nginx error log regex matching (#442)

* Loosen Nginx error log regex matching Since the fields can be empty in some circustances, it is better to use '*' instead of '+' to match the values. * Update changelog & Add tests
vectordotdev · Sep 13, 2023 · e6ca833 · e6ca833
1 parent 5b1792c
commit e6ca833
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 5 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,7 @@
 
 ## unreleased
 
+- `parse_nginx_log` doesn't fail if the values of key-value pairs in error logs is missing (https://github.com/vectordotdev/vrl/pull/442)
 - `parse_key_value` can now parse values enclosed in single quote characters (https://github.com/vectordotdev/vrl/pull/382)
 - added `pretty` parameter for `encode_json` vrl function to produce pretty-printed JSON string (https://github.com/vectordotdev/vrl/pull/370)
 - `encode_gzip` and `encode_zlib` now correctly check the compression level (preventing a panic) (https://github.com/vectordotdev/vrl/pull/393)

diff --git a/src/stdlib/log_util.rs b/src/stdlib/log_util.rs
@@ -163,11 +163,11 @@ pub(crate) static REGEX_NGINX_ERROR_LOG: Lazy<Regex> = Lazy::new(|| {
         \s+(?P<message>[^,]*)                                                    # Match any character
         (,\s+excess:\s+(?P<excess>[^\s]+)\sby\szone\s"(?P<zone>[^,]+)")?         # Match any character after ', excess: ' until ' by zone ' and the rest of characters
         (,\s+client:\s+(?P<client>[^,]+))?                                       # Match any character after ', client: '
-        (,\s+server:\s+(?P<server>[^,]+))?                                       # Match any character after ', server: '
-        (,\s+request:\s+"(?P<request>[^"]+)")?                                   # Match any character after ', request: '
-        (,\s+upstream:\s+"(?P<upstream>[^"]+)")?                                 # Match any character after ', upstream: '
-        (,\s+host:\s+"(?P<host>[^"]+)")?                                         # Match any character then ':' then any character after ', host: '
-        (,\s+refer?rer:\s+"(?P<referer>[^"]+)")?                                 # Match any character after ', referrer: '
+        (,\s+server:\s+(?P<server>[^,]*))?                                       # Match any character after ', server: '
+        (,\s+request:\s+"(?P<request>[^"]*)")?                                   # Match any character after ', request: '
+        (,\s+upstream:\s+"(?P<upstream>[^"]*)")?                                 # Match any character after ', upstream: '
+        (,\s+host:\s+"(?P<host>[^"]*)")?                                         # Match any character then ':' then any character after ', host: '
+        (,\s+refer?rer:\s+"(?P<referer>[^"]*)")?                                 # Match any character after ', referrer: '
         \s*$                                                                     # Match any number of whitespaces (to be discarded).
     "#)
     .expect("failed compiling regex for Nginx error log")

diff --git a/src/stdlib/parse_nginx_log.rs b/src/stdlib/parse_nginx_log.rs
@@ -422,6 +422,28 @@ mod tests {
             tdef: TypeDef::object(kind_error()).fallible(),
         }
 
+        error_line_with_empty_values {
+            args: func_args![
+                value: r#"2023/09/08 13:50:28 [warn] 3#3: *531 an upstream response is buffered to a temporary file /var/lib/nginx/tmp/fastcgi/6/03/0000000036 while reading upstream, client: 10.224.1.1, server: , request: "GET / HTTP/1.1", upstream: "fastcgi://127.0.0.1:9000", host: "", referrer: """#,
+                format: "error"
+            ],
+            want: Ok(btreemap! {
+                "timestamp" => Value::Timestamp(DateTime::parse_from_rfc3339("2023-09-08T13:50:28Z").unwrap().into()),
+                "severity" => "warn",
+                "pid" => 3,
+                "tid" => 3,
+                "cid" => 531,
+                "message" => "an upstream response is buffered to a temporary file /var/lib/nginx/tmp/fastcgi/6/03/0000000036 while reading upstream",
+                "client" => "10.224.1.1",
+                "server" => "",
+                "request" => "GET / HTTP/1.1",
+                "upstream" => "fastcgi://127.0.0.1:9000",
+                "host" => "",
+                "referer" => "",
+            }),
+            tdef: TypeDef::object(kind_error()).fallible(),
+        }
+
         error_line_with_upstream {
             args: func_args![
                 value: r#"2022/04/15 08:16:13 [error] 7164#7164: *20 connect() failed (113: No route to host) while connecting to upstream, client: 10.244.0.0, server: test.local, request: "GET / HTTP/2.0", upstream: "http://127.0.0.1:80/""#,