Merge pull request #5 from lbausch/develop

Version 0.2.0
lbausch · Jun 3, 2024 · 3a832f2 · 3a832f2
2 parents 8700679 + 53873a6
commit 3a832f2
Show file tree

Hide file tree

Showing 7 changed files with 70 additions and 45 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -14,22 +14,8 @@ jobs:
       fail-fast: true
       matrix:
         version:
-          - 7.7.1
-          - 7.8.1
-          - 7.9.3
-          - 7.10.2
-          - 7.11.2
-          - 7.12.1
-          - 7.13.2
-          - 7.14.2
-          - 7.15.2
-          - 7.16.3
-          - 7.17.6
-          - 8.0.1
-          - 8.1.3
-          - 8.2.3
-          - 8.3.3
-          - 8.4.3
+          - 7.17.21
+          - 8.13.4
 
     runs-on: ubuntu-latest
     container:
@@ -38,7 +24,7 @@ jobs:
 
     steps:
       - name: Check Out Repository Code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Validate Configuration
         run: |
@@ -57,25 +43,11 @@ jobs:
       fail-fast: true
       matrix:
         version:
-          - 7.7.1
-          - 7.8.1
-          - 7.9.3
-          - 7.10.2
-          - 7.11.2
-          - 7.12.1
-          - 7.13.2
-          - 7.14.2
-          - 7.15.2
-          - 7.16.3
-          - 7.17.6
-          - 8.0.1
-          - 8.1.3
-          - 8.2.3
-          - 8.3.3
-          - 8.4.3
+          - 7.17.21
+          - 8.13.4
 
     runs-on: ubuntu-latest
-    container: python:3.10-bullseye
+    container: python:3.12-bookworm
 
     services:
       elasticsearch:
@@ -97,11 +69,11 @@ jobs:
           curl --silent ${{ env.ES_HOST }}/_cat/health
 
       - name: Check Out Repository Code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Run Unit Tests
         working-directory: tests
         run: |
-          pip3 install deepdiff==5.8.1
+          pip3 install deepdiff==6.7.1
           python MainPipeline.py --failfast --verbose
           python RejectPipeline.py --failfast --verbose
diff --git a/module/exim4/main/config/main.yml b/module/exim4/main/config/main.yml
@@ -1,14 +1,15 @@
-type: log
+type: filestream
+id: exim4-mainlog
 paths:
 {{ range $i, $path := .paths }}
  - {{$path}}
 {{ end }}
-exclude_files: [".gz$"]
+prospector.scanner.exclude_files: ['\.gz$']
 
 # https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-input-log.html#filebeat-input-log-include-lines
 include_lines:
   # Matches lines beginning with date and time followed by an optional timezone offset, the Exim message ID and the flag e.g:
   # 1970-01-01 13:37:00 +0100 aBc123-dEF456-H7 =>
   # https://www.exim.org/exim-html-current/doc/html/spec_html/ch-how_exim_receives_and_delivers_mail.html#SECTmessiden
   # https://www.exim.org/exim-html-current/doc/html/spec_html/ch-log_files.html#SECID250
-  - '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}( [+-]\d{4})? [0-9a-zA-Z]{6}-[0-9a-zA-Z]{6}-[0-9a-zA-Z]{2} (<=|=>|->|>>|==|\*\*)'
+  - '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}( [+-]\d{4})? [0-9a-zA-Z]{6}-[0-9a-zA-Z]{6,11}-[0-9a-zA-Z]{2,4} (<=|=>|->|>>|==|\*\*)'
diff --git a/module/exim4/main/ingest/pipeline.json b/module/exim4/main/ingest/pipeline.json
@@ -16,7 +16,7 @@
           "EXIM4_DATETIME": "%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND}",
           "EXIM4_TIMEZONE": "[+-]\\d{4}",
 
-          "EXIM4_ID": "[0-9a-zA-Z]{6}-[0-9a-zA-Z]{6}-[0-9a-zA-Z]{2}",
+          "EXIM4_ID": "[0-9a-zA-Z]{6}-[0-9a-zA-Z]{6,11}-[0-9a-zA-Z]{2,4}",
 
           "EXIM4_FLAG_RECEPTION": "<=",
           "EXIM4_FLAGS_DELIVERY": "=>|->|>>|==|\\*\\*",

diff --git a/module/exim4/reject/config/reject.yml b/module/exim4/reject/config/reject.yml
@@ -1,13 +1,14 @@
-type: log
+type: filestream
+id: exim4-rejectlog
 paths:
 {{ range $i, $path := .paths }}
  - {{$path}}
 {{ end }}
-exclude_files: [".gz$"]
+prospector.scanner.exclude_files: ['\.gz$']
 
 # https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-input-log.html#filebeat-input-log-include-lines
 include_lines:
   # Matches lines beginning with date and time followed by an optional timezone offset and an optional Exim message ID, e.g:
   # 1970-01-01 13:37:00 +0100 aBc123-dEF456-H7
   # https://www.exim.org/exim-html-current/doc/html/spec_html/ch-how_exim_receives_and_delivers_mail.html#SECTmessiden
-  - '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}( [+-]\d{4})?( [0-9a-zA-Z]{6}-[0-9a-zA-Z]{6}-[0-9a-zA-Z]{2})?'
+  - '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}( [+-]\d{4})?( [0-9a-zA-Z]{6}-[0-9a-zA-Z]{6,11}-[0-9a-zA-Z]{2,4})?'
diff --git a/module/exim4/reject/ingest/pipeline.json b/module/exim4/reject/ingest/pipeline.json
@@ -8,20 +8,21 @@
           "%{EXIM4_REJECT_LOG_LINE}"
         ],
         "pattern_definitions": {
-          "EXIM4_REJECT_LOG_LINE": "^%{EXIM4_DATETIME_TIMEZONE}( %{EXIM4_ID:exim4.id})?(%{SPACE}%{EXIM4_REJECT_FIELD})*( %{EXIM4_RECIPIENT_ADDRESS})? %{GREEDYDATA:exim4.message}",
+          "EXIM4_REJECT_LOG_LINE": "^%{EXIM4_DATETIME_TIMEZONE}( %{EXIM4_ID:exim4.id})?(%{SPACE}%{EXIM4_REJECT_FIELD})*( %{EXIM4_RECIPIENT_ADDRESS})?( %{EXIM4_AUTHENTICATOR})? %{GREEDYDATA:exim4.message}",
 
           "EXIM4_DATETIME_TIMEZONE": "%{EXIM4_DATETIME:exim4.datetime}( %{EXIM4_TIMEZONE:exim4.timezone})?",
           "EXIM4_DATETIME": "%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND}",
           "EXIM4_TIMEZONE": "[+-]\\d{4}",
 
-          "EXIM4_ID": "[0-9a-zA-Z]{6}-[0-9a-zA-Z]{6}-[0-9a-zA-Z]{2}",
+          "EXIM4_ID": "[0-9a-zA-Z]{6}-[0-9a-zA-Z]{6,11}-[0-9a-zA-Z]{2,4}",
 
           "EXIM4_REJECT_FIELD": "%{EXIM4_REMOTEHOST}|%{EXIM4_SENDER_ADDRESS}|%{EXIM4_TLS_CIPHER_SUITE}|%{EXIM4_CV}",
 
           "EXIM4_REMOTEHOST": "H=\\(?\\[?%{IPORHOST:exim4.remote_host}?\\]?\\)?(%{SPACE}\\(\\[?%{IPORHOST:exim4.helo_name}\\]?\\))? \\[%{IP:exim4.remote_addr}\\](:%{POSINT:exim4.remote_addr_port})?",
           "EXIM4_SENDER_ADDRESS": "F=<%{NOTSPACE:exim4.sender_address}>",
           "EXIM4_TLS_CIPHER_SUITE": "X=%{NOTSPACE:exim4.tls.cipher_suite}",
           "EXIM4_CV": "CV=%{WORD:exim4.tls.cert_verification_status}",
+          "EXIM4_AUTHENTICATOR": "A=%{NOTSPACE:exim4.authenticator}",
 
           "EXIM4_RECIPIENT_ADDRESS": "(temporarily )?rejected RCPT <%{NOTSPACE:exim4.recipient_address}>:"
         }

diff --git a/tests/MainPipeline.py b/tests/MainPipeline.py
@@ -18,6 +18,33 @@ def test_pipeline(self):
             }
         })
 
+    def test_exim497_message_id_format(self):
+        message = '2024-06-02 05:54:33 +0200 1sDcIh-000000004YY-3wAT => mail@example.com F=<sender@example.com> R=dnslookup T=remote_smtp H=example.com [127.0.0.1] X=TLS1.2:ECDHE-RSA-AES128-GCM-SHA256:128 CV=no DN="/CN=*.example.com" C="250 OK id=1sDcIj-00BWdE-0W"'
+
+        response = self.request(message)
+        source = self.source(response)
+
+        self.assertSourceEquals(source, {
+            '@timestamp': '2024-06-02T05:54:33.000+02:00',
+            'exim4': {
+                'message_raw': message,
+                'id': '1sDcIh-000000004YY-3wAT',
+                'flag': '=>',
+                'final_address': 'mail@example.com',
+                'sender_address': 'sender@example.com',
+                'router': 'dnslookup',
+                'transport': 'remote_smtp',
+                'remote_host': 'example.com',
+                'remote_addr': '127.0.0.1',
+                'tls': {
+                    'cert_verification_status': 'no',
+                    'cipher_suite': 'TLS1.2:ECDHE-RSA-AES128-GCM-SHA256:128'
+                },
+                'distinguished_name': '/CN=*.example.com',
+                'smtp_confirmation': '250 OK id=1sDcIj-00BWdE-0W',
+            },
+        })
+
     def test_local_user(self):
         message = '2021-05-04 13:37:00 +0100 1fnm7Z-00DoYa-KK <= localuser@host.tld U=localuser P=local S=1512 T="Cron <localuser@host> /usr/bin/wget http://foo.tld --output-document=/h" for recipient@remotehost.tld'
 

diff --git a/tests/RejectPipeline.py b/tests/RejectPipeline.py
@@ -18,6 +18,29 @@ def test_pipeline(self):
             }
         })
 
+    def test_exim497_message_id_format(self):
+        message = '2024-06-02 05:50:20 +0200 1sDcEe-000000004BC-0oYZ H=example.com [127.0.0.1] X=TLS1.2:ECDHE-RSA-AES256-GCM-SHA384:256 CV=no F=<sender@example.com> A=fixed_cram:foo rejected after DATA: spam'
+
+        response = self.request(message)
+        source = self.source(response)
+
+        self.assertSourceEquals(source, {
+            '@timestamp': '2024-06-02T05:50:20.000+02:00',
+            'exim4': {
+                'message_raw': message,
+                'id': '1sDcEe-000000004BC-0oYZ',
+                'remote_host': 'example.com',
+                'remote_addr': '127.0.0.1',
+                'tls': {
+                    'cert_verification_status': 'no',
+                    'cipher_suite': 'TLS1.2:ECDHE-RSA-AES256-GCM-SHA384:256'
+                },
+                'sender_address': 'sender@example.com',
+                'authenticator':  'fixed_cram:foo',
+                'message': 'rejected after DATA: spam',
+            },
+        })
+
     def test_greylisting(self):
         message = "2021-05-04 13:37:00 +0100 H=mail.remotehost.tld [123.123.123.123]:1337 X=TLSv1.2:ECDHE-RSA-AES128-GCM-SHA256:128 CV=no F=<mail@sender.tld> temporarily rejected RCPT <mail@recipient.tld>: Deferred due to greylisting. Host: '123.123.123.123' From: 'mail@sender.tld' To: 'mail@recipient.tld' SPF: 'none'"