Merge pull request #102 from datalust/dev

Release

README.md

@@ -519,9 +519,9 @@ cs(Referer) sc-status sc-substatus sc-win32-status sc-bytes cs-bytes time-taken
 The extraction pattern is wrapped in the example for display purposes, and must appear all in one string argument when invoked.
 
 ```shell
-seqcli ingest -i http.log --invalid-data=ignore -x "{@t:w3cdt} {ServerIP} {@m:={Method} {RequestPath} 
-{Query}} {Port:nat} {Username} {ClientIP} {UserAgent} {Referer} {StatusCode:nat} {Substatus:nat} 
+seqcli ingest -i http.log --invalid-data=ignore -x "{@t:w3cdt} {ServerIP} {@m:={Method} {RequestPath}} 
+{Query} {Port:nat} {Username} {ClientIP} {UserAgent} {Referer} {StatusCode:nat} {Substatus:nat} 
 {Win32Status:nat} {ResponseBytes:nat} {RequestBytes:nat} {Elapsed}{:n}"
 ```
 
-A nested `{@m:=` pattern is used to collect a substring of the event for display as the event's message.
+A nested `{@m:=` pattern is used to collect a substring of the log line for display as the event's message.

src/SeqCli/Cli/Commands/IngestCommand.cs

@@ -39,7 +39,7 @@ class IngestCommand : Command
         readonly PropertiesFeature _properties;
         readonly SendFailureHandlingFeature _sendFailureHandlingFeature;
         readonly ConnectionFeature _connection;
-        string _filter, _pattern = DefaultPattern;
+        string _filter, _pattern = DefaultPattern, _level, _message;
         bool _json;
 
         public IngestCommand(SeqConnectionFactory connectionFactory)
@@ -61,6 +61,16 @@ public IngestCommand(SeqConnectionFactory connectionFactory)
                 "Filter expression to select a subset of events",
                 v => _filter = string.IsNullOrWhiteSpace(v) ? null : v.Trim());
 
+            Options.Add(
+                "m=|message=",
+                "A message to associate with the ingested events; https://messagetemplates.org syntax is supported",
+                v => _message = string.IsNullOrWhiteSpace(v) ? null : v.Trim());
+
+            Options.Add("l=|level=",
+                "The level or severity to associate with the ingested events; this will override any " +
+                "level information present in the events themselves",
+                v => _level = string.IsNullOrWhiteSpace(v) ? null : v.Trim());
+
             _sendFailureHandlingFeature = Enable<SendFailureHandlingFeature>();            
             _connection = Enable<ConnectionFeature>();
         }
@@ -73,6 +83,9 @@ protected override async Task<int> Run()
                 foreach (var property in _properties.Properties)
                     enrichers.Add(new ScalarPropertyEnricher(property.Key, property.Value));
 
+                if (_level != null)
+                    enrichers.Add(new ScalarPropertyEnricher(SurrogateLevelProperty.PropertyName, _level));
+
                 Func<LogEvent, bool> filter = null;
                 if (_filter != null)
                 {
@@ -87,12 +100,16 @@ protected override async Task<int> Run()
                         (ILogEventReader)new JsonLogEventReader(input) :
                         new PlainTextLogEventReader(input, _pattern);
 
+                    reader = new EnrichingReader(reader, enrichers);
+
+                    if (_message != null)
+                        reader = new StaticMessageTemplateReader(reader, _message);
+
                     _connectionFactory.TryGetApiKey(_connection, out var apiKey);
                     return await LogShipper.ShipEvents(
                         _connectionFactory.Connect(_connection),
                         apiKey,
                         reader,
-                        enrichers,
                         _invalidDataHandlingFeature.InvalidDataHandling,
                         _sendFailureHandlingFeature.SendFailureHandling,
                         filter);

src/SeqCli/Ingestion/EnrichingReader.cs

@@ -0,0 +1,34 @@
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Serilog.Core;
+
+namespace SeqCli.Ingestion
+{
+    class EnrichingReader : ILogEventReader
+    {
+        readonly ILogEventReader _inner;
+        readonly IReadOnlyCollection<ILogEventEnricher> _enrichers;
+
+        public EnrichingReader(
+            ILogEventReader inner,
+            IReadOnlyCollection<ILogEventEnricher> enrichers)
+        {
+            _inner = inner ?? throw new ArgumentNullException(nameof(inner));
+            _enrichers = enrichers ?? throw new ArgumentNullException(nameof(enrichers));
+        }
+
+        public async Task<ReadResult> TryReadAsync()
+        {
+            var result = await _inner.TryReadAsync();
+
+            if (result.LogEvent != null)
+            {
+                foreach (var enricher in _enrichers)
+                    enricher.Enrich(result.LogEvent, null);
+            }
+
+            return result;
+        }
+    }
+}

src/SeqCli/Ingestion/ILogEventReader.cs

@@ -1,5 +1,4 @@
 using System.Threading.Tasks;
-using Serilog.Events;
 
 namespace SeqCli.Ingestion
 {

src/SeqCli/Ingestion/LogShipper.cs

@@ -23,7 +23,6 @@
 using SeqCli.Api;
 using SeqCli.Levels;
 using Serilog;
-using Serilog.Core;
 using Serilog.Events;
 
 namespace SeqCli.Ingestion
@@ -39,14 +38,12 @@ public static async Task<int> ShipEvents(
             SeqConnection connection,
             string apiKey,
             ILogEventReader reader,
-            List<ILogEventEnricher> enrichers,
             InvalidDataHandling invalidDataHandling,
             SendFailureHandling sendFailureHandling,
             Func<LogEvent, bool> filter = null)
         {
             if (connection == null) throw new ArgumentNullException(nameof(connection));
             if (reader == null) throw new ArgumentNullException(nameof(reader));
-            if (enrichers == null) throw new ArgumentNullException(nameof(enrichers));
 
             var batch = await ReadBatchAsync(reader, filter, BatchSize, invalidDataHandling);
             var retries = 0;
@@ -59,7 +56,6 @@ public static async Task<int> ShipEvents(
                         connection,
                         apiKey,
                         batch.LogEvents,
-                        enrichers,
                         sendFailureHandling != SendFailureHandling.Ignore);
                 }
                 catch (Exception ex)
@@ -138,7 +134,6 @@ static async Task<bool> SendBatchAsync(
             SeqConnection connection,
             string apiKey,
             IReadOnlyCollection<LogEvent> batch,
-            IReadOnlyCollection<ILogEventEnricher> enrichers,
             bool logSendFailures)
         {
             if (batch.Count == 0)
@@ -148,11 +143,7 @@ static async Task<bool> SendBatchAsync(
             using (var builder = new StringWriter())
             {
                 foreach (var evt in batch)
-                {
-                    foreach (var enricher in enrichers)
-                        enricher.Enrich(evt, null);
                     Formatter.Format(evt, builder);
-                }
 
                 content = new StringContent(builder.ToString(), Encoding.UTF8, ApiConstants.ClefMediatType);
             }

src/SeqCli/Ingestion/StaticMessageTemplateReader.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Serilog.Events;
+using Serilog.Parsing;
+
+namespace SeqCli.Ingestion
+{
+    class StaticMessageTemplateReader : ILogEventReader
+    {
+        readonly ILogEventReader _inner;
+        readonly MessageTemplate _messageTemplate;
+
+        public StaticMessageTemplateReader(ILogEventReader inner, string messageTemplate)
+        {
+            _inner = inner ?? throw new ArgumentNullException(nameof(inner));
+            _messageTemplate = new MessageTemplateParser().Parse(messageTemplate);
+        }
+
+        public async Task<ReadResult> TryReadAsync()
+        {
+            var result = await _inner.TryReadAsync();
+
+            if (result.LogEvent == null)
+                return result;
+
+            var evt = new LogEvent(
+                result.LogEvent.Timestamp,
+                result.LogEvent.Level,
+                result.LogEvent.Exception,
+                _messageTemplate,
+                result.LogEvent.Properties.Select(kv => new LogEventProperty(kv.Key, kv.Value)));
+
+            return new ReadResult(evt, result.IsAtEnd);
+        }
+    }
+}

src/SeqCli/PlainText/PlainTextLogEventReader.cs

@@ -7,7 +7,6 @@
 using SeqCli.PlainText.LogEvents;
 using SeqCli.PlainText.Parsers;
 using SeqCli.PlainText.Patterns;
-using Serilog.Events;
 
 namespace SeqCli.PlainText
 {

test/SeqCli.Tests/PlainText/StaticMessageTemplateReaderTests.cs

@@ -0,0 +1,21 @@
+using System.Threading.Tasks;
+using SeqCli.Ingestion;
+using SeqCli.Tests.Support;
+using Xunit;
+
+namespace SeqCli.Tests.PlainText
+{
+    public class StaticMessageTemplateReaderTests
+    {
+        [Fact]
+        public async Task ReaderSubstitutesMessageTemplate()
+        {
+            var evt = Some.LogEvent();
+            const string mt = "This is a message template";
+            var reader = new FixedLogEventReader(new ReadResult(evt, false));
+            var wrapper = new StaticMessageTemplateReader(reader, mt);
+            var result = await wrapper.TryReadAsync();
+            Assert.Equal(mt, result.LogEvent.MessageTemplate.Text);
+        }
+    }
+}

test/SeqCli.Tests/Support/FixedLogEventReader.cs

@@ -0,0 +1,20 @@
+using System.Threading.Tasks;
+using SeqCli.Ingestion;
+
+namespace SeqCli.Tests.Support
+{
+    class FixedLogEventReader : ILogEventReader
+    {
+        readonly ReadResult _result;
+
+        public FixedLogEventReader(ReadResult result)
+        {
+            _result = result;
+        }
+
+        public Task<ReadResult> TryReadAsync()
+        {
+            return Task.FromResult(_result);
+        }
+    }
+}

test/SeqCli.Tests/Support/Some.cs

@@ -0,0 +1,20 @@
+using System;
+using System.Linq;
+using Serilog.Events;
+using Serilog.Parsing;
+
+namespace SeqCli.Tests.Support
+{
+    static class Some
+    {
+        public static LogEvent LogEvent()
+        {
+            return new LogEvent(
+                DateTimeOffset.UtcNow, 
+                LogEventLevel.Information,
+                null,
+                new MessageTemplateParser().Parse("Test"),
+                Enumerable.Empty<LogEventProperty>());
+        }
+    }
+}