A simple wildcard implementation for ingest based on Directory.GetFiles()

src/SeqCli/Cli/Commands/IngestCommand.cs

@@ -28,7 +28,7 @@
 namespace SeqCli.Cli.Commands
 {
     [Command("ingest", "Send log events from a file or `STDIN`",
-        Example = "seqcli ingest -i events.txt --json --filter=\"@Level <> 'Debug'\" -p Environment=Test")]
+        Example = "seqcli ingest -i log-*.txt --json --filter=\"@Level <> 'Debug'\" -p Environment=Test")]
     class IngestCommand : Command
     {
         const string DefaultPattern = "{@m:line}";
@@ -45,7 +45,7 @@ class IngestCommand : Command
         public IngestCommand(SeqConnectionFactory connectionFactory)
         {
             _connectionFactory = connectionFactory;
-            _fileInputFeature = Enable(new FileInputFeature("File to ingest"));
+            _fileInputFeature = Enable(new FileInputFeature("File to ingest", supportsWildcard: true));
             _invalidDataHandlingFeature = Enable<InvalidDataHandlingFeature>();
             _properties = Enable<PropertiesFeature>();
 
@@ -94,26 +94,36 @@ protected override async Task<int> Run()
                     filter = evt => true.Equals(eval(evt));
                 }
 
-                using (var input = _fileInputFeature.OpenInput())
+                var connection = _connectionFactory.Connect(_connection);
+
+                foreach (var input in _fileInputFeature.OpenInputs())
                 {
-                    var reader = _json ?
-                        (ILogEventReader)new JsonLogEventReader(input) :
-                        new PlainTextLogEventReader(input, _pattern);
-
-                    reader = new EnrichingReader(reader, enrichers);
-
-                    if (_message != null)
-                        reader = new StaticMessageTemplateReader(reader, _message);
-
-                    _connectionFactory.TryGetApiKey(_connection, out var apiKey);
-                    return await LogShipper.ShipEvents(
-                        _connectionFactory.Connect(_connection),
-                        apiKey,
-                        reader,
-                        _invalidDataHandlingFeature.InvalidDataHandling,
-                        _sendFailureHandlingFeature.SendFailureHandling,
-                        filter);
+                    using (input)
+                    {
+                        var reader = _json
+                            ? (ILogEventReader) new JsonLogEventReader(input)
+                            : new PlainTextLogEventReader(input, _pattern);
+
+                        reader = new EnrichingReader(reader, enrichers);
+
+                        if (_message != null)
+                            reader = new StaticMessageTemplateReader(reader, _message);
+
+                        _connectionFactory.TryGetApiKey(_connection, out var apiKey);
+                        var exit = await LogShipper.ShipEvents(
+                            connection,
+                            apiKey,
+                            reader,
+                            _invalidDataHandlingFeature.InvalidDataHandling,
+                            _sendFailureHandlingFeature.SendFailureHandling,
+                            filter);
+
+                        if (exit != 0)
+                            return exit;
+                    }
                 }
+
+                return 0;
             }
             catch (Exception ex)
             {

src/SeqCli/Cli/Features/FileInputFeature.cs

@@ -13,33 +13,55 @@
 // limitations under the License.
 
 using System;
+using System.Collections.Generic;
 using System.IO;
+using SeqCli.Util;
 
 namespace SeqCli.Cli.Features
 {
     class FileInputFeature : CommandFeature
     {
         readonly string _description;
+        readonly bool _supportsWildcard;
 
-        public FileInputFeature(string description)
+        public FileInputFeature(string description, bool supportsWildcard = false)
         {
             _description = description;
+            _supportsWildcard = supportsWildcard;
         }
 
         public string InputFilename { get; private set; }
 
         public override void Enable(OptionSet options)
         {
+            var wildcardHelp = _supportsWildcard ? $", including the `{DirectoryExt.Wildcard}` wildcard" : "";
             options.Add("i=|input=",
-                $"{_description}; if not specified, `STDIN` will be used",
+                $"{_description}{wildcardHelp}; if not specified, `STDIN` will be used",
                 v => InputFilename = string.IsNullOrWhiteSpace(v) ? null : v.Trim());
         }
 
+        static TextReader OpenText(string filename)
+        {
+            return new StreamReader(
+                File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite));
+        }
+
         public TextReader OpenInput()
         {
-            return InputFilename != null ? 
-                new StreamReader(File.Open(InputFilename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) : 
-                Console.In;
+            return InputFilename != null ? OpenText(InputFilename) : Console.In;
+        }
+
+        public IEnumerable<TextReader> OpenInputs()
+        {
+            if (InputFilename == null || !DirectoryExt.IncludesWildcard(InputFilename))
+            {
+                yield return OpenInput();
+            }
+            else
+            {
+                foreach (var path in DirectoryExt.GetFiles(InputFilename))
+                    yield return OpenText(path);
+            }
         }
     }
 }

src/SeqCli/Util/DirectoryExt.cs

@@ -0,0 +1,47 @@
+// Copyright 2019 Datalust Pty Ltd and Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+
+namespace SeqCli.Util
+{
+    static class DirectoryExt
+    {
+        public const char Wildcard = '*';
+
+        public static bool IncludesWildcard(string filePath)
+        {
+            if (filePath == null) throw new ArgumentNullException(nameof(filePath));
+            return filePath.Contains(Wildcard);
+        }
+
+        public static IEnumerable<string> GetFiles(string filePathWithWildcard)
+        {
+            if (!IncludesWildcard(filePathWithWildcard))
+                throw new ArgumentException("The path does not contain a wildcard.");
+
+            var directory = Path.GetDirectoryName(filePathWithWildcard);
+            if (string.IsNullOrWhiteSpace(directory))
+                directory = ".";
+            else if (IncludesWildcard(directory))
+                throw new ArgumentException("The wildcard may not appear in the directory path.");
+
+            var searchPattern = Path.GetFileName(filePathWithWildcard);
+
+            return Directory.GetFiles(directory, searchPattern, SearchOption.TopDirectoryOnly);
+        }
+    }
+}

test/SeqCli.EndToEnd/Data/log-2.txt

@@ -0,0 +1 @@
+Loop 4

test/SeqCli.EndToEnd/IngestCommand/SimpleTextIngestionTestCase.cs

@@ -14,14 +14,13 @@ public async Task ExecuteAsync(
             ILogger logger,
             CliCommandRunner runner)
         {
-            var inputFile = Path.Combine("Data", "events.txt");
-            Assert.True(File.Exists(inputFile));
+            var inputFiles = Path.Combine("Data", "log-*.txt");
 
-            var exit = runner.Exec("ingest", $"-i {inputFile}");
+            var exit = runner.Exec("ingest", $"-i {inputFiles}");
             Assert.Equal(0, exit);
 
             var events = await connection.Events.ListAsync();
-            Assert.Equal(3, events.Count);
+            Assert.Equal(4, events.Count);
         }
     }
 }

test/SeqCli.EndToEnd/SeqCli.EndToEnd.csproj

@@ -13,6 +13,9 @@
     <PackageReference Include="Seq.Api" Version="5.1.0" />
   </ItemGroup>
   <ItemGroup>
+    <None Update="Data\log-2.txt">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
     <None Update="Data\levelled-events.txt">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
@@ -25,7 +28,7 @@
     <None Update="Data\events.ndjson">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
-    <None Update="Data\events.txt">
+    <None Update="Data\log-1.txt">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
   </ItemGroup>