[WIP] Added more extensive error handling

.env

@@ -0,0 +1,3 @@
+TAG=5.6.2
+ELASTIC_VERSION=5.6.2
+ELASTIC_PASSWORD=changeme

.gitignore

@@ -34,6 +34,10 @@ TestResult.xml
 [Rr]eleasePS/
 dlldata.c
 
+buffer*.json
+*.bookmark
+failures.txt
+
 *_i.c
 *_p.c
 *_i.h

.vscode/launch.json

@@ -0,0 +1,28 @@
+{
+   // Use IntelliSense to find out which attributes exist for C# debugging
+   // Use hover for the description of the existing attributes
+   // For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md
+   "version": "0.2.0",
+   "configurations": [
+        {
+            "name": ".NET Core Launch (console)",
+            "type": "coreclr",
+            "request": "launch",
+            "preLaunchTask": "build",
+            // If you have changed target frameworks, make sure to update the program path.
+            "program": "${workspaceRoot}/sample/Serilog.Sinks.Elasticsearch.Sample/bin/Debug/netcoreapp1.1/Serilog.Sinks.Elasticsearch.Sample.dll",
+            "args": [],
+            "cwd": "${workspaceRoot}/sample/Serilog.Sinks.Elasticsearch.Sample",
+            // For more information about the 'console' field, see https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md#console-terminal-window
+            "console": "internalConsole",
+            "stopAtEntry": false,
+            "internalConsoleOptions": "openOnSessionStart"
+        },
+        {
+            "name": ".NET Core Attach",
+            "type": "coreclr",
+            "request": "attach",
+            "processId": "${command:pickProcess}"
+        }
+    ]
+}

.vscode/tasks.json

@@ -0,0 +1,16 @@
+{
+    "version": "0.1.0",
+    "command": "dotnet",
+    "isShellCommand": true,
+    "args": [],
+    "tasks": [
+        {
+            "taskName": "build",
+            "args": [
+                "${workspaceRoot}/sample/Serilog.Sinks.Elasticsearch.Sample/Serilog.Sinks.Elasticsearch.Sample.csproj"
+            ],
+            "isBuildCommand": true,
+            "problemMatcher": "$msCompile"
+        }
+    ]
+}

CHANGES.md

@@ -1,5 +1,14 @@
 == Changelog
 
+5.5
+ * Errors from Elasticsearch can now be handled. Either by looking into the selflog, sending the failty events to another sink, handle the logevent yourself by using a callback or let the sink throw an exception.
+ * BOM fix for buffered option.
+ * The creation of the template might fail. You can now specify what kind of action should be taken if this happens.
+ * Added a sample application.
+ * Added a docker-compose file that allows you to start a local elasticsearch and kibana instance.
+ * ConnectionTimeout is now set to be 5 seconds instead of 1 minute.
+ * You can now set the queueSizeLimit, which limits the amount of events stored in the PeriodicBatching buffer. Does not impact the durable buffer.
+
 5.4
  * Added support for pipelines in Elasticsearch. Pipelines allows you to change the ingress data by running it through Processors (https://www.elastic.co/blog/new-way-to-ingest-part-1).
 

README.md

@@ -131,6 +131,36 @@ In your `appsettings.json` file, under the `Serilog` node, :
 
 See the XML `<appSettings>` example above for a discussion of available `Args` options.
 
+### Handling errors
+
+From version 5.5 you have the option to specify how to handle issues with Elasticsearch. Since the sink delivers in a batch, it might be possible that one or more events could actually nog be stored in the Elasticseach store.
+Can be a mapping issue for example. It is hard to find out what happened here. There is a new option called *EmitEventFailure* which is an enum (flagged) with the following options:
+
+ - WriteToSelfLog, the default option in which the errors are written to the SelfLog.
+ - WriteToFailureSink, the failed events are send to another sink. Make sure to configure this one by setting the FailureSink option.
+ - ThrowException, in which an exception is raised.
+ - RaiseCallback, the failure callback function will be called when the event cannot be submitted to Elasticsearch. Make sure to set the FailureCallback option to handle the event.
+
+An example:
+
+```csharp
+.WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri("http://localhost:9200")) 
+                {
+                    FailureCallback = e => Console.WriteLine("Unable to submit event " + e.MessageTemplate),
+                    EmitEventFailure = EmitEventFailureHandling.WriteToSelfLog |
+                                       EmitEventFailureHandling.WriteToFailureSink |
+                                       EmitEventFailureHandling.RaiseCallback,
+                    FailureSink = new FileSink("./failures.txt", new JsonFormatter(), null)
+                })
+```
+
+With the AutoRegisterTemplate option the sink will write a default template to Elasticsearch. When this template is not there, you might not want to index as it can influence the data quality.
+Since version 5.5 you can use the RegisterTemplateFailure option. Set it to one of the following options:
+
+ - IndexAnyway; the default option, the events will be send to the server
+ - IndexToDeadletterIndex; using the deadletterindex format, it will write the events to the deadletter queue. When you fix your template mapping, you can copy your data into the right index.
+ - FailSink; this will simply fail the sink by raising an exception.
+
 ### Breaking changes for version 4
 
 Starting from version 4, the sink has been upgraded to work with Serilog 2.0 and has .NET Core support.

docker-compose.readme.txt

@@ -0,0 +1,16 @@
+Elastic (http://localhost:9200)= elastic:changeme
+Elastic Head: http://localhost:9100/?auth_user=elastic&auth_password=changeme
+Kibana: http://localhost:5601
+
+To start the components:
+docker-compose up
+
+add the -d option to run in the background
+
+To scale the nodes:
+
+docker-compose up --scale elasticsearch=3
+
+To shutdown:
+
+docker-compose down

docker-compose.yml

@@ -0,0 +1,84 @@
+version: '2'
+
+services:
+
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:${TAG}
+    volumes:
+      - esdata2:/usr/share/elasticsearch/data
+    ports:
+      - "9200:9200"
+      - "9300"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    mem_limit: 1g
+    environment:
+      - cluster.name=docker-cluster
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - discovery.type=zen
+      - discovery.zen.ping.unicast.hosts=elasticsearch
+      - http.cors.enabled=true
+      - http.cors.allow-origin=*
+      - http.cors.allow-headers=Content-Type, Access-Control-Allow-Headers, Authorization, X-Requested-With
+      - http.cors.allow-credentials=true
+    networks:
+      - elk
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:${TAG}
+    environment:
+      - "ELASTICSEARCH_URL=http://elasticsearch:9200"
+      - server.name=kibana
+      - server.host="0"
+      - xpack.security.enabled=true
+      - xpack.monitoring.enabled=true
+      - xpack.ml.enabled=false
+      - xpack.graph.enabled=false
+      - xpack.reporting.enabled=false
+      - xpack.grokdebugger.enabled=false
+    ports:
+      - "5601:5601"
+    networks:
+      - esnet
+    depends_on:
+      - elasticsearch
+
+  head:
+    image: mobz/elasticsearch-head:5
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    mem_limit: 1g
+    ports:
+      - "9100:9100"
+    networks:
+      - elk
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:${TAG}
+    environment:
+      - "ELASTICSEARCH_URL=http://elasticsearch:9200"
+    ports:
+      - "5601:5601"
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch
+
+volumes:
+  esdata1:
+    driver: local
+  esdata2:
+    driver: local
+  #esconfig1:
+  #  driver: local
+  #esconfig2:
+  #  driver: local
+
+networks:
+  elk:
+    driver: bridge

sample/Serilog.Sinks.Elasticsearch.Sample/Program.cs

@@ -0,0 +1,53 @@
+using System;
+using Serilog;
+using Serilog.Debugging;
+using Serilog.Formatting.Json;
+using Serilog.Sinks.File;
+using Serilog.Sinks.SystemConsole.Themes;
+
+namespace Serilog.Sinks.Elasticsearch.Sample
+{
+    class Program
+    {
+        static void Main(string[] args)
+        {
+            Log.Logger = new LoggerConfiguration()
+                .MinimumLevel.Debug()
+                .WriteTo.Console(theme: SystemConsoleTheme.Literate)
+                .WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri("http://elastic:changeme@localhost:9200")) // for the docker-compose implementation
+                {
+                    AutoRegisterTemplate = true,
+                    //BufferBaseFilename = "./buffer",
+                    RegisterTemplateFailure = RegisterTemplateRecovery.IndexAnyway,
+                    FailureCallback = e => Console.WriteLine("Unable to submit event " + e.MessageTemplate),
+                    EmitEventFailure = EmitEventFailureHandling.WriteToSelfLog |
+                                       EmitEventFailureHandling.WriteToFailureSink |
+                                       EmitEventFailureHandling.RaiseCallback,
+                    FailureSink = new FileSink("./failures.txt", new JsonFormatter(), null)
+                })
+                .CreateLogger();
+
+            // Enable the selflog output
+            SelfLog.Enable(Console.Error);
+
+            Log.Information("Hello, world!");
+
+            int a = 10, b = 0;
+            try
+            {
+                Log.Debug("Dividing {A} by {B}", a, b);
+                Console.WriteLine(a / b);
+            }
+            catch (Exception ex)
+            {
+                Log.Error(ex, "Something went wrong");
+            }
+
+            // Introduce a failure by storing a field as a different type
+            Log.Debug("Reusing {A} by {B}", "string", true);
+
+            Log.CloseAndFlush();
+            Console.Read();
+        }
+    }
+}

sample/Serilog.Sinks.Elasticsearch.Sample/Serilog.Sinks.Elasticsearch.Sample.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp1.1</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Serilog" Version="2.5.0" />
+    <PackageReference Include="Serilog.Sinks.Console" Version="3.1.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\Serilog.Sinks.Elasticsearch\Serilog.Sinks.Elasticsearch.csproj" />
+  </ItemGroup>
+
+</Project>

serilog-sinks-elasticsearch.sln

@@ -1,6 +1,6 @@
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio 15
-VisualStudioVersion = 15.0.26730.16
+VisualStudioVersion = 15.0.26730.3
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Files", "Files", "{148431F6-5BA9-4987-80CF-DF9F23F54947}"
 	ProjectSection(SolutionItems) = preProject
@@ -19,6 +19,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serilog.Sinks.Elasticsearch
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serilog.Sinks.Elasticsearch.Tests", "test\Serilog.Sinks.Elasticsearch.Tests\Serilog.Sinks.Elasticsearch.Tests.csproj", "{CA358673-C4B6-49D0-8EC5-D6CB50A11CC0}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serilog.Sinks.Elasticsearch.Sample", "sample\Serilog.Sinks.Elasticsearch.Sample\Serilog.Sinks.Elasticsearch.Sample.csproj", "{253B37AB-D82E-4A5F-BA16-F1BE398818C8}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -33,8 +35,15 @@ Global
 		{CA358673-C4B6-49D0-8EC5-D6CB50A11CC0}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{CA358673-C4B6-49D0-8EC5-D6CB50A11CC0}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{CA358673-C4B6-49D0-8EC5-D6CB50A11CC0}.Release|Any CPU.Build.0 = Release|Any CPU
+		{253B37AB-D82E-4A5F-BA16-F1BE398818C8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{253B37AB-D82E-4A5F-BA16-F1BE398818C8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{253B37AB-D82E-4A5F-BA16-F1BE398818C8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{253B37AB-D82E-4A5F-BA16-F1BE398818C8}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {2A076FF0-B90E-451A-B858-3144CE509516}
+	EndGlobalSection
 EndGlobal

serilog-sinks-elasticsearch.sln.iml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="RIDER_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$" isTestSource="false" />
+    </content>
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj

@@ -29,8 +29,8 @@
   <ItemGroup>
     <PackageReference Include="Elasticsearch.Net" Version="5.5.0" />
     <PackageReference Include="Serilog" Version="2.5.0" />
-    <PackageReference Include="Serilog.Sinks.File" Version="3.2.0" />
-    <PackageReference Include="Serilog.Sinks.PeriodicBatching" Version="2.1.0" />
+    <PackageReference Include="Serilog.Sinks.File" Version="4.0.0" />
+    <PackageReference Include="Serilog.Sinks.PeriodicBatching" Version="2.1.1" />
     <PackageReference Include="Serilog.Sinks.RollingFile" Version="3.3.0" />
   </ItemGroup>