Merge branch 'dev'

.env

@@ -1,3 +1,3 @@
-TAG=5.6.2
-ELASTIC_VERSION=5.6.2
+TAG=6.1.3
+ELASTIC_VERSION=6.1.3
 ELASTIC_PASSWORD=changeme

CHANGES.md

@@ -1,5 +1,9 @@
 == Changelog
 
+6.1
+ * Updated to elasticsearch 6 libraries (#153)
+ * Fix field index option for 6.1+ template to use boolean value. (#148)
+
 5.7
  * Supporting ES 6 template defitions while still supporting old versions. (https://github.com/serilog/serilog-sinks-elasticsearch/pull/142) See for details https://www.elastic.co/blog/strings-are-dead-long-live-strings
  * Pipeline decider added.

GitVersion.yml

@@ -1,4 +1,4 @@
-next-version: 5.6.0
+next-version: 6.0.0
 branches: {}
 ignore:
   sha: []

README.md

@@ -14,6 +14,7 @@ The Serilog Elasticsearch sink project is a sink (basically a writer) for the Se
 - Durable mode; store the logevents first on disk before delivering them to ES making sure you never miss events if you have trouble connecting to your ES cluster.
 - Automatically create the right mappings for the best usage of the log events in ES or automatically upload your own custom mapping.
 - Starting from version 3, compatible with Elasticsearch 2.
+- Version 6.x supports the new Elasticsearch.net version 6.x library.
 
 ## Quick start
 
@@ -161,6 +162,11 @@ Since version 5.5 you can use the RegisterTemplateFailure option. Set it to one
  - IndexToDeadletterIndex; using the deadletterindex format, it will write the events to the deadletter queue. When you fix your template mapping, you can copy your data into the right index.
  - FailSink; this will simply fail the sink by raising an exception.
 
+
+### Breaking changes for version 6
+
+Starting from version 6, the sink has been upgraded to work with Elasticsearch 6.0 and has support for the new templates used by ES 6.
+
 ### Breaking changes for version 4
 
 Starting from version 4, the sink has been upgraded to work with Serilog 2.0 and has .NET Core support.

sample/Serilog.Sinks.Elasticsearch.Sample/Serilog.Sinks.Elasticsearch.Sample.csproj

@@ -6,7 +6,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Serilog" Version="2.5.0" />
+    <PackageReference Include="Serilog" Version="2.6.0" />
     <PackageReference Include="Serilog.Sinks.Console" Version="3.1.1" />
   </ItemGroup>
 

src/Serilog.Sinks.Elasticsearch/Properties/AssemblyInfo.cs

@@ -3,13 +3,13 @@
 
 [assembly: AssemblyTitle("Serilog.Sinks.Elasticsearch")]
 [assembly: AssemblyDescription("Serilog sink for Elasticsearch")]
-[assembly: AssemblyCopyright("Copyright © Serilog Contributors 2017")]
+[assembly: AssemblyCopyright("Copyright © Serilog Contributors 2018")]
 
 [assembly: InternalsVisibleTo("Serilog.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100fb8d13fd344a1c" +
                                                        "6fe0fe83ef33c1080bf30690765bc6eb0df26ebfdf8f21670c64265b30db09f73a0dea5b3db4c9" +
                                                        "d18dbf6d5a25af5ce9016f281014d79dc3b4201ac646c451830fc7e61a2dfd633d34c39f87b818" +
                                                        "94191652df5ac63cc40c77f3542f702bda692e6e8a9158353df189007a49da0f3cfd55eb250066" +
                                                        "b19485ec")]
-[assembly: AssemblyVersion("5.3.0.0")]
-[assembly: AssemblyInformationalVersion("5.3.0-unstable.32+Branch.dev.Sha.7e77498e8afaaec96b508de3e8ec6bd3891b3556")]
-[assembly: AssemblyFileVersion("5.3.0.0")]
+[assembly: AssemblyVersion("6.0.0.0")]
+[assembly: AssemblyInformationalVersion("6.0.0-unstable.32+Branch.dev.Sha.7e77498e8afaaec96b508de3e8ec6bd3891b3556")]
+[assembly: AssemblyFileVersion("6.0.0.0")]

src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.Symbols.nuspec

@@ -2,7 +2,7 @@
 <package xmlns="http://schemas.microsoft.com/packaging/2010/07/nuspec.xsd">
   <metadata>
     <id>Serilog.Sinks.Elasticsearch</id>
-    <version>2.0.60.2</version>
+    <version>$version$</version>
     <authors>Michiel van Oudheusden, Martijn Laarman, Mogens Heller Grabe</authors>
     <description>The perfect way for .NET apps to write structured log events to Elasticsearch.</description>
     <language>en-US</language>
@@ -11,11 +11,11 @@
     <iconUrl>http://serilog.net/images/serilog-sink-nuget.png</iconUrl>
     <tags>serilog logging elasticsearch</tags>
     <dependencies>
-      <dependency id="Serilog" version="2.4.0" />
-      <dependency id="Elasticsearch.Net" version="5.4.0" />
-      <dependency id="Serilog.Sinks.File" version="3.2.0" />
-      <dependency id="Serilog.Sinks.PeriodicBatching" version="2.1.0" />
-      <dependency id="Serilog.Sinks.RollingFile" version="3.3.0" />
+       <dependency id="Serilog" version="2.6.0" />
+       <dependency id="Elasticsearch.Net" version="6.0.0" />
+       <dependency id="Serilog.Sinks.File" version="4.0.0" />
+       <dependency id="Serilog.Sinks.PeriodicBatching" version="2.1.1" />
+       <dependency id="Serilog.Sinks.RollingFile" version="3.3.0" />
     </dependencies>
   </metadata>
   <files>

src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.nuspec

@@ -11,10 +11,10 @@
     <iconUrl>http://serilog.net/images/serilog-sink-nuget.png</iconUrl>
     <tags>serilog logging elasticsearch</tags>
     <dependencies>
-      <dependency id="Serilog" version="2.4.0" />
-      <dependency id="Elasticsearch.Net" version="5.4.0" />
-      <dependency id="Serilog.Sinks.File" version="3.2.0" />
-      <dependency id="Serilog.Sinks.PeriodicBatching" version="2.1.0" />
+      <dependency id="Serilog" version="2.6.0" />
+      <dependency id="Elasticsearch.Net" version="6.0.0" />
+      <dependency id="Serilog.Sinks.File" version="4.0.0" />
+      <dependency id="Serilog.Sinks.PeriodicBatching" version="2.1.1" />
       <dependency id="Serilog.Sinks.RollingFile" version="3.3.0" />
     </dependencies>
   </metadata>

src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj

@@ -1,7 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <VersionPrefix>5.1.0</VersionPrefix>
+    <VersionPrefix>6.0.0</VersionPrefix>
+    <VersionSuffix>alpha</VersionSuffix>
     <Authors>Michiel van Oudheusden, Martijn Laarman, Mogens Heller Grabe, Serilog Contributors</Authors>
     <TargetFrameworks>net45;netstandard1.3</TargetFrameworks>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
@@ -23,12 +24,11 @@
     <GenerateAssemblyFileVersionAttribute>false</GenerateAssemblyFileVersionAttribute>
     <GenerateAssemblyInformationalVersionAttribute>false</GenerateAssemblyInformationalVersionAttribute>
     <GenerateAssemblyVersionAttribute>false</GenerateAssemblyVersionAttribute>
-    <Version>5.2.0</Version>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Elasticsearch.Net" Version="5.5.0" />
-    <PackageReference Include="Serilog" Version="2.5.0" />
+    <PackageReference Include="Elasticsearch.Net" Version="6.0.0" />
+    <PackageReference Include="Serilog" Version="2.6.0" />
     <PackageReference Include="Serilog.Sinks.File" Version="4.0.0" />
     <PackageReference Include="Serilog.Sinks.PeriodicBatching" Version="2.1.1" />
     <PackageReference Include="Serilog.Sinks.RollingFile" Version="3.3.0" />

src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchLogShipper.cs

@@ -201,7 +201,7 @@ void OnTick()
                         if (count > 0)
                         {
 
-                            var response = _state.Client.Bulk<DynamicResponse>(payload);
+                            var response = _state.Client.Bulk<DynamicResponse>(PostData.MultiJson(payload));
 
                             if (response.Success)
                             {
@@ -261,7 +261,7 @@ void OnTick()
             }
         }
 
-        static void HandleBulkResponse(ElasticsearchResponse<DynamicResponse> response, List<string> payload)
+        static void HandleBulkResponse(DynamicResponse response, List<string> payload)
         {
             int i = 0;
             var items = response.Body["items"];

src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs

@@ -53,7 +53,16 @@ public ElasticsearchSink(ElasticsearchSinkOptions options)
         /// </remarks>
         protected override void EmitBatch(IEnumerable<LogEvent> events)
         {
-            var result = this.EmitBatchChecked<DynamicResponse>(events);
+            DynamicResponse result;
+            try
+            {
+                result = this.EmitBatchChecked<DynamicResponse>(events);
+            }
+            catch (Exception ex)
+            {
+                HandleException(ex, events);
+                return;
+            }
 
             // Handle the results from ES, check if there are any errors.
             if (result.Success && result.Body?["errors"] == true)
@@ -123,7 +132,7 @@ protected override void EmitBatch(IEnumerable<LogEvent> events)
         /// </summary>
         /// <param name="events">The events to emit.</param>
         /// <returns>Response from Elasticsearch</returns>
-        protected virtual ElasticsearchResponse<T> EmitBatchChecked<T>(IEnumerable<LogEvent> events) where T : class
+        protected virtual T EmitBatchChecked<T>(IEnumerable<LogEvent> events) where T : class, IElasticsearchResponse, new()
         {
             // ReSharper disable PossibleMultipleEnumeration
             if (events == null || !events.Any())
@@ -134,37 +143,28 @@ protected virtual ElasticsearchResponse<T> EmitBatchChecked<T>(IEnumerable<LogEv
                 return null;
             }
 
-            try
+            var payload = new List<string>();
+            foreach (var e in events)
             {
-                var payload = new List<string>();
-                foreach (var e in events)
-                {
-                    var indexName = _state.GetIndexForEvent(e, e.Timestamp.ToUniversalTime());
-                    var action = default(object);
+                var indexName = _state.GetIndexForEvent(e, e.Timestamp.ToUniversalTime());
+                var action = default(object);
 
-                    var pipelineName = _state.Options.PipelineNameDecider?.Invoke(e) ?? _state.Options.PipelineName;
-                    if (string.IsNullOrWhiteSpace(pipelineName))
-                    {
-                        action = new { index = new { _index = indexName, _type = _state.Options.TypeName } };
-                    }
-                    else
-                    {
-                        action = new { index = new { _index = indexName, _type = _state.Options.TypeName, pipeline = pipelineName } };
-                    }
-                    var actionJson = _state.Serialize(action);
-                    payload.Add(actionJson);
-                    var sw = new StringWriter();
-                    _state.Formatter.Format(e, sw);
-                    payload.Add(sw.ToString());
+                var pipelineName = _state.Options.PipelineNameDecider?.Invoke(e) ?? _state.Options.PipelineName;
+                if (string.IsNullOrWhiteSpace(pipelineName))
+                {
+                    action = new { index = new { _index = indexName, _type = _state.Options.TypeName } };
                 }
-                return _state.Client.Bulk<T>(payload);
-            }
-            catch (Exception ex)
-            {
-                HandleException(ex, events);
-
-                return new ElasticsearchResponse<T>(ex);
+                else
+                {
+                    action = new { index = new { _index = indexName, _type = _state.Options.TypeName, pipeline = pipelineName } };
+                }
+                var actionJson = _state.Serialize(action);
+                payload.Add(actionJson);
+                var sw = new StringWriter();
+                _state.Formatter.Format(e, sw);
+                payload.Add(sw.ToString());
             }
+            return _state.Client.Bulk<T>(PostData.MultiJson(payload));
         }
 
         /// <summary>

src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs

@@ -114,7 +114,7 @@ private static object GetTemplateESv6(
                         },
                         properties = new Dictionary<string, object>
                         {
-                            {"message", new {type = "text", index = "analyzed"}},
+                            {"message", new {type = "text", index = "true"}},
                             {
                                 "exceptions", new
                                 {
@@ -124,8 +124,8 @@ private static object GetTemplateESv6(
                                         {"Depth", new {type = "integer"}},
                                         {"RemoteStackIndex", new {type = "integer"}},
                                         {"HResult", new {type = "integer"}},
-                                        {"StackTraceString", new {type = "text", index = "analyzed"}},
-                                        {"RemoteStackTraceString", new {type = "text", index = "analyzed"}},
+                                        {"StackTraceString", new {type = "text", index = "true"}},
+                                        {"RemoteStackTraceString", new {type = "text", index = "true"}},
                                         {
                                             "ExceptionMessage", new
                                             {

src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs

@@ -67,11 +67,7 @@ private ElasticsearchSinkState(ElasticsearchSinkOptions options)
 
             _options = options;
 
-            Func<ConnectionConfiguration, IElasticsearchSerializer> serializerFactory = null;
-            if (options.Serializer != null)
-                serializerFactory = s => options.Serializer;
-
-            var configuration = new ConnectionConfiguration(options.ConnectionPool, options.Connection, serializerFactory)
+            var configuration = new ConnectionConfiguration(options.ConnectionPool, options.Connection, options.Serializer)
                 .RequestTimeout(options.ConnectionTimeout);
 
             if (options.ModifyConnectionSettings != null)
@@ -134,11 +130,12 @@ public void RegisterTemplateIfNeeded()
                     }
                 }
 
-                var result = _client.IndicesPutTemplateForAll<DynamicResponse>(_templateName, GetTemplateData());
+                var result = _client.IndicesPutTemplateForAll<DynamicResponse>(_templateName, GetTempatePostData());
 
                 if (!result.Success)
                 {
-                    SelfLog.WriteLine("Unable to create the template. {0}", result.ServerError);
+                    ((IElasticsearchResponse)result).TryGetServerErrorReason(out var serverError);
+                    SelfLog.WriteLine("Unable to create the template. {0}", serverError);
 
                     if (_options.RegisterTemplateFailure == RegisterTemplateRecovery.FailSink)
                         throw new Exception($"Unable to create the template named {_templateName}.", result.OriginalException);
@@ -160,6 +157,22 @@ public void RegisterTemplateIfNeeded()
             }
         }
 
+        private PostData GetTempatePostData()
+        {
+            //PostData no longer exposes an implict cast from object.  Previously it supported that and would inspect the object Type to
+            //determine if it it was a litteral string to write directly or if it was an object that it needed to serialse.  Now the onus is 
+            //on us to tell it what type we are passing otherwise if the user specified the template as a json string it would be serialised again.
+            var template = GetTemplateData();
+            if (template is string)
+            {
+                return PostData.String((string)template);
+            }
+            else
+            {
+                return PostData.Serializable(template);
+            }
+        }
+
         private object GetTemplateData()
         {
             if (_options.GetTemplateContent != null)

test/Serilog.Sinks.Elasticsearch.Tests/Discrepancies/ElasticsearchDefaultSerializerTests.cs

@@ -5,7 +5,7 @@ namespace Serilog.Sinks.Elasticsearch.Tests.Discrepancies
 {
     public class ElasticsearchDefaultSerializerTests : ElasticsearchSinkUniformityTestsBase
     {
-        public ElasticsearchDefaultSerializerTests() : base(new ElasticsearchDefaultSerializer()) { }
+        public ElasticsearchDefaultSerializerTests() : base(new LowLevelRequestResponseSerializer()) { }
 
         [Fact]
         public void Should_SerializeToExpandedExceptionObjectWhenExceptionIsSet()

test/Serilog.Sinks.Elasticsearch.Tests/Discrepancies/JsonNetSerializerTests.cs

@@ -1,12 +1,13 @@
 using Elasticsearch.Net;
 using Nest;
+using Nest.JsonNetSerializer;
 using Xunit;
 
 namespace Serilog.Sinks.Elasticsearch.Tests.Discrepancies
 {
     public class JsonNetSerializerTests : ElasticsearchSinkUniformityTestsBase
     {
-        public JsonNetSerializerTests() : base(new JsonNetSerializer(new ConnectionSettings())) { }
+        public JsonNetSerializerTests() : base(JsonNetSerializer.Default(LowLevelRequestResponseSerializer.Instance, new ConnectionSettings())) { }
 
         [Fact]
         public void Should_SerializeToExpandedExceptionObjectWhenExceptionIsSet()