From adebe6a0bf7b9f69ab41ab6d09cb1acfabcdfac1 Mon Sep 17 00:00:00 2001 From: "Kraemer, Benjamin" Date: Mon, 23 Nov 2020 13:49:50 +0100 Subject: [PATCH 1/3] Merges logic for pr/14 and pr/22. --- .../LogLabelProvider.cs | 7 +- .../Labels/DefaultLogLabelProvider.cs | 25 ++++- .../Labels/ILogLabelProvider.cs | 5 +- src/Serilog.Sinks.Loki/LokiBatchFormatter.cs | 104 ++++++++++++++---- .../LokiFormatterStrategy.cs | 19 ++++ .../LokiSinkConfiguration.cs | 15 +++ src/Serilog.Sinks.Loki/LokiSinkExtensions.cs | 32 ++++-- .../Infrastructure/TestLabelProvider.cs | 2 + 8 files changed, 175 insertions(+), 34 deletions(-) create mode 100644 src/Serilog.Sinks.Loki/LokiFormatterStrategy.cs create mode 100644 src/Serilog.Sinks.Loki/LokiSinkConfiguration.cs diff --git a/src/Serilog.Sinks.Loki.Example/LogLabelProvider.cs b/src/Serilog.Sinks.Loki.Example/LogLabelProvider.cs index 9d95284..c4b4d4b 100644 --- a/src/Serilog.Sinks.Loki.Example/LogLabelProvider.cs +++ b/src/Serilog.Sinks.Loki.Example/LogLabelProvider.cs @@ -16,7 +16,12 @@ public IList GetLabels() public IList PropertiesAsLabels { get; set; } = new List { - "MyPropertyName" + "MyLabelPropertyName" }; + public IList PropertiesToAppend { get; set; } = new List + { + "MyAppendPropertyName" + }; + public LokiFormatterStrategy FormatterStrategy { get; set; } = LokiFormatterStrategy.SpecificPropertiesAsLabelsOrAppended; } } \ No newline at end of file diff --git a/src/Serilog.Sinks.Loki/Labels/DefaultLogLabelProvider.cs b/src/Serilog.Sinks.Loki/Labels/DefaultLogLabelProvider.cs index 8b7e1cd..9a97df4 100644 --- a/src/Serilog.Sinks.Loki/Labels/DefaultLogLabelProvider.cs +++ b/src/Serilog.Sinks.Loki/Labels/DefaultLogLabelProvider.cs @@ -1,14 +1,33 @@ using System.Collections.Generic; +using System.Linq; namespace Serilog.Sinks.Loki.Labels { class DefaultLogLabelProvider : ILogLabelProvider { - public IList GetLabels() + public DefaultLogLabelProvider() : this(null) + { + } + + public DefaultLogLabelProvider(IEnumerable labels, + IEnumerable propertiesAsLabels = null, + IEnumerable propertiesToAppend = null, + LokiFormatterStrategy formatterStrategy = LokiFormatterStrategy.SpecificPropertiesAsLabelsAndRestAppended) { - return new List(); + this.Labels = labels?.ToList() ?? new List(); + this.PropertiesAsLabels = propertiesAsLabels?.ToList() ?? new List(); + this.PropertiesToAppend = propertiesToAppend?.ToList() ?? new List(); + this.FormatterStrategy = formatterStrategy; } - public IList PropertiesAsLabels { get; set; } = new List(); + public IList GetLabels() + { + return this.Labels; + } + + private IList Labels { get; } + public IList PropertiesAsLabels { get; } + public IList PropertiesToAppend { get; } + public LokiFormatterStrategy FormatterStrategy { get; } } } \ No newline at end of file diff --git a/src/Serilog.Sinks.Loki/Labels/ILogLabelProvider.cs b/src/Serilog.Sinks.Loki/Labels/ILogLabelProvider.cs index 62cbbd6..70b9436 100644 --- a/src/Serilog.Sinks.Loki/Labels/ILogLabelProvider.cs +++ b/src/Serilog.Sinks.Loki/Labels/ILogLabelProvider.cs @@ -5,6 +5,9 @@ namespace Serilog.Sinks.Loki.Labels public interface ILogLabelProvider { IList GetLabels(); - IList PropertiesAsLabels { get; set; } + + IList PropertiesAsLabels { get; } + IList PropertiesToAppend { get; } + LokiFormatterStrategy FormatterStrategy { get; } } } \ No newline at end of file diff --git a/src/Serilog.Sinks.Loki/LokiBatchFormatter.cs b/src/Serilog.Sinks.Loki/LokiBatchFormatter.cs index 3f8269a..c6922a8 100644 --- a/src/Serilog.Sinks.Loki/LokiBatchFormatter.cs +++ b/src/Serilog.Sinks.Loki/LokiBatchFormatter.cs @@ -4,6 +4,7 @@ using System.Linq; using Serilog.Events; using Serilog.Formatting; +using Serilog.Parsing; using Serilog.Sinks.Http; using Serilog.Sinks.Loki.Labels; @@ -11,22 +12,46 @@ namespace Serilog.Sinks.Loki { using System.Text; - internal class LokiBatchFormatter : IBatchFormatter + internal class LokiBatchFormatter : IBatchFormatter { - private readonly IList _globalLabels; - private readonly IList _propertiesAsLabels; + public ILogLabelProvider LogLabelProvider { get; } public LokiBatchFormatter() { - _globalLabels = new List(); + this.LogLabelProvider = new DefaultLogLabelProvider(); } public LokiBatchFormatter(ILogLabelProvider logLabelProvider) { - _globalLabels = logLabelProvider.GetLabels(); - _propertiesAsLabels = logLabelProvider.PropertiesAsLabels; + this.LogLabelProvider = logLabelProvider; } + [Obsolete("Assign to LokiBatchFormatter.GlobalLabels instead.")] + public LokiBatchFormatter(IList globalLabels) + { + this.LogLabelProvider = new DefaultLogLabelProvider(globalLabels); + } + + // This avoids additional quoting as described in https://github.com/serilog/serilog/issues/936 + private static void RenderMessage(TextWriter tw, LogEvent logEvent) + { + bool IsString(LogEventPropertyValue pv) + { + return pv is ScalarValue sv && sv.Value is string; + } + + foreach(var t in logEvent.MessageTemplate.Tokens) + { + if (t is PropertyToken pt && + logEvent.Properties.TryGetValue(pt.PropertyName, out var propVal) && + IsString(propVal)) + tw.Write(((ScalarValue)propVal).Value); + else + t.Render(logEvent.Properties, tw); + } + tw.Write('\n'); + } + public void Format(IEnumerable logEvents, ITextFormatter formatter, TextWriter output) { if (logEvents == null) @@ -45,13 +70,15 @@ public void Format(IEnumerable logEvents, ITextFormatter formatter, Te content.Streams.Add(stream); stream.Labels.Add(new LokiLabel("level", GetLevel(logEvent.Level))); - foreach (LokiLabel globalLabel in _globalLabels) + foreach (LokiLabel globalLabel in this.LogLabelProvider.GetLabels()) stream.Labels.Add(new LokiLabel(globalLabel.Key, globalLabel.Value)); var time = logEvent.Timestamp.ToString("o"); - var sb = new StringBuilder(); - sb.AppendLine(logEvent.RenderMessage()); + using (var tw = new StringWriter(sb)) + { + RenderMessage(tw, logEvent); + } if (logEvent.Exception != null) // AggregateException adds a Environment.Newline to the end of ToString(), so we trim it off sb.AppendLine(logEvent.Exception.ToString().TrimEnd()); @@ -63,21 +90,25 @@ public void Format(IEnumerable logEvents, ITextFormatter formatter, Te // To avoid this, remove all quotes from the value. // We also remove any \r\n newlines and replace with \n new lines to prevent "bad request" responses // We also remove backslashes and replace with forward slashes, Loki doesn't like those either - var propertyValue = property.Value.ToString().Replace("\"", "").Replace("\r\n", "\n").Replace("\\", "/"); - if (_propertiesAsLabels.Contains(property.Key, StringComparer.OrdinalIgnoreCase)) - { - stream.Labels.Add(new LokiLabel(property.Key, propertyValue)); - } - else + var propertyValue = property.Value.ToString().Replace("\r\n", "\n"); + + switch (DetermineHandleActionForProperty(property.Key)) { - sb.Append($" {property.Key}={propertyValue}"); + case HandleAction.Discard: + continue; + case HandleAction.SendAsLabel: + propertyValue = propertyValue.Replace("\"", "").Replace("\\", "/"); + stream.Labels.Add(new LokiLabel(property.Key, propertyValue)); + break; + case HandleAction.AppendToMessage: + sb.Append($" {property.Key}={propertyValue}"); + break; } } // Loki doesn't like \r\n for new line, and we can't guarantee the message doesn't have any // in it, so we replace \r\n with \n on the final message - // We also flip backslashes to forward slashes, Loki doesn't like those either. - stream.Entries.Add(new LokiEntry(time, sb.ToString().Replace("\\", "/").Replace("\r\n", "\n"))); + stream.Entries.Add(new LokiEntry(time, sb.ToString().Replace("\r\n", "\n"))); } if (content.Streams.Count > 0) @@ -99,5 +130,40 @@ private static string GetLevel(LogEventLevel level) default: return level.ToString().ToLower(); } } + + private HandleAction DetermineHandleActionForProperty(string propertyName) + { + var provider = this.LogLabelProvider; + switch (provider.FormatterStrategy) + { + case LokiFormatterStrategy.AllPropertiesAsLabels: + return HandleAction.SendAsLabel; + + case LokiFormatterStrategy.SpecificPropertiesAsLabelsAndRestDiscarded: + return provider.PropertiesAsLabels.Contains(propertyName) + ? HandleAction.SendAsLabel + : HandleAction.Discard; + + case LokiFormatterStrategy.SpecificPropertiesAsLabelsAndRestAppended: + return provider.PropertiesAsLabels.Contains(propertyName) + ? HandleAction.SendAsLabel + : HandleAction.AppendToMessage; + + //case LokiFormatterStrategy.SpecificPropertiesAsLabelsOrAppended: + default: + return provider.PropertiesAsLabels.Contains(propertyName) + ? HandleAction.SendAsLabel + : provider.PropertiesToAppend.Contains(propertyName) + ? HandleAction.AppendToMessage + : HandleAction.Discard; + } + } + + private enum HandleAction + { + Discard, + SendAsLabel, + AppendToMessage + } } -} \ No newline at end of file +} diff --git a/src/Serilog.Sinks.Loki/LokiFormatterStrategy.cs b/src/Serilog.Sinks.Loki/LokiFormatterStrategy.cs new file mode 100644 index 0000000..c41c465 --- /dev/null +++ b/src/Serilog.Sinks.Loki/LokiFormatterStrategy.cs @@ -0,0 +1,19 @@ +namespace Serilog.Sinks.Loki { + public enum LokiFormatterStrategy { + /// All Serilog Event properties will be sent as labels + AllPropertiesAsLabels, + + /// Specific Serilog Event properties will be sent as labels. + /// The rest of properties will be discarder. + SpecificPropertiesAsLabelsAndRestDiscarded, + + /// Specific Serilog Event properties will be sent as labels. + /// The rest of properties will be appended to the log message. + SpecificPropertiesAsLabelsAndRestAppended, + + /// Specific Serilog Event properties will be sent as labels. + /// Other specific properties will be appended to the log message. + /// The rest of properties will be discarded + SpecificPropertiesAsLabelsOrAppended + } +} diff --git a/src/Serilog.Sinks.Loki/LokiSinkConfiguration.cs b/src/Serilog.Sinks.Loki/LokiSinkConfiguration.cs new file mode 100644 index 0000000..03f1b50 --- /dev/null +++ b/src/Serilog.Sinks.Loki/LokiSinkConfiguration.cs @@ -0,0 +1,15 @@ +using System.Collections.Generic; +using Serilog.Sinks.Http; +using Serilog.Sinks.Loki.Labels; + +namespace Serilog.Sinks.Loki +{ + public class LokiSinkConfiguration + { + public string LokiUrl { get; set; } + public string LokiUsername { get; set; } + public string LokiPassword { get; set; } + public ILogLabelProvider LogLabelProvider { get; set; } + public IHttpClient HttpClient { get; set; } + } +} diff --git a/src/Serilog.Sinks.Loki/LokiSinkExtensions.cs b/src/Serilog.Sinks.Loki/LokiSinkExtensions.cs index 08f3b60..6615547 100644 --- a/src/Serilog.Sinks.Loki/LokiSinkExtensions.cs +++ b/src/Serilog.Sinks.Loki/LokiSinkExtensions.cs @@ -1,3 +1,4 @@ +using System; using Serilog.Configuration; using Serilog.Sinks.Http; using Serilog.Sinks.Loki.Labels; @@ -6,13 +7,30 @@ namespace Serilog.Sinks.Loki { public static class LokiSinkExtensions { + public static LoggerConfiguration LokiHttp(this LoggerSinkConfiguration sinkConfiguration, string serverUrl) + => sinkConfiguration.LokiHttp(new NoAuthCredentials(serverUrl)); + + public static LoggerConfiguration LokiHttp(this LoggerSinkConfiguration sinkConfiguration, string serverUrl, string username, string password) + => sinkConfiguration.LokiHttp(new BasicAuthCredentials(serverUrl, username, password)); + public static LoggerConfiguration LokiHttp(this LoggerSinkConfiguration sinkConfiguration, LokiCredentials credentials, ILogLabelProvider labelProvider = null, LokiHttpClient httpClient = null) - => LokiHttpImpl(sinkConfiguration, credentials, labelProvider, httpClient); - + => LokiHttpImpl(sinkConfiguration, credentials, labelProvider, httpClient); + + public static LoggerConfiguration LokiHttp(this LoggerSinkConfiguration sinkConfiguration, Func configFactory) + => LokiHttpImpl(sinkConfiguration, configFactory()); + + private static LoggerConfiguration LokiHttpImpl(this LoggerSinkConfiguration serilogConfig, LokiSinkConfiguration lokiConfig) + { + var credentials = string.IsNullOrWhiteSpace(lokiConfig.LokiUsername) + ? (LokiCredentials)new NoAuthCredentials(lokiConfig.LokiUrl) + : new BasicAuthCredentials(lokiConfig.LokiUrl, lokiConfig.LokiUsername, lokiConfig.LokiPassword); + + return LokiHttpImpl(serilogConfig, credentials, lokiConfig.LogLabelProvider, lokiConfig.HttpClient); + } + private static LoggerConfiguration LokiHttpImpl(this LoggerSinkConfiguration sinkConfiguration, LokiCredentials credentials, ILogLabelProvider logLabelProvider, IHttpClient httpClient) { var formatter = new LokiBatchFormatter(logLabelProvider ?? new DefaultLogLabelProvider()); - var client = httpClient ?? new DefaultLokiHttpClient(); if (client is LokiHttpClient c) { @@ -21,11 +39,5 @@ private static LoggerConfiguration LokiHttpImpl(this LoggerSinkConfiguration sin return sinkConfiguration.Http(LokiRouteBuilder.BuildPostUri(credentials.Url), batchFormatter: formatter, httpClient: client); } - - public static LoggerConfiguration LokiHttp(this LoggerSinkConfiguration sinkConfiguration, string serverUrl) - => sinkConfiguration.LokiHttp(new NoAuthCredentials(serverUrl)); - - public static LoggerConfiguration LokiHttp(this LoggerSinkConfiguration sinkConfiguration, string serverUrl, string username, string password) - => sinkConfiguration.LokiHttp(new BasicAuthCredentials(serverUrl, username, password)); } -} \ No newline at end of file +} diff --git a/test/Serilog.Sinks.Loki.Tests/Infrastructure/TestLabelProvider.cs b/test/Serilog.Sinks.Loki.Tests/Infrastructure/TestLabelProvider.cs index 532aef6..c458850 100644 --- a/test/Serilog.Sinks.Loki.Tests/Infrastructure/TestLabelProvider.cs +++ b/test/Serilog.Sinks.Loki.Tests/Infrastructure/TestLabelProvider.cs @@ -14,5 +14,7 @@ public IList GetLabels() } public IList PropertiesAsLabels { get; set; } = new List(); + public IList PropertiesToAppend { get; set; } = new List(); + public LokiFormatterStrategy FormatterStrategy { get; set; } = LokiFormatterStrategy.SpecificPropertiesAsLabelsAndRestAppended; } } \ No newline at end of file From 88fb792c446ef143e000ac3059ed9f660c375ca5 Mon Sep 17 00:00:00 2001 From: "Kraemer, Benjamin" Date: Mon, 23 Nov 2020 15:18:00 +0100 Subject: [PATCH 2/3] Cleanup usings --- .../Models/ErrorViewModel.cs | 2 -- src/Serilog.Sinks.Loki.ExampleWebApp/Startup.cs | 8 +------- src/Serilog.Sinks.Loki/ContextualLabels.cs | 2 -- src/Serilog.Sinks.Loki/DefaultLokiHttpClient.cs | 6 +----- .../Fixtures/HttpClientTestFixture.cs | 2 -- test/Serilog.Sinks.Loki.Tests/Labels/LocalLabelsTests.cs | 3 --- 6 files changed, 2 insertions(+), 21 deletions(-) diff --git a/src/Serilog.Sinks.Loki.ExampleWebApp/Models/ErrorViewModel.cs b/src/Serilog.Sinks.Loki.ExampleWebApp/Models/ErrorViewModel.cs index 439182d..d36a6c2 100644 --- a/src/Serilog.Sinks.Loki.ExampleWebApp/Models/ErrorViewModel.cs +++ b/src/Serilog.Sinks.Loki.ExampleWebApp/Models/ErrorViewModel.cs @@ -1,5 +1,3 @@ -using System; - namespace Serilog.Sinks.Loki.ExampleWebApp.Models { public class ErrorViewModel diff --git a/src/Serilog.Sinks.Loki.ExampleWebApp/Startup.cs b/src/Serilog.Sinks.Loki.ExampleWebApp/Startup.cs index f91ab56..00c3093 100644 --- a/src/Serilog.Sinks.Loki.ExampleWebApp/Startup.cs +++ b/src/Serilog.Sinks.Loki.ExampleWebApp/Startup.cs @@ -1,12 +1,6 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.HttpsPolicy; -using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; diff --git a/src/Serilog.Sinks.Loki/ContextualLabels.cs b/src/Serilog.Sinks.Loki/ContextualLabels.cs index 944c00d..fe34ef6 100644 --- a/src/Serilog.Sinks.Loki/ContextualLabels.cs +++ b/src/Serilog.Sinks.Loki/ContextualLabels.cs @@ -1,5 +1,3 @@ -using Serilog.Core; - namespace Serilog.Sinks.Loki { /* public static class ContextualLabels diff --git a/src/Serilog.Sinks.Loki/DefaultLokiHttpClient.cs b/src/Serilog.Sinks.Loki/DefaultLokiHttpClient.cs index e110c82..6685084 100644 --- a/src/Serilog.Sinks.Loki/DefaultLokiHttpClient.cs +++ b/src/Serilog.Sinks.Loki/DefaultLokiHttpClient.cs @@ -1,8 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace Serilog.Sinks.Loki +namespace Serilog.Sinks.Loki { public class DefaultLokiHttpClient : LokiHttpClient { diff --git a/test/Serilog.Sinks.Loki.Tests/Fixtures/HttpClientTestFixture.cs b/test/Serilog.Sinks.Loki.Tests/Fixtures/HttpClientTestFixture.cs index 2e10ee4..b8c4c60 100644 --- a/test/Serilog.Sinks.Loki.Tests/Fixtures/HttpClientTestFixture.cs +++ b/test/Serilog.Sinks.Loki.Tests/Fixtures/HttpClientTestFixture.cs @@ -1,6 +1,4 @@ using System; -using JustEat.HttpClientInterception; -using Xunit; namespace Serilog.Sinks.Loki.Tests { diff --git a/test/Serilog.Sinks.Loki.Tests/Labels/LocalLabelsTests.cs b/test/Serilog.Sinks.Loki.Tests/Labels/LocalLabelsTests.cs index fd499b2..a30a6ea 100644 --- a/test/Serilog.Sinks.Loki.Tests/Labels/LocalLabelsTests.cs +++ b/test/Serilog.Sinks.Loki.Tests/Labels/LocalLabelsTests.cs @@ -1,7 +1,4 @@ -using System.Linq; -using Newtonsoft.Json; using Serilog.Sinks.Loki.Tests.Infrastructure; -using Shouldly; using Xunit; namespace Serilog.Sinks.Loki.Tests.Labels From c64b7180108626ac0a393abdb6ff18da50c077a9 Mon Sep 17 00:00:00 2001 From: "Kraemer, Benjamin" Date: Mon, 23 Nov 2020 15:29:12 +0100 Subject: [PATCH 3/3] Fixed example --- src/Serilog.Sinks.Loki.Example/Program.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Serilog.Sinks.Loki.Example/Program.cs b/src/Serilog.Sinks.Loki.Example/Program.cs index cae6f82..f3ffc00 100644 --- a/src/Serilog.Sinks.Loki.Example/Program.cs +++ b/src/Serilog.Sinks.Loki.Example/Program.cs @@ -13,7 +13,7 @@ static void Main(string[] args) Logger log = new LoggerConfiguration() .MinimumLevel.Verbose() .Enrich.FromLogContext() - .Enrich.WithProperty("MyPropertyName","MyPropertyValue") + .Enrich.WithProperty("MyLabelPropertyName","MyPropertyValue") .Enrich.WithThreadId() .WriteTo.Console() .WriteTo.LokiHttp(credentials, new LogLabelProvider(), new LokiExampleHttpClient()) @@ -46,6 +46,12 @@ static void Main(string[] args) log.Fatal("Fatal with Property A"); } + using (LogContext.PushProperty("MyAppendPropertyName", 1)) + { + log.Warning("Warning with Property MyAppendPropertyName"); + log.Fatal("Fatal with Property MyAppendPropertyName"); + } + log.Dispose(); } }