Support redaction through Microsoft​.Extensions​.Compliance​.Redaction ?

[zyofeng]

Currently I'm using Destructurama.Attributed to redact sensitive information in the logs. That is quite static, and would be nice to be able to customize redaction using Microsoft​.Extensions​.Compliance​.Redaction and more precisely through DataClassification attribute.

Something like this (adapted from Destructurama.Attributed):

using System.Collections;
using System.Collections.Concurrent;
using System.Diagnostics.CodeAnalysis;
using System.Linq.Expressions;
using System.Reflection;
using Destructurama.Attributed;
using Microsoft.Extensions.Compliance.Classification;
using Microsoft.Extensions.Compliance.Redaction;
using Serilog.Core;
using Serilog.Debugging;
using Serilog.Events;

namespace NZFM.Common.ApplicationInsightsLogging.AspNetCore;

internal sealed class RedactionDestructuringPolicy(IRedactorProvider redactorProvider) : IDestructuringPolicy
{
    private static readonly ConcurrentDictionary<Type, CacheEntry> _cache = new();

    internal static readonly IPropertyOptionalIgnoreAttribute Instance = new NotLoggedIfNullAttribute();

    public bool TryDestructure(object value, ILogEventPropertyValueFactory propertyValueFactory,
        [NotNullWhen(true)] out LogEventPropertyValue? result)
    {
        var cached = _cache.GetOrAdd(value.GetType(), CreateCacheEntry);
        result = cached.DestructureFunc(value, propertyValueFactory);
        return cached.CanDestructure;
    }

    private CacheEntry CreateCacheEntry(Type type)
    {
        static T? GetCustomAttribute<T>(PropertyInfo propertyInfo) => propertyInfo.GetCustomAttributes().OfType<T>().FirstOrDefault();

        var classDestructurer = type.GetCustomAttributes().OfType<ITypeDestructuringAttribute>().FirstOrDefault();
        if (classDestructurer != null)
            return new(classDestructurer.CreateLogEventPropertyValue);

        var properties = GetPropertiesRecursive(type).ToList();
        if (properties.All(pi =>
            GetCustomAttribute<IPropertyDestructuringAttribute>(pi) == null
            && GetCustomAttribute<IPropertyOptionalIgnoreAttribute>(pi) == null
            && pi.GetCustomAttributes()
                .All(attr => attr is not DataClassificationAttribute)))
        {
            return CacheEntry.Ignore;
        }

        var optionalIgnoreAttributes = properties
            .Select(pi => new { pi, Attribute = GetCustomAttribute<IPropertyOptionalIgnoreAttribute>(pi) })
            .Where(o => o.Attribute != null)
            .ToDictionary(o => o.pi, o => o.Attribute);

        var destructuringAttributes = properties
            .Select(pi => new { pi, Attribute = GetCustomAttribute<IPropertyDestructuringAttribute>(pi) })
            .Where(o => o.Attribute != null)
            .ToDictionary(o => o.pi, o => o.Attribute);

        var dataClassificationAttributes = properties
            .Select(pi => new
            {
                pi, Attribute = GetCustomAttribute<DataClassificationAttribute>(pi)
            })
            .Where(o => o.Attribute != null)
            .ToDictionary(o => o.pi, o => o.Attribute);

        if (!optionalIgnoreAttributes.Any() && !destructuringAttributes.Any() && !dataClassificationAttributes.Any() && typeof(IEnumerable).IsAssignableFrom(type))
            return CacheEntry.Ignore;

        var propertiesWithAccessors = properties.Select(p => (p, Compile(p))).ToList();
        return new CacheEntry((o, f) => MakeStructure(o, propertiesWithAccessors,
            optionalIgnoreAttributes, 
            destructuringAttributes,
            dataClassificationAttributes, f, type));

        static Func<object, object> Compile(PropertyInfo property)
        {
            var objParameterExpr = Expression.Parameter(typeof(object), "instance");
            var instanceExpr = Expression.Convert(objParameterExpr, property.DeclaringType);
            var propertyExpr = Expression.Property(instanceExpr, property);
            var propertyObjExpr = Expression.Convert(propertyExpr, typeof(object));
            return Expression.Lambda<Func<object, object>>(propertyObjExpr, objParameterExpr).Compile();
        }
    }
    private static IEnumerable<PropertyInfo> GetPropertiesRecursive(Type type)
    {
        var seenNames = new HashSet<string>();

        while (type != typeof(object))
        {
            var unseenProperties = type
                .GetProperties(BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly)
                .Where(p => p.CanRead && p.GetMethod.IsPublic && p.GetIndexParameters().Length == 0 &&
                            !seenNames.Contains(p.Name));

            foreach (var propertyInfo in unseenProperties)
            {
                seenNames.Add(propertyInfo.Name);
                yield return propertyInfo;
            }

            type = type.BaseType;
        }
    }
    private LogEventPropertyValue MakeStructure(
        object o,
        List<(PropertyInfo Property, Func<object, object> Accessor)> loggedProperties,
        Dictionary<PropertyInfo, IPropertyOptionalIgnoreAttribute> optionalIgnoreAttributes,
        Dictionary<PropertyInfo, IPropertyDestructuringAttribute> destructuringAttributes,
        Dictionary<PropertyInfo, DataClassificationAttribute> dataClassificationAttributes,
        ILogEventPropertyValueFactory propertyValueFactory,
        Type type)
    {
        var structureProperties = new List<LogEventProperty>();
        foreach (var (pi, accessor) in loggedProperties)
        {
            object propValue;
            try
            {
                propValue = accessor(o);
            }
            catch (Exception ex)
            {
                SelfLog.WriteLine("The property accessor {0} threw exception {1}", pi, ex);
                propValue = $"The property accessor threw an exception: {ex.GetType().Name}";
            }

            if (optionalIgnoreAttributes.TryGetValue(pi, out var optionalIgnoreAttribute) && optionalIgnoreAttribute.ShouldPropertyBeIgnored(pi.Name, propValue, pi.PropertyType))
                continue;

            if (Instance.ShouldPropertyBeIgnored(pi.Name, propValue, pi.PropertyType))
                continue;

            if (destructuringAttributes.TryGetValue(pi, out var destructuringAttribute))
            {
                if (destructuringAttribute.TryCreateLogEventProperty(pi.Name, propValue, propertyValueFactory, out var property))
                    structureProperties.Add(property);
            }
            else if (dataClassificationAttributes.TryGetValue(pi, out var dataClassificationAttribute))
            {
                var redactor = redactorProvider.GetRedactor(dataClassificationAttribute.Classification);
                var redacted = redactor.Redact(propValue);
                if (!string.IsNullOrEmpty(redacted))
                    structureProperties.Add(new LogEventProperty(pi.Name, new ScalarValue(redacted)));
            }
            else
            {
                structureProperties.Add(new(pi.Name, propertyValueFactory.CreatePropertyValue(propValue, true)));
            }
        }

        return new StructureValue(structureProperties, type.Name);
    }

    internal static void Clear()
    {
        _cache.Clear();
    }
}

internal readonly struct CacheEntry
{
    public CacheEntry(Func<object, ILogEventPropertyValueFactory, LogEventPropertyValue> destructureFunc)
    {
        CanDestructure = true;
        DestructureFunc = destructureFunc;
    }

    private CacheEntry(bool canDestructure,
        Func<object, ILogEventPropertyValueFactory, LogEventPropertyValue?> destructureFunc)
    {
        CanDestructure = canDestructure;
        DestructureFunc = destructureFunc;
    }

    public bool CanDestructure { get; }

    public Func<object, ILogEventPropertyValueFactory, LogEventPropertyValue?> DestructureFunc { get; }

    public static CacheEntry Ignore { get; } = new(false, (_, _) => null);
}

[nblumhardt]

Hi @zyofeng, thanks for dropping by! This would be nice to see :+1:

It's not really in scope for this repository, and I'm not sure if any Serilog maintainers have time carved out for it right now; perhaps NZ Funds would be open to publishing this independently as an add-on package, in the same way that the Destructurama packages are published?

[zyofeng]

Happy to take this one and create a separate package. But I would need some suggestion on how to handle IRedactorFactory dependancy, my understanding of serilog design is that it's agnostic to the underlying DI library?

[nblumhardt]

Hi Mike,

Yes, that's normally the case - when using Serilog.Extensions.Hosting, however, there's an AddSerilog() overload that provides IServiceCollection, from which the factory can be retrieved:

builder.Services.AddSerilog((services, loggerConfiguration) => loggerConfiguration
  .Destructure.WithRedaction(services.GetRequiredService<IRedactorFactory>()))

(I'm not 100% sure of the parameter ordering etc., but this should be pretty close :-))

Keen to hear how you go, and please drop me a line if you need any more info.

[zyofeng]

I've put something together, feedbacks are welcome

https://github.com/zyofeng/serilog-redaction/tree/master/src

[nblumhardt]

Looks great! :+1:

[zyofeng]

Published!

https://www.nuget.org/packages/Serilog.Redaction/1.0.1