Recovery failure on deserializing snapshot

Version Information Version of Akka.NET? 1.5.13 Which Akka.NET Modules? Akka.Persistence.Redis, Akka.Serialization.Hyperion

Describe the bug Rarely, an actor couldn’t get started due to snapshot recovery failure, and this exception occurs:

---> System.Runtime.Serialization.SerializationException: Failed to deserialize object of type [oms.models.ActorSnapshots.UserActorState] from the stream. Cause: Failed to deserialize object of type [System.Collections.Generic.List`1[domain.Entities.OrderBaseEntity]] from the stream. Cause: Failed to deserialize object of type [domain.Entities.OrderSellEntity] from the stream. Cause: Unable to cast object of type 'domain.Entities.TradeItem' to type 'domain.Enums.OrderValidityEnum'.

To Reproduce it's not reproducible.

Expected behavior To Deserialize Snapshot state successfully. This is the State of the actor which is persisted :

public class UserActorState
{
    public List<OrderBaseEntity> Orders { get; set; } = new();
}

public abstract class OrderBaseEntity : BaseEntity
{
    public sealed override string Id { get; set; }
    public string ParentId { get; set; }
    public string ReferenceId { get; set; } = string.Empty;// algo parentId 
    public long Hon { get; set; }
    public string SymbolIsin { get; set; } 
    public string SymbolName { get; set; }
    public OrderValidityEnum Validity { get; set; }  
    public DateTime? ValidityDate { get; set; } 
    public decimal Price { get; set; } 
    public long Quantity { get; set; } 
    public virtual OrderSide Side { get; protected set; } 
    public string CustomerIsin { get; set; } 
    public OrderState OrderState { get; set; }
    public OrderFrom OrderFrom { get; set; }
    public DateTime? ModifiedAt { get; set; }
    public DateTime? ParentCreationDate { get; set; }
    public long ParentBlock { get; set; }
    public bool IsModifyToUpperCost { get; set; } = false;
    public long ParentRemainedQuantity { get; set; }
    public int OrderErrorCode { get; set; }
    public string OrderErrorText { get; set; }
    public OrderActionEnum OrderAction { get; set; } = OrderActionEnum.Add;
    public int? HidePrice { get; set; }
    public byte? SettlementDelay { get; set; }
    public int CSize { get; protected set; } = 1;//option
    public List<TradeItem> Trades { get; set; } = new();
}

public abstract class BaseEntity
{
    protected BaseEntity()
    {
    }

    protected BaseEntity(string identifier)
    {
        Id = identifier;
    }

    public virtual DateTime CreateDateTime { get; set; } = DateTime.Now;

    [Key, MaxLength(36)]
    public virtual string Id { get; set; } = Ulid.NewUlid().ToString();

    private List<IDomainEvent> _domainEvents;

    [JsonIgnore]
    public IReadOnlyCollection<IDomainEvent> DomainEvents => _domainEvents?.AsReadOnly();
}

public class TradeItem
{
    public TradeItem(DateTime dateTime,
        int tradeNumber,
        string orderId,
        string symbolIsin,
        OrderSide side,
        int? hidePrice,
        long quantity,
        int price,
        bool isCancel)
    {
        DateTime   =dateTime;
        TradeNumber=tradeNumber;
        OrderId    =orderId;
        SymbolIsin =symbolIsin;
        Side       =side;
        HidePrice  =hidePrice;
        Quantity   =quantity;
        Price      =price;
        IsCancel   =isCancel;
    }
    public DateTime DateTime { get; set; }
    public int TradeNumber { get; set; }
    public string OrderId { get; set; }
    public string SymbolIsin { get; set; }
    public OrderSide Side { get; set; }
    public int? HidePrice { get; set; }
    public long Quantity { get; set; }
    public int Price { get; set; }
    public bool IsCancel { get; set; } = false;
}

public class OrderBuyEntity : OrderBaseEntity
{
   public override OrderSide Side { get; protected set; } = OrderSide.Buy;
}

public class OrderSellEntity : OrderBaseEntity
{
   public override OrderSide Side { get; protected set; } = OrderSide.Sell;
}

public enum OrderValidityEnum
{
    DAY = 0,
    GOOD_TILL_DATE = 1
}

Actual behavior Sometimes Failed to recover snapshot state

Environment docker, .net8

Additional context I've configured Hyperion serializer like this:

akka { actor { provider = cluster serializers { hyperion = "Akka.Serialization.HyperionSerializer, Akka.Serialization.Hyperion" } serialization-bindings { "System.Object" = hyperion } }

akkadotnet / Akka.Persistence.Redis

Recovery failure on deserializing snapshot #297