Is it possible to use a custom pickler for `System.Collections.Immutable.ImmutableList` ?

[marklam]

Here's what I've tried:

#r "nuget:FsPickler"

open System.IO
open System.Collections.Immutable
open MBrace.FsPickler

let mkPickler<'t> (resolver : IPicklerResolver) =
    let seqPickler = resolver.Resolve<'t seq> ()

    let writer (w : WriteState) (ns : ImmutableList<'t>) =
        seqPickler.Write w "value" (ns :> 't seq)

    let reader (r : ReadState) : ImmutableList<'t> =
        ImmutableList.CreateRange<'t>(seqPickler.Read r "value")

    Pickler.FromPrimitives(reader, writer)

let resolver : IPicklerResolver =
    let registry = CustomPicklerRegistry()
    registry.RegisterFactory mkPickler
    PicklerCache.FromCustomPicklerRegistry registry

let binarySerializer = FsPickler.CreateBinarySerializer(picklerResolver = resolver)

let storeAndCount value =
    use s = new MemoryStream()
    binarySerializer.Serialize (s, value)
    s.ToArray().Length

let items = seq { 1 .. 100 }
let immutableList = ImmutableList.CreateRange items

printfn "Storing items gives %d bytes" (storeAndCount items)
printfn "Storing ImmutableList gives %d bytes" (storeAndCount immutableList)

But the ImmutableList doesn't seem to trigger my serializer:

Storing items gives 213 bytes
MBrace.FsPickler.NonSerializableTypeException: Type 'System.Collections.Immutable.ImmutableList`1[System.Int32]' is not serializable.
   at MBrace.FsPickler.Utils.Exn`1.get_Value() in C:\Users\eitsarpa\devel\mbrace\FsPickler\src\FsPickler\Utils\Utils.fs:line 59
   at MBrace.FsPickler.PicklerCache.MBrace-FsPickler-IPicklerResolver-Resolve[T]() in C:\Users\eitsarpa\devel\mbrace\FsPickler\src\FsPickler\PicklerGeneration\PicklerCache.fs:line 75
   at MBrace.FsPickler.FsPicklerSerializer.Serialize[T](Stream stream, T value, FSharpOption`1 pickler, FSharpOption`1 streamingContext, FSharpOption`1 encoding, FSharpOption`1 leaveOpen) in C:\Users\eitsarpa\devel\mbrace\FsPickler\src\FsPickler\FsPickler\Serializer.fs:line 60
   at FSI_0004.storeAndCount[a](a value) in C:\Users\markl\AppData\Local\Temp\uw5uvhcg..fsx:line 27
   at <StartupCode$FSI_0004>.$FSI_0004.main@() in C:\Users\markl\AppData\Local\Temp\uw5uvhcg..fsx:line 34
   at System.RuntimeMethodHandle.InvokeMethod(Object target, Void** arguments, Signature sig, Boolean isConstructor)
   at System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags invokeAttr)
Stopped due to error

[ingted]

You need to provide the generic type parameter:

open System.IO
open System.Collections.Immutable
open MBrace.FsPickler

let mkPickler<'t> (resolver : IPicklerResolver) =
    let tPickler = resolver.Resolve<'t> ()
    let intPickler = resolver.Resolve<int> ()

    let writer (w : WriteState) (ns : ImmutableList<'t>) =
        let tArr = ns |> Seq.toArray
        intPickler.Write w "cnt" tArr.Length
        tArr
        |> Array.iter (fun t ->
            tPickler.Write w "value" t
        )

    let reader (r : ReadState) : ImmutableList<'t> =
        let l = intPickler.Read r "cnt"
        let tArr = 
            [|0..(l-1)|]
            |> Array.map (fun _ -> 
                tPickler.Read r "value"
            )
        ImmutableList.CreateRange<'t> tArr

    Pickler.FromPrimitives(reader, writer)

let resolver : IPicklerResolver =
    let registry = CustomPicklerRegistry()
    registry.RegisterFactory mkPickler
    PicklerCache.FromCustomPicklerRegistry registry

let p = mkPickler<int> resolver

let binarySerializer = FsPickler.CreateBinarySerializer() //picklerResolver = resolver)

let storeAndCount value =
    use s = new MemoryStream()
    binarySerializer.Serialize (s, value, p)
    s.ToArray().Length

let items = seq { 1 .. 100 }
let immutableList = ImmutableList.CreateRange items

printfn "Storing ImmutableList gives %d bytes" (storeAndCount immutableList)

let resolver2 : IPicklerResolver =
    let registry = CustomPicklerRegistry()
    registry.RegisterFactory mkPickler<int>
    PicklerCache.FromCustomPicklerRegistry registry

let binarySerializer2 = FsPickler.CreateBinarySerializer(picklerResolver = resolver2)

let storeAndCount2 value =
    use s = new MemoryStream()
    binarySerializer2.Serialize (s, value)
    s.ToArray().Length

printfn "Storing items gives %d bytes" (storeAndCount2 items)
printfn "Storing ImmutableList gives %d bytes" (storeAndCount2 immutableList)

[ingted]

[ingted]

    /// Registers a user-specified pickler factory
    member __.RegisterFactory<'T>(factory : IPicklerResolver -> Pickler<'T>) : unit =
        registerPickler (fun () -> picklerFactories.[typeof<'T>] <- fun r -> factory r :> Pickler)

[ingted]

Hi @marklam ,

I implemented a version supporting type with generic parameter. And I push it here (I compiled FsPickler with .net 9)

FsPickler9

And the picklerFactory register code should be declared with a static method...

Even though .NET 9 is likely to be the last version for FsPickler, as BinaryFormatter is no longer supported—a tear of the times—if it's still useful, please take it.

open System
open System.IO
open System.Collections.Immutable
open MBrace.FsPickler

type A () =
    static member mkPickler<'t> (resolver : IPicklerResolver) =
        let seqPickler = resolver.Resolve<'t seq> ()

        let writer (w : WriteState) (ns : ImmutableList<'t>) =
            seqPickler.Write w "value" (ns :> 't seq)

        let reader (r : ReadState) : ImmutableList<'t> =
            ImmutableList.CreateRange<'t>(seqPickler.Read r "value")

        Pickler.FromPrimitives(reader, writer)

let mkPickler (t:Type) (argTyps:Type[]) (resolver : IPicklerResolver) =
    let gmi = typedefof<A>.GetMethod("mkPickler").GetGenericMethodDefinition() //to receive [| typeof<int> |]
    let mi = gmi.MakeGenericMethod argTyps
    mi.Invoke(null, [|resolver|]) |> unbox<_>

let resolver =
    let registry = CustomPicklerRegistry()
    registry.RegisterFactory<ImmutableList<'t>> mkPickler
    PicklerCache.FromCustomPicklerRegistry registry :> IPicklerResolver

let binarySerializer = FsPickler.CreateBinarySerializer(picklerResolver = resolver)

let storeAndCount value =
    use s = new MemoryStream()
    binarySerializer.Serialize (s, value)
    s.ToArray().Length

let items = seq { 1 .. 100 }
let immutableList = ImmutableList.CreateRange items

let imutlistArr = storeAndCount immutableList

printfn "Storing items gives %d bytes" (storeAndCount items).Length
printfn "Storing ImmutableList gives %d bytes" imutlistArr.Length

let imutlist = binarySerializer.UnPickle<ImmutableList<int>> imutlistArr // unbox<ImmutableList<int>>

printfn "Unpickled count: %d" imutlist.Count

And the output result:

Storing items gives 213 bytes
Storing ImmutableList gives 2696 bytes
Unpickled count: 100

[ingted]

By the way, FsPickler9 doesn't need customized pickler to support ImmutableList...

[marklam]

Thanks for the suggestions!

[ingted]

I fixed some issues commit

And all these styles are supported now:

registry.RegisterFactory<ImmutableList<'t>> mkPickler

registry.RegisterFactory<ImmutableList<int>> A.mkPickler<int>

registry.RegisterFactory A.mkPickler<int>

The result would be

Storing items gives 213 bytes
Storing ImmutableList gives 478 bytes
Unpickled count: 100

Please replace the code in resolver variable

open System
open System.IO
open System.Collections.Immutable
open MBrace.FsPickler

type A () =
    static member mkPickler<'t> (resolver : IPicklerResolver) =
        let tt = typeof<'t>
        let tPickler = resolver.Resolve<'t> ()
        let intPickler = resolver.Resolve<int> ()

        let writer (w : WriteState) (ns : ImmutableList<'t>) =
            let tArr = ns |> Seq.toArray
            intPickler.Write w "cnt" tArr.Length
            tArr
            |> Array.iter (fun t ->
                tPickler.Write w "value" t
            )

        let reader (r : ReadState) : ImmutableList<'t> =
            let l = intPickler.Read r "cnt"
            let tArr = 
                [|0..(l-1)|]
                |> Array.map (fun _ -> 
                    tPickler.Read r "value"
                )
            ImmutableList.CreateRange<'t> tArr

        Pickler.FromPrimitives(reader, writer) //:> Pickler

let mkPickler (t:Type) (argTyps:Type[]) (resolver : IPicklerResolver) =
    //let methodTypArg = t.GetGenericTypeDefinition().MakeGenericType argTyps
    let gmi = typedefof<A>.GetMethod("mkPickler").GetGenericMethodDefinition()
    let mi = gmi.MakeGenericMethod argTyps
    let p = mi.Invoke(null, [|resolver|]) 
    p |> unbox<Pickler>

let resolver =
    let registry = CustomPicklerRegistry()

    registry.RegisterFactory A.mkPickler<int>

    PicklerCache.FromCustomPicklerRegistry registry :> IPicklerResolver

let binarySerializer = FsPickler.CreateBinarySerializer(picklerResolver = resolver)
//let binarySerializer = FsPickler.CreateBinarySerializer()

let storeAndCount value =
    use s = new MemoryStream()
    binarySerializer.Serialize (s, value)
    s.ToArray()

let items = seq { 1 .. 100 }
let immutableList = ImmutableList.CreateRange items

let seqArr = storeAndCount items
let imutlistArr = storeAndCount immutableList

printfn "Storing items gives %d bytes" seqArr.Length
printfn "Storing ImmutableList gives %d bytes" imutlistArr.Length

let seq__ = binarySerializer.UnPickle<seq<int>> seqArr |> Seq.toArray// unbox<ImmutableList<int>>
let imutlist = binarySerializer.UnPickle<ImmutableList<int>> imutlistArr // unbox<ImmutableList<int>>

printfn "Unpickled count: %d" imutlist.Count

And

let binarySerializer = FsPickler.CreateBinarySerializer()

The result would be

Storing items gives 213 bytes
Storing ImmutableList gives 2696 bytes
Unpickled count: 100

Also works!!

[ingted]

PS. to run the unit tests of fspickler, please build fspickler9 with release profile...

Not sure why these codes work

binarySerializer.Pickle (new HashSet<int>([|1;2;3;|])) |> binarySerializer.UnPickle<HashSet<int>>
binarySerializer.Pickle (new HashSet<int*string>([|5, "gg";2, "yy";3 , "oo";|])) |> binarySerializer.UnPickle<HashSet<int*string>>

But unit test failed (so as others...)

    [<Test; Category("Generic BCL Types")>]
    member __.``BCL: System-Collections-Generic-HashSet`` () =
        if runsOnMono then () else // skip due to mono 5.2 bug

        let testSet (data : seq<'T>) =
            let data = data |> Seq.distinct |> Seq.toList
            let d = new HashSet<'T>(data)
            let data' = testRoundtrip d |> Seq.toList
            data' |> should equal data

        Check.QuickThrowOnFail<seq<int64>> testSet
        Check.QuickThrowOnFail<seq<string>> testSet
        Check.QuickThrowOnFail<seq<int * string>> testSet