microsoft / FASTER

Fast persistent recoverable log and key-value store + cache, in C# and C++.
https://aka.ms/FASTER
MIT License
6.29k stars 563 forks source link

Checkpoints get deleted on recovery #884

Closed cschuchardt88 closed 9 months ago

cschuchardt88 commented 10 months ago

I set CheckpointSettings.RemoveOutdated, LogDevice.recoverDevice, ObjectLogDevice.recoverDevice and FasterKV.tryRecoverLatest all to false. Than i recover a checkpoint that is a snapshot. Now how do i get FasterDb to stop deleting the recovered checkpoint snapshot or overwriting it? Also everytime i create a snapshot the old checkpoints get deleted as well. How do i fix this?

_logSettings = new LogSettings()
{
    LogDevice = new ManagedLocalStorageDevice(Path.Combine(storePath, "LOG"), recoverDevice: false, osReadBuffering: true),
    ObjectLogDevice = new ManagedLocalStorageDevice(Path.Combine(storePath, "DATA"), recoverDevice: false, osReadBuffering: true),
    PageSizeBits = 9,
    MemorySizeBits = 21,
    SegmentSizeBits = 21,
    MutableFraction = 0.1,
    ReadCopyOptions = new ReadCopyOptions(ReadCopyFrom.AllImmutable, ReadCopyTo.MainLog),
    ReadCacheSettings = new ReadCacheSettings()
    {
        PageSizeBits = 21,
        MemorySizeBits = 30,
    },
};
_checkpointSettings = new CheckpointSettings()
{
    CheckpointManager = new DeviceLogCommitCheckpointManager(
        new LocalStorageNamedDeviceFactory(),
        new ConstructCheckpointNamingScheme(storePath)),
    RemoveOutdated = false,
};
Store = new(
    1L << 20,
    _logSettings,
    _checkpointSettings,
    serializerSettings: new SerializerSettings<byte[], byte[]>()
    {
        keySerializer = () => new ByteArrayBinaryObjectSerializer(),
        valueSerializer = () => new ByteArrayBinaryObjectSerializer(),
    },
    comparer: new ByteArrayFasterEqualityComparer(),
    tryRecoverLatest: false);
SessionPool = new AsyncPool<ClientSession<byte[], byte[], byte[], byte[], Empty, ByteArrayFunctions>>(
        _logSettings.LogDevice.ThrottleLimit,
        () => Store.For(new ByteArrayFunctions()).NewSession<ByteArrayFunctions>());
badrishc commented 10 months ago

If you create your own DeviceLogCommitCheckpointManager then you need to set removeOutdated to false in its constructor.