Help with running embedded with S3 Sync

[heynemann]

What would you like to be added or enhanced

I'm trying to understand how to run in embedded mode with S3 storage sync. Haven't found any examples in doc and been trying to understand the codebase for it, but it's been hard. I've managed to get the store to work locally and managed to create an S3 Remote Storage but haven't found how to plug em.

Why is this needed

To run immudb embedded with S3 sync.

Additional context

[ostafen]

Hi, @heynemann, thanks for the feedback. I'll get back to you shortly with a code snippet for running immudb with remote S3 storage.

[heynemann]

Any news @ostafen ?

[ostafen]

Hi, @heynemann, the following snippet should fit your scenario:

package main

import (
    "context"
    "log"
    "os"
    "path/filepath"
    "strings"

    "github.com/codenotary/immudb/embedded/appendable"
    "github.com/codenotary/immudb/embedded/appendable/multiapp"
    "github.com/codenotary/immudb/embedded/appendable/remoteapp"
    "github.com/codenotary/immudb/embedded/remotestorage/s3"
    "github.com/codenotary/immudb/embedded/store"
)

func getS3RemotePath(rootPath, subPath string) string {
    fsPath := filepath.Join(rootPath, subPath)

    return strings.ReplaceAll(
        fsPath+"/",
        string(filepath.Separator), "/",
    )
}

type Options struct {
    S3Endpoint            string
    S3RoleEnabled         bool
    S3Role                string
    S3AccessKeyID         string
    S3SecretKey           string
    S3BucketName          string
    S3Location            string
    S3PathPrefix          string
    S3InstanceMetadataURL string
}

func main() {
    var s3Opts Options // set your s3 options

    remoteStorage, err := s3.Open(
        s3Opts.S3Endpoint,
        s3Opts.S3RoleEnabled,
        s3Opts.S3Role,
        s3Opts.S3AccessKeyID,
        s3Opts.S3SecretKey,
        s3Opts.S3BucketName,
        s3Opts.S3Location,
        s3Opts.S3PathPrefix,
        s3Opts.S3InstanceMetadataURL,
    )
    if err != nil {
        panic(err)
    }

    opts := store.DefaultOptions()

    opts.WithAppFactory(func(rootPath, subPath string, opts *multiapp.Options) (appendable.Appendable, error) {
        remoteAppOpts := remoteapp.DefaultOptions()
        remoteAppOpts.Options = *opts

        s3Path := getS3RemotePath(rootPath, subPath)
        return remoteapp.Open(
            filepath.Join(rootPath, subPath),
            s3Path,
            remoteStorage,
            remoteAppOpts,
        )
    }).
        WithFileSize(1 << 20). // Reduce file size for better cache granularity
        WithAppRemoveFunc(func(rootPath, subPath string) error {
            s3Path := getS3RemotePath(rootPath, subPath)

            err = os.RemoveAll(filepath.Join(rootPath, subPath))
            if err != nil {
                return err
            }
            return remoteStorage.RemoveAll(context.Background(), s3Path)
        })

    store, err := store.Open("./data", opts)
    if err != nil {
        log.Fatal(err)
    }
    defer store.Close()
       // use your store
}

Let me know if you need further assistance.