Tool to convert an existing styfry LMDB to grain MongoDB

It would be nice to have a tool that can take a strfy database and convert it to a document store to use with grain MongoDB.

I looked at the two queries for both and came up with some example code. I don't have a strfry instance running and I don't know where to begin to try to connect to someone elses LMDB. Right now this is a very basic idea.

package main

import (
    "context"
    "encoding/json"
    "fmt"
    "log"

    "github.com/bmatsuo/lmdb-go/lmdb"
    "go.mongodb.org/mongo-driver/bson"
    "go.mongodb.org/mongo-driver/mongo"
    "go.mongodb.org/mongo-driver/mongo/options"
    "go.mongodb.org/mongo-driver/mongo/writeconcern"
)

const (
    lmdbPath     = "path/to/lmdb"             // Replace with your LMDB path
    mongoDBURI   = "mongodb://localhost:27017" // Replace with your MongoDB URI
    databaseName = "your_mongo_db_name"        // Replace with your MongoDB database name
)

type Event struct {
    ID        string     `json:"id" bson:"id"`
    PubKey    string     `json:"pubkey" bson:"pubkey"`
    CreatedAt int64      `json:"created_at" bson:"created_at"`
    Kind      int        `json:"kind" bson:"kind"`
    Tags      [][]string `json:"tags" bson:"tags"`
    Content   string     `json:"content" bson:"content"`
    Sig       string     `json:"sig" bson:"sig"`
}

func main() {
    // Initialize MongoDB client with write concern for durability
    client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(mongoDBURI).SetWriteConcern(writeconcern.New(writeconcern.WMajority())))
    if err != nil {
        log.Fatalf("Failed to connect to MongoDB: %v", err)
    }
    defer client.Disconnect(context.TODO())

    // Initialize LMDB environment and transaction
    env, err := lmdb.NewEnv()
    if err != nil {
        log.Fatalf("Failed to create LMDB environment: %v", err)
    }
    defer env.Close()

    env.SetMaxDBs(1000) // Adjust max DBs as needed
    err = env.Open(lmdbPath, lmdb.Readonly, 0644)
    if err != nil {
        log.Fatalf("Failed to open LMDB environment: %v", err)
    }

    txn, err := env.BeginTxn(nil, lmdb.Readonly)
    if err != nil {
        log.Fatalf("Failed to begin LMDB transaction: %v", err)
    }
    defer txn.Abort()

    // Replace with your LMDB DBI initialization
    dbi, err := txn.OpenDBI("mydbi", 0)
    if err != nil {
        log.Fatalf("Failed to open LMDB DBI: %v", err)
    }

    // Iterate over LMDB using a cursor
    cursor, err := txn.OpenCursor(dbi)
    if err != nil {
        log.Fatalf("Failed to open LMDB cursor: %v", err)
    }
    defer cursor.Close()

    for {
        key, value, err := cursor.Get(nil, nil, lmdb.Next)
        if err == lmdb.NotFound {
            break
        }
        if err != nil {
            log.Fatalf("Failed to retrieve key/value from LMDB: %v", err)
        }

        // Decode the LMDB value into the Event struct
        var event Event
        if err := json.Unmarshal(value, &event); err != nil {
            return fmt.Errorf("failed to decode event: %v", err)
        }

        // Insert event into MongoDB
        collectionName := fmt.Sprintf("event-kind%d", event.Kind)
        collection := client.Database(databaseName).Collection(collectionName)

        _, err = collection.InsertOne(context.TODO(), event)
        if err != nil {
            log.Fatalf("Failed to insert event into MongoDB: %v", err)
        }
    }

    fmt.Println("Data transfer from LMDB to MongoDB completed successfully")
}

0ceanSlim / grain

Tool to convert an existing styfry LMDB to grain MongoDB #4