Performance issue when generating 10k elements in array

[mdrokz]

Problem

Hello first of all thanks for the great library! it is really useful

Im using faker to seed my local database and i want to generate 10k elements the issue is if i use fakers default method faker.FakeData it hangs & takes up all of my memory

users := []domain.User{}

err = faker.FakeData(&users, options.WithRandomMapAndSliceMinSize(5000), options.WithRandomMapAndSliceMaxSize(10000))

But if i do it manually like this it finishes in under 1 second

for i := 0; i < 10000; i++ {
        user := domain.User{}
        // err = faker.FakeData(&user)
        user.Name = faker.Name()
        user.Email = faker.Email()
        user.PhoneNumber = faker.Phonenumber()
        user.OrganizationID = org.ID

        user.Locations = []domain.Location{}
        user.Services = []domain.Service{}

        _ = faker.FakeData(&user.Locations, options.WithRandomMapAndSliceMinSize(5), options.WithRandomMapAndSliceMaxSize(10))
        _ = faker.FakeData(&user.Services, options.WithRandomMapAndSliceMinSize(5), options.WithRandomMapAndSliceMaxSize(10))

        if err != nil {
            panic(err)
        }
        users = append(users, user)

    }

here is the full code & the models

domain.go

type User struct {
    ID             uuid.UUID      `json:"id" gorm:"primary_key; unique; type:uuid; column:id; default:uuid_generate_v4();"`
    OrganizationID uuid.UUID      `json:"organization_id" faker:"orgID"`
    Organization   *Organization  `json:"organization"`
    Name           string         `json:"name" faker:"name"`
    Email          string         `json:"email" faker:"email"`
    PhoneNumber    string         `json:"phone_number" faker:"phone_number"`
    Photo          string         `json:"photo" faker:"-"`
    AuditLogs      []AuditLog     `json:"audit_logs" gorm:"foreignKey:PerformerID;references:ID;" faker:"-"`
    Locations      []Location     `json:"locations" gorm:"many2many:location_user;" faker:"locations"`
    Services       []Service      `json:"services" gorm:"many2many:service_user;" faker:"services"`
    FilterPresets  datatypes.JSON `json:"filter_presets" gorm:"type:jsonb" faker:"-"`
    Role           userRole       `json:"role" sql:"type:user_role" faker:"-"`
    CreatedAt      time.Time      `json:"created_at" gorm:"autoCreateTime" faker:"-"`
}

type Location struct {
    ID              uuid.UUID      `json:"id" gorm:"primary_key; unique; type:uuid; column:id; default:uuid_generate_v4();" faker:"-"`
    OrganizationID  uuid.UUID      `json:"organization_id" gorm:"type:uuid;" faker:"orgID"`
    Name            string         `json:"name" gorm:"type:varchar(200); not null;" faker:"cityName"`
    Description     string         `json:"description" faker:"sentence"`
    Addresses       pq.StringArray `json:"addresses" gorm:"type:text[]" faker:"-"`
    Logo            string         `json:"logo" faker:"-"`
    SocialLinks     pq.StringArray `json:"social_links" gorm:"type:text[]" faker:"-"`
    WorkingHours    datatypes.JSON `json:"working_hours" gorm:"type:jsonb; not null;" faker:"jsonb"`
    ServiceSettings datatypes.JSON `json:"service_settings" gorm:"type:jsonb; not null;" faker:"jsonb"`
    Kiosks          datatypes.JSON `json:"kiosks" gorm:"type:jsonb; not null;" faker:"jsonb"`
    TV              datatypes.JSON `json:"tv" gorm:"type:jsonb; not null;" faker:"jsonb"`
    Bookings        []*Booking     `json:"bookings" gorm:"foreignKey:LocationID;references:ID;" faker:"-"`
    Services        []*Service     `json:"services" gorm:"many2many:location_service;" faker:"-"`
    Users           []User         `json:"users" gorm:"many2many:location_user;" faker:"-"`
    CreatedAt       time.Time      `json:"created_at" gorm:"autoCreateTime" faker:"-"`
}

type Service struct {
    ID               uuid.UUID      `json:"id" gorm:"primary_key; unique; type:uuid; column:id; default:uuid_generate_v4();" faker:"orgID"`
    Name             string         `json:"name" gorm:"type:varchar(200); not null;" faker:"name"`
    Description      string         `json:"description" faker:"sentence"`
    Bookings         []*Booking     `json:"bookings" gorm:"foreignKey:ServiceID;references:ID;" faker:"-"`
    Locations        []*Location    `json:"locations" gorm:"many2many:location_service;" faker:"-"`
    Users            []User         `json:"users" gorm:"many2many:service_user;" faker:"-"`
    BookingForm      datatypes.JSON `json:"booking_form" gorm:"type:jsonb" faker:"jsonb"`
    WelcomePage      datatypes.JSON `json:"welcome_page" gorm:"type:jsonb" faker:"jsonb"`
    ConfirmationPage datatypes.JSON `json:"confirmation_page" gorm:"type:jsonb" faker:"jsonb"`
    CreatedAt        time.Time      `json:"created_at" gorm:"autoCreateTime" faker:"-"`
}

main.go

package main

import (
    "log"
    "os"
    "reflect"
    "time"

    "github.com/IBM/fp-go/option"
    "github.com/go-faker/faker/v4"
    "github.com/go-faker/faker/v4/pkg/options"
    "github.com/makerstudio-io/livqueue/libs/domain"
    "github.com/makerstudio-io/livqueue/libs/utils"
    "gorm.io/driver/postgres"
    "gorm.io/gorm"
    "gorm.io/gorm/logger"
)

var (
    _ = faker.AddProvider("cityName", func(v reflect.Value) (interface{}, error) {
        return faker.GetRealAddress().City, nil
    })

    _ = faker.AddProvider("locations", func(v reflect.Value) (interface{}, error) {
        locations := []domain.Location{}
        err := faker.FakeData(&locations, options.WithRandomMapAndSliceMinSize(5), options.WithRandomMapAndSliceMaxSize(10))
        return locations, err
    })

    _ = faker.AddProvider("services", func(v reflect.Value) (interface{}, error) {
        services := []domain.Service{}
        err := faker.FakeData(&services, options.WithRandomMapAndSliceMinSize(5), options.WithRandomMapAndSliceMaxSize(10))
        return services, err
    })

    _ = faker.AddProvider("jsonb", func(v reflect.Value) (interface{}, error) {
        return []byte{'{', '}'}, nil
    })

    _ = faker.AddProvider("users", func(v reflect.Value) (interface{}, error) {
        user := domain.User{}
        err := faker.FakeData(&user)
        return user, err
    })
)

// _ = faker.AddProvider()

func main() {

    conn := utils.ConstructDBConnString(utils.PostgresConfig{
        SslMode: option.Some("disable"),
    })

    newLogger := logger.New(
        log.New(os.Stdout, "\r\n", log.LstdFlags), // io writer
        logger.Config{
            SlowThreshold:             time.Second,   // Slow SQL threshold
            LogLevel:                  logger.Silent, // Log level
            IgnoreRecordNotFoundError: true,          // Ignore ErrRecordNotFound error for logger
            ParameterizedQueries:      false,         // Don't include params in the SQL log
            Colorful:                  true,          // Disable color
        },
    )

    opts := gorm.Config{
        Logger: newLogger,
    }

    db, err := gorm.Open(postgres.Open(conn), &opts)

    if err != nil {
        panic(err)
    }

    var org domain.Organization

    res := db.Model(&domain.Organization{}).First(&org)

    if res.Error != nil {
        panic(res.Error)
    }

    _ = faker.AddProvider("orgID", func(v reflect.Value) (interface{}, error) {
        return org.ID, nil
    })

    users := []domain.User{}

    // err = faker.FakeData(&users, options.WithRandomMapAndSliceMinSize(5000), options.WithRandomMapAndSliceMaxSize(10000))

    // for loop over 10000
    for i := 0; i < 10000; i++ {
        user := domain.User{}
        // err = faker.FakeData(&user)
        user.Name = faker.Name()
        user.Email = faker.Email()
        user.PhoneNumber = faker.Phonenumber()
        user.OrganizationID = org.ID

        user.Locations = []domain.Location{}
        user.Services = []domain.Service{}

        _ = faker.FakeData(&user.Locations, options.WithRandomMapAndSliceMinSize(5), options.WithRandomMapAndSliceMaxSize(10))
        _ = faker.FakeData(&user.Services, options.WithRandomMapAndSliceMinSize(5), options.WithRandomMapAndSliceMaxSize(10))

        if err != nil {
            panic(err)
        }
        users = append(users, user)

    }

    if err != nil {
        panic(err)
    }

    println(len(users), len(users[0].Locations), users[0].Locations[0].WorkingHours, len(users[0].Services))

    req := db.CreateInBatches(&users, 500)

    if req.Error != nil {
        panic(req.Error)
    }
}

[bayaderpack]

Do it in goroutine try like this func GenerateData() []domain.User{}{ var wg sync.WaitGroup users := []domain.User{} for i := 0; i < 10000; i++ { wg.Add(1) go func(no int) { defer wg.Done()

                user := domain.User{}
    // err = faker.FakeData(&user)
    user.Name = faker.Name()
    user.Email = faker.Email()
    user.PhoneNumber = faker.Phonenumber()
    user.OrganizationID = org.ID

    user.Locations = []domain.Location{}
    user.Services = []domain.Service{}

    _ = faker.FakeData(&user.Locations, options.WithRandomMapAndSliceMinSize(5), options.WithRandomMapAndSliceMaxSize(10))
    _ = faker.FakeData(&user.Services, options.WithRandomMapAndSliceMinSize(5), options.WithRandomMapAndSliceMaxSize(10))

    if err != nil {
        panic(err)
    }
    users = append(users, user)
    }(i + 1)
}

wg.Wait()
return users

} I dont test the code I just thinking about something like this

[bxcodec]

Interesting. Will need a focus weekend to work on this. I'll pick up and investigate whenever I'm free. But if you have the fixes, please help raise the PR whenever you're ready.