search

go-gorm / gorm

The fantastic ORM library for Golang, aims to be developer friendly
https://gorm.io
MIT License
36.71k stars 3.92k forks source link

fatal error: concurrent map read and map write [with reproduction] #6298

Open glebtv opened 1 year ago

glebtv commented 1 year ago

GORM Playground Link

Reproducible, but might need to run 100+ times to get it.

Had to disable migrations, siince they seem to parse the schemas and issue doesn't occur

https://github.com/go-gorm/playground/pull/594

Description


2023/05/05 19:48:44 /data/playground/main_test.go:17
[0.616ms] [rows:0] SELECT * FROM `users` WHERE `users`.`deleted_at` IS NULL
PASS
ok      gorm.io/playground  0.005s
2023/05/05 19:48:44 testing sqlite3...
fatal error: concurrent map read and map write

goroutine 7 [running]:
gorm.io/gorm/schema.Schema.LookUpField(...)
    /data/playground/gorm/schema/schema.go:66
gorm.io/gorm/schema.(*Schema).guessRelation(0xc0001d0000?, 0xc0000d4cf0?, 0xc0001d1a40?, 0x9?)
    /data/playground/gorm/schema/relationship.go:506 +0x13b6
gorm.io/gorm/schema.(*Schema).parseRelation(0xc0001d0000, 0xc0001d1a40)
    /data/playground/gorm/schema/relationship.go:88 +0x3e5
gorm.io/gorm/schema.ParseWithSpecialTableName({0xa87ec0, 0xc000013320?}, 0xc0001a0420, {0xc22710?, 0xc00018cff0?}, {0x0, 0x0})
    /data/playground/gorm/schema/schema.go:321 +0x2416
gorm.io/gorm.(*Statement).ParseWithSpecialTableName(0xc0001c0380, {0xa87ec0?, 0xc000013320?}, {0x0?, 0x4167e8?})
    /data/playground/gorm/statement.go:492 +0x65
gorm.io/gorm.(*Statement).Parse(...)
    /data/playground/gorm/statement.go:488
gorm.io/gorm.(*processor).Execute(0xc00018e640, 0xc00018d080?)
    /data/playground/gorm/callbacks.go:105 +0x195
gorm.io/gorm.(*DB).Find(0xc00006af60?, {0xa87ec0?, 0xc000013320}, {0x0, 0x0, 0x0})
    /data/playground/gorm/finisher_api.go:172 +0x137
gorm.io/playground.TestGORM(0x0?)
    /data/playground/main_test.go:17 +0x9d
testing.tRunner(0xc000182d00, 0xb894c8)
    /usr/lib/go/src/testing/testing.go:1576 +0x10b
created by testing.(*T).Run
    /usr/lib/go/src/testing/testing.go:1629 +0x3ea

goroutine 1 [chan receive]:
testing.(*T).Run(0xc000182b60, {0xb4968e?, 0x5268c5?}, 0xb894c8)
    /usr/lib/go/src/testing/testing.go:1630 +0x405
testing.runTests.func1(0x1053620?)
    /usr/lib/go/src/testing/testing.go:2036 +0x45
testing.tRunner(0xc000182b60, 0xc0000e5c88)
    /usr/lib/go/src/testing/testing.go:1576 +0x10b
testing.runTests(0xc000197040?, {0xffcba0, 0x1, 0x1}, {0xc0000af758?, 0x100c0000e5d10?, 0x1052c00?})
    /usr/lib/go/src/testing/testing.go:2034 +0x489
testing.(*M).Run(0xc000197040)
    /usr/lib/go/src/testing/testing.go:1906 +0x63a
main.main()
    _testmain.go:47 +0x1aa

goroutine 6 [select]:
database/sql.(*DB).connectionOpener(0xc0000abee0, {0xc20bf0, 0xc00018e7d0})
    /usr/lib/go/src/database/sql/sql.go:1218 +0x8d
created by database/sql.OpenDB
    /usr/lib/go/src/database/sql/sql.go:791 +0x18d

goroutine 8 [runnable]:
gorm.io/gorm.(*Statement).AddClause(0xc000406000, {0xc207c0, 0xc0004022a0})
    /data/playground/gorm/statement.go:269 +0x71
gorm.io/gorm.(*Statement).AddClauseIfNotExists(0xc000406000, {0xc207c0, 0xc0004022a0})
    /data/playground/gorm/statement.go:279 +0x11c
gorm.io/gorm/callbacks.BuildQuerySQL(0xc000402000)
    /data/playground/gorm/callbacks/query.go:259 +0xbff
gorm.io/gorm/callbacks.Query(0xc000402000)
    /data/playground/gorm/callbacks/query.go:17 +0x45
gorm.io/gorm.(*processor).Execute(0xc00018e640, 0xc00018d080?)
    /data/playground/gorm/callbacks.go:130 +0x3ce
gorm.io/gorm.(*DB).Find(0x0?, {0xa87e80?, 0xc000400000}, {0x0, 0x0, 0x0})
    /data/playground/gorm/finisher_api.go:172 +0x137
gorm.io/playground.TestGORM.func1()
    /data/playground/main_test.go:14 +0x75
created by gorm.io/playground.TestGORM
    /data/playground/main_test.go:12 +0x25
exit status 2
FAIL    gorm.io/playground  0.010s
glebtv commented 1 year ago

1488 might have more info, but it's closed and seems to be about another place

glebtv commented 1 year ago

Different stack trace from another run (same code)

2023/05/05 19:59:26 testing sqlite3...
fatal error: concurrent map read and map write

goroutine 7 [running]:
gorm.io/gorm/callbacks.Preload(0xc00020d3e0)
    /data/playground/gorm/callbacks/query.go:297 +0x5fd
gorm.io/gorm.(*processor).Execute(0xc00020e640, 0xc00020d080?)
    /data/playground/gorm/callbacks.go:130 +0x3ce
gorm.io/gorm.(*DB).Find(0xc00006af60?, {0xa87ec0?, 0xc000013320}, {0x0, 0x0, 0x0})
    /data/playground/gorm/finisher_api.go:172 +0x137
gorm.io/playground.TestGORM(0x0?)
    /data/playground/main_test.go:17 +0x9d
testing.tRunner(0xc000202d00, 0xb894c8)
    /usr/lib/go/src/testing/testing.go:1576 +0x10b
created by testing.(*T).Run
    /usr/lib/go/src/testing/testing.go:1629 +0x3ea

goroutine 1 [chan receive]:
testing.(*T).Run(0xc000202b60, {0xb4968e?, 0x5268c5?}, 0xb894c8)
    /usr/lib/go/src/testing/testing.go:1630 +0x405
testing.runTests.func1(0x1053620?)
    /usr/lib/go/src/testing/testing.go:2036 +0x45
testing.tRunner(0xc000202b60, 0xc0000e5c88)
    /usr/lib/go/src/testing/testing.go:1576 +0x10b
testing.runTests(0xc000218fa0?, {0xffcba0, 0x1, 0x1}, {0xc0000af680?, 0x100c0000e5d10?, 0x1052c00?})
    /usr/lib/go/src/testing/testing.go:2034 +0x489
testing.(*M).Run(0xc000218fa0)
    /usr/lib/go/src/testing/testing.go:1906 +0x63a
main.main()
    _testmain.go:47 +0x1aa

goroutine 6 [select]:
database/sql.(*DB).connectionOpener(0xc0000abc70, {0xc20bf0, 0xc00020e7d0})
    /usr/lib/go/src/database/sql/sql.go:1218 +0x8d
created by database/sql.OpenDB
    /usr/lib/go/src/database/sql/sql.go:791 +0x18d

goroutine 8 [runnable]:
gorm.io/gorm.(*Statement).AddClause(0xc00042a000, {0xc207c0, 0xc000426390})
    /data/playground/gorm/statement.go:272 +0x14b
gorm.io/gorm.(*Statement).AddClauseIfNotExists(0xc00042a000, {0xc207c0, 0xc000426390})
    /data/playground/gorm/statement.go:279 +0x11c
gorm.io/gorm/callbacks.BuildQuerySQL(0xc000426000)
    /data/playground/gorm/callbacks/query.go:259 +0xbff
gorm.io/gorm/callbacks.Query(0xc000426000)
    /data/playground/gorm/callbacks/query.go:17 +0x45
gorm.io/gorm.(*processor).Execute(0xc00020e640, 0xc00020d080?)
    /data/playground/gorm/callbacks.go:130 +0x3ce
gorm.io/gorm.(*DB).Find(0x0?, {0xa87e80?, 0xc000424000}, {0x0, 0x0, 0x0})
    /data/playground/gorm/finisher_api.go:172 +0x137
gorm.io/playground.TestGORM.func1()
    /data/playground/main_test.go:14 +0x75
created by gorm.io/playground.TestGORM
    /data/playground/main_test.go:12 +0x25
exit status 2
FAIL    gorm.io/playground  0.010s
glebtv commented 1 year ago

Just in case, this happens with separate contexts too,

func TestGORM(t *testing.T) {
    go func() {
        cms := make([]Company, 0)
        DB.WithContext(context.Background()).Find(&cms)
    }()
    users := make([]User, 0)
    DB.WithContext(context.Background()).Preload("Toys").Find(&users)
}

also with a different stack:

fatal error: concurrent map read and map write

goroutine 7 [running]:
gorm.io/gorm/callbacks.Preload(0xc00020d470)
    /data/playground/gorm/callbacks/query.go:297 +0x5fd
gorm.io/gorm.(*processor).Execute(0xc00020e640, 0xc00020d080?)
    /data/playground/gorm/callbacks.go:130 +0x3ce
gorm.io/gorm.(*DB).Find(0xc00020d050?, {0xa87ec0?, 0xc000013320}, {0x0, 0x0, 0x0})
    /data/playground/gorm/finisher_api.go:172 +0x137
gorm.io/playground.TestGORM(0x0?)
    /data/playground/main_test.go:18 +0xd6
testing.tRunner(0xc000202d00, 0xb894c8)
    /usr/lib/go/src/testing/testing.go:1576 +0x10b
created by testing.(*T).Run
    /usr/lib/go/src/testing/testing.go:1629 +0x3ea

goroutine 1 [chan receive]:
testing.(*T).Run(0xc000202b60, {0xb4968e?, 0x5268c5?}, 0xb894c8)
    /usr/lib/go/src/testing/testing.go:1630 +0x405
testing.runTests.func1(0x1053620?)
    /usr/lib/go/src/testing/testing.go:2036 +0x45
testing.tRunner(0xc000202b60, 0xc0000e5c88)
    /usr/lib/go/src/testing/testing.go:1576 +0x10b
testing.runTests(0xc000218fa0?, {0xffcba0, 0x1, 0x1}, {0x0?, 0x100c000234668?, 0x1052c00?})
    /usr/lib/go/src/testing/testing.go:2034 +0x489
testing.(*M).Run(0xc000218fa0)
    /usr/lib/go/src/testing/testing.go:1906 +0x63a
main.main()
    _testmain.go:47 +0x1aa

goroutine 6 [select]:
database/sql.(*DB).connectionOpener(0xc00023a000, {0xc20c30, 0xc00020e7d0})
    /usr/lib/go/src/database/sql/sql.go:1218 +0x8d
created by database/sql.OpenDB
    /usr/lib/go/src/database/sql/sql.go:791 +0x18d

goroutine 8 [runnable]:
gorm.io/gorm/callbacks.BuildQuerySQL(0xc000428090)
    /data/playground/gorm/callbacks/query.go:102 +0xa3b
gorm.io/gorm/callbacks.Query(0xc000428090)
    /data/playground/gorm/callbacks/query.go:17 +0x45
gorm.io/gorm.(*processor).Execute(0xc00020e640, 0xc00020d080?)
    /data/playground/gorm/callbacks.go:130 +0x3ce
gorm.io/gorm.(*DB).Find(0xc00020d050?, {0xa87e80?, 0xc000424000}, {0x0, 0x0, 0x0})
    /data/playground/gorm/finisher_api.go:172 +0x137
gorm.io/playground.TestGORM.func1()
    /data/playground/main_test.go:15 +0xb1
created by gorm.io/playground.TestGORM
    /data/playground/main_test.go:13 +0x25
exit status 2
FAIL    gorm.io/playground  0.010s
glebtv commented 1 year ago

For anyone hitting this - forcing GORM to parse schemas before doing things in coroutines seems to resolve this

```go
func ForceParseModelSchema(value interface{}) error {
    stmt := &gorm.Statement{DB: db}
    return stmt.ParseWithSpecialTableName(value, stmt.Table)
}

ForceParseModelSchema(&Project{})
iTanken commented 1 year ago

@jinzhu I have encountered this issue as well:

fatal error: concurrent map read and map write

goroutine 389 [running]:
gorm.io/gorm.(*Statement).Build(0xc00332e1c0, {0x5d39400?, 0x7, 0x4?})
        /vendor/gorm.io/gorm/statement.go:472 +0xe9
gorm.io/gorm/callbacks.BuildQuerySQL(0xc001e6f710)
        /vendor/gorm.io/gorm/callbacks/query.go:264 +0xc51
gorm.io/gorm/callbacks.Query(0xc001e6f710)
        /vendor/gorm.io/gorm/callbacks/query.go:17 +0x45
gorm.io/gorm.(*processor).Execute(0xc000b4c6e0, 0xc000d76420?)
        /vendor/gorm.io/gorm/callbacks.go:130 +0x3ce
gorm.io/gorm.(*DB).Find(0x0?, {0x19655c0?, 0xc002c22a08}, {0x0, 0x0, 0x0})
        /vendor/gorm.io/gorm/finisher_api.go:170 +0x137
LocalSrv/web/services.(*BizClipboardService).GetList(0xc001e5d8c0)
        /web/services/biz_clipboard_service.go:95 +0x598
LocalSrv/web/controllers.BizClipboardController.GetList({0xc002c229f0?}, 0xc000860d00)
        /web/controllers/biz_clipboard_controller.go:106 +0x1f5
github.com/gin-gonic/gin.(*Context).Next(...)
        /vendor/github.com/gin-gonic/gin/context.go:174
github.com/gin-gonic/gin.(*Engine).handleHTTPRequest(0xc001cfc4e0, 0xc000860d00)
        /vendor/github.com/gin-gonic/gin/gin.go:620 +0x66b
github.com/gin-gonic/gin.(*Engine).ServeHTTP(0xc001cfc4e0, {0x4093880?, 0xc0030857a0}, 0xc002a8f400)
        /vendor/github.com/gin-gonic/gin/gin.go:576 +0x1dd
net/http.serverHandler.ServeHTTP({0x408e860?}, {0x4093880, 0xc0030857a0}, 0xc002a8f400)
        /go/src/net/http/server.go:2936 +0x316
net/http.(*conn).serve(0xc002a8bd40, {0x4094f78, 0xc00075b1a0})
        /go/src/net/http/server.go:1995 +0x612
created by net/http.(*Server).Serve
        /go/src/net/http/server.go:3089 +0x5ed
Fatal Error Code ```go package services // ... type BizClipboardService struct { ctx *gin.Context pageNo int pageSize int result *gorm.DB } // ... func (s *BizClipboardService) GetList() (data []model.BizClipboardRecord, err error) { s.result = database.DBConn.GormPaginate(s.pageNo, s.pageSize) if s.pageSize == 0 { s.result.Limit(10) // default limit } target := model.BizClipboardRecord{ClientIP: s.ctx.ClientIP()} s.result.Where(&target) keyword := fmt.Sprintf("%%%s%%", s.ctx.GetString("keyword")) s.result.Where("content like ?", keyword).Or("remark like ?", keyword) s.result.Order("id DESC").Find(&data) // line #95 : fatal error: concurrent map read and map write err = s.result.Error return } // ... ``` ```go package database // ... type Connection struct { SQLConn *sql.DB GormConn *gorm.DB } var DBConn = Connection{} func init() { var err error // ... if DBConn.GormConn, err = gorm.Open(dialector, gormConfig); err == nil { // ... } } func (db *Connection) GormPaginate(pageNo, pageSize int) (DB *gorm.DB) { var offset int if pageNo > 0 && pageSize > 0 { offset = (pageNo - 1) * pageSize DB = db.GormConn.Limit(pageSize).Offset(offset) } else { DB = db.GormConn.Limit(-1).Offset(-1) } return } // ... ```
Yenuo commented 5 months ago

这个问题到现在还没有处理掉？？？

sjc5 commented 1 month ago

Just hit this as well.

Beginning of stack trace:

fatal error: concurrent map read and map write

goroutine 95 [running]:
gorm.io/gorm/callbacks.BuildQuerySQL(0x1400017f830)
    /Users/sjc/go/pkg/mod/gorm.io/gorm@v1.25.11/callbacks/query.go:131 +0xc00
gorm.io/gorm/callbacks.Query(0x1400017f830)
    /Users/sjc/go/pkg/mod/gorm.io/gorm@v1.25.11/callbacks/query.go:16 +0x38
gorm.io/gorm.(*processor).Execute(0x1400020e2d0, 0x140004e46f0?)
    /Users/sjc/go/pkg/mod/gorm.io/gorm@v1.25.11/callbacks.go:130 +0x34c
gorm.io/gorm.(*DB).Find(0x100f9da30?, {0x10247ed00, 0x140003de438}, {0x0, 0x0, 0x0})
    /Users/sjc/go/pkg/mod/gorm.io/gorm@v1.25.11/finisher_api.go:170 +0x114
[...]

Userland code:

// db is of type *gorm.DB
db.Find(&rows) // <-- error happens here