search

marcboeker / go-duckdb

go-duckdb provides a database/sql driver for the DuckDB database engine.
MIT License
579 stars 90 forks source link

Uncaught exception of type duckdb::InternalException #186

Open alex-ilyin opened 3 months ago

alex-ilyin commented 3 months ago

I'm getting this exception from time to time when playing with DuckDB's memory limit and setting it so that my workload doesn't fit in. Looks like the problem is coming from the Go client and not from DuckDB itself. Here is the full error message (attaching stack trace below):

libc++abi: terminating due to uncaught exception of type duckdb::InternalException: {"exception_type":"INTERNAL","exception_message":"Attempted to access index 40 within vector of size 40"} SIGABRT: abort PC=0x7ff80f32c7a2 m=0 sigcode=0 signal arrived during cgo execution

go-duckdb version used: 1.6.1

Here is the code I was executing. It doesn't return the error every time, usually it ends with an Out of Memory Error, which is expected

package main

import (
    "database/sql"
    "errors"
    "fmt"
    _ "github.com/marcboeker/go-duckdb"
    "log"
    "time"
)

func testDuck2() {
    fmt.Println()

    db, err := sql.Open("duckdb", "~/test-duck.db")
    if err != nil {
        log.Fatal(err)
    }
    defer db.Close()

    _, err = db.Exec(`SET memory_limit = '150MB';
        CREATE OR REPLACE VIEW metrics AS
        SELECT date, SUM(failure) as failures
        FROM read_csv('~/datasets/backblaze_data_Q4_2023.csv')
        GROUP BY date;`)
    if err != nil {
        log.Fatal(err)
    }

    start := time.Now()
    _, err = db.Exec("COPY metrics TO '~/results.csv'")
    if err != nil {
        log.Fatal(err)
    }

    fmt.Println("Done in", time.Since(start))
}

func main() {
    testDuck2()
}

I used this dataset for my tests (Q4 2023): https://www.backblaze.com/cloud-storage/resources/hard-drive-test-data#downloadingTheRawTestData The only thing I did with it is merging all csvs into one using DuckDB CLI like that:

create table data as from 'backblaze_data_Q4_2023/*.csv'; copy data to 'backblaze_data_Q4_2023.csv';

Here is the full stack trace I got:

libc++abi: terminating due to uncaught exception of type duckdb::InternalException: {"exception_type":"INTERNAL","exception_message":"Attempted to access index 40 within vector of size 40"}
SIGABRT: abort
PC=0x7ff80f32c7a2 m=0 sigcode=0
signal arrived during cgo execution

goroutine 1 [syscall]:
runtime.cgocall(0x1001e1e10, 0xc0000566c8)
        /usr/local/opt/go/libexec/src/runtime/cgocall.go:157 +0x4b fp=0xc0000566a0 sp=0xc000056668 pc=0x10000626b
github.com/marcboeker/go-duckdb._Cfunc_duckdb_execute_pending(0x600003cdc1b0, 0xc000092150)
        _cgo_gotypes.go:1114 +0x4b fp=0xc0000566c8 sp=0xc0000566a0 pc=0x1001dadab
github.com/marcboeker/go-duckdb.(*stmt).execute.func6(0xc000028500?, 0x0?)
        /Users/alexandre.ilin/go/pkg/mod/github.com/marcboeker/go-duckdb@v1.6.1/statement.go:223 +0x65 fp=0xc000056708 sp=0xc0000566c8 pc=0x1001dffa5
github.com/marcboeker/go-duckdb.(*stmt).execute(0xc00009a030, {0x1019632f8, 0x101d5b900}, {0x101d5b900?, 0xc000056830?, 0x1001dc845?})
        /Users/alexandre.ilin/go/pkg/mod/github.com/marcboeker/go-duckdb@v1.6.1/statement.go:223 +0x277 fp=0xc0000567b8 sp=0xc000056708 pc=0x1001dfb77
github.com/marcboeker/go-duckdb.(*stmt).ExecContext(0xc00008e000?, {0x1019632f8?, 0x101d5b900?}, {0x101d5b900?, 0x0?, 0x101d2b5a0?})
        /Users/alexandre.ilin/go/pkg/mod/github.com/marcboeker/go-duckdb@v1.6.1/statement.go:162 +0x47 fp=0xc000056840 sp=0xc0000567b8 pc=0x1001df6e7
github.com/marcboeker/go-duckdb.(*conn).ExecContext(0xc00008e000, {0x1019632f8, 0x101d5b900}, {0x1019177a6, 0x33}, {0x101d5b900, 0x0, 0x0})
        /Users/alexandre.ilin/go/pkg/mod/github.com/marcboeker/go-duckdb@v1.6.1/connection.go:63 +0x33d fp=0xc000056910 sp=0xc000056840 pc=0x1001dbc1d
database/sql.ctxDriverExec({0x1019632f8?, 0x101d5b900?}, {0x101962440?, 0xc00008e000?}, {0x0?, 0x0?}, {0x1019177a6?, 0x1018cf1c0?}, {0x101d5b900, 0x0, ...})
        /usr/local/opt/go/libexec/src/database/sql/ctxutil.go:31 +0xd7 fp=0xc000056998 sp=0xc000056910 pc=0x1000b0cb7
database/sql.(*DB).execDC.func2()
        /usr/local/opt/go/libexec/src/database/sql/sql.go:1675 +0x165 fp=0xc000056a58 sp=0xc000056998 pc=0x1000b5965
database/sql.withLock({0x101962950, 0xc000090000}, 0xc000056c10)
        /usr/local/opt/go/libexec/src/database/sql/sql.go:3502 +0x82 fp=0xc000056a98 sp=0xc000056a58 pc=0x1000b5de2
database/sql.(*DB).execDC(0x1?, {0x1019632f8?, 0x101d5b900}, 0xc000090000, 0x30?, {0x1019177a6, 0x33}, {0x0, 0x0, 0x0})
        /usr/local/opt/go/libexec/src/database/sql/sql.go:1670 +0x24f fp=0xc000056cb0 sp=0xc000056a98 pc=0x1000b52cf
database/sql.(*DB).exec(0x102a12878?, {0x1019632f8, 0x101d5b900}, {0x1019177a6, 0x33}, {0x0, 0x0, 0x0}, 0x60?)
        /usr/local/opt/go/libexec/src/database/sql/sql.go:1655 +0xdb fp=0xc000056d28 sp=0xc000056cb0 pc=0x1000b4ffb
database/sql.(*DB).ExecContext.func1(0xf8?)
        /usr/local/opt/go/libexec/src/database/sql/sql.go:1634 +0x4f fp=0xc000056d90 sp=0xc000056d28 pc=0x1000b4eaf
database/sql.(*DB).retry(0xc000056e10?, 0xc000056e08)
        /usr/local/opt/go/libexec/src/database/sql/sql.go:1538 +0x42 fp=0xc000056dd8 sp=0xc000056d90 pc=0x1000b4c82
database/sql.(*DB).ExecContext(0x10190c013?, {0x1019632f8?, 0x101d5b900?}, {0x1019177a6?, 0x0?}, {0x0?, 0xc0000466d8?, 0x10003fe3c?})
        /usr/local/opt/go/libexec/src/database/sql/sql.go:1633 +0xc8 fp=0xc000056e70 sp=0xc000056dd8 pc=0x1000b4dc8
database/sql.(*DB).Exec(...)
        /usr/local/opt/go/libexec/src/database/sql/sql.go:1647
main.testDuck2()
        /Users/alexandre.ilin/GolandProjects/duckdb-playground/main.go:31 +0x167 fp=0xc000056f30 sp=0xc000056e70 pc=0x1001e0847
main.main()
        /Users/alexandre.ilin/GolandProjects/duckdb-playground/main.go:40 +0xf fp=0xc000056f40 sp=0xc000056f30 pc=0x1001e09af
runtime.main()
        /usr/local/opt/go/libexec/src/runtime/proc.go:267 +0x2bb fp=0xc000056fe0 sp=0xc000056f40 pc=0x10003855b
runtime.goexit()
        /usr/local/opt/go/libexec/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000056fe8 sp=0xc000056fe0 pc=0x100066701

goroutine 2 [force gc (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
        /usr/local/opt/go/libexec/src/runtime/proc.go:398 +0xce fp=0xc000046fa8 sp=0xc000046f88 pc=0x1000389ae
runtime.goparkunlock(...)
        /usr/local/opt/go/libexec/src/runtime/proc.go:404
runtime.forcegchelper()
        /usr/local/opt/go/libexec/src/runtime/proc.go:322 +0xb3 fp=0xc000046fe0 sp=0xc000046fa8 pc=0x100038833
runtime.goexit()
        /usr/local/opt/go/libexec/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000046fe8 sp=0xc000046fe0 pc=0x100066701
created by runtime.init.6 in goroutine 1
        /usr/local/opt/go/libexec/src/runtime/proc.go:310 +0x1a

goroutine 3 [GC sweep wait]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
        /usr/local/opt/go/libexec/src/runtime/proc.go:398 +0xce fp=0xc000047778 sp=0xc000047758 pc=0x1000389ae
runtime.goparkunlock(...)
        /usr/local/opt/go/libexec/src/runtime/proc.go:404
runtime.bgsweep(0x0?)
        /usr/local/opt/go/libexec/src/runtime/mgcsweep.go:280 +0x94 fp=0xc0000477c8 sp=0xc000047778 pc=0x100025b74
runtime.gcenable.func1()
        /usr/local/opt/go/libexec/src/runtime/mgc.go:200 +0x25 fp=0xc0000477e0 sp=0xc0000477c8 pc=0x10001af05
runtime.goexit()
        /usr/local/opt/go/libexec/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0000477e8 sp=0xc0000477e0 pc=0x100066701
created by runtime.gcenable in goroutine 1
        /usr/local/opt/go/libexec/src/runtime/mgc.go:200 +0x66

goroutine 4 [GC scavenge wait]:
runtime.gopark(0xc000070000?, 0x101960640?, 0x1?, 0x0?, 0xc000007380?)
        /usr/local/opt/go/libexec/src/runtime/proc.go:398 +0xce fp=0xc000047f70 sp=0xc000047f50 pc=0x1000389ae
runtime.goparkunlock(...)
        /usr/local/opt/go/libexec/src/runtime/proc.go:404
runtime.(*scavengerState).park(0x101d2ad20)
        /usr/local/opt/go/libexec/src/runtime/mgcscavenge.go:425 +0x49 fp=0xc000047fa0 sp=0xc000047f70 pc=0x100023429
runtime.bgscavenge(0x0?)
        /usr/local/opt/go/libexec/src/runtime/mgcscavenge.go:653 +0x3c fp=0xc000047fc8 sp=0xc000047fa0 pc=0x1000239bc
runtime.gcenable.func2()
        /usr/local/opt/go/libexec/src/runtime/mgc.go:201 +0x25 fp=0xc000047fe0 sp=0xc000047fc8 pc=0x10001aea5
runtime.goexit()
        /usr/local/opt/go/libexec/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000047fe8 sp=0xc000047fe0 pc=0x100066701
created by runtime.gcenable in goroutine 1
        /usr/local/opt/go/libexec/src/runtime/mgc.go:201 +0xa5

goroutine 18 [finalizer wait]:
runtime.gopark(0x198?, 0x10190a340?, 0x1?, 0x9b?, 0x0?)
        /usr/local/opt/go/libexec/src/runtime/proc.go:398 +0xce fp=0xc000046620 sp=0xc000046600 pc=0x1000389ae
runtime.runfinq()
        /usr/local/opt/go/libexec/src/runtime/mfinal.go:193 +0x107 fp=0xc0000467e0 sp=0xc000046620 pc=0x100019f27
runtime.goexit()
        /usr/local/opt/go/libexec/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0000467e8 sp=0xc0000467e0 pc=0x100066701
created by runtime.createfing in goroutine 1
        /usr/local/opt/go/libexec/src/runtime/mfinal.go:163 +0x3d

goroutine 19 [select]:
runtime.gopark(0xc000042788?, 0x2?, 0x0?, 0x0?, 0xc000042784?)
        /usr/local/opt/go/libexec/src/runtime/proc.go:398 +0xce fp=0xc000042630 sp=0xc000042610 pc=0x1000389ae
runtime.selectgo(0xc000042788, 0xc000042780, 0x0?, 0x0, 0x0?, 0x1)
        /usr/local/opt/go/libexec/src/runtime/select.go:327 +0x725 fp=0xc000042750 sp=0xc000042630 pc=0x100048325
database/sql.(*DB).connectionOpener(0xc00011c8f0, {0x1019633d8, 0xc000152050})
        /usr/local/opt/go/libexec/src/database/sql/sql.go:1218 +0x87 fp=0xc0000427b8 sp=0xc000042750 pc=0x1000b3527
database/sql.OpenDB.func1()
        /usr/local/opt/go/libexec/src/database/sql/sql.go:791 +0x28 fp=0xc0000427e0 sp=0xc0000427b8 pc=0x1000b24c8
runtime.goexit()
        /usr/local/opt/go/libexec/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0000427e8 sp=0xc0000427e0 pc=0x100066701
created by database/sql.OpenDB in goroutine 1
        /usr/local/opt/go/libexec/src/database/sql/sql.go:791 +0x165

goroutine 36 [select]:
runtime.gopark(0xc0000a27b0?, 0x2?, 0x0?, 0x0?, 0xc0000a2794?)
        /usr/local/opt/go/libexec/src/runtime/proc.go:398 +0xce fp=0xc0000a2640 sp=0xc0000a2620 pc=0x1000389ae
runtime.selectgo(0xc0000a27b0, 0xc0000a2790, 0x0?, 0x0, 0x0?, 0x1)
        /usr/local/opt/go/libexec/src/runtime/select.go:327 +0x725 fp=0xc0000a2760 sp=0xc0000a2640 pc=0x100048325
github.com/marcboeker/go-duckdb.(*stmt).execute.func5()
        /Users/alexandre.ilin/go/pkg/mod/github.com/marcboeker/go-duckdb@v1.6.1/statement.go:211 +0x74 fp=0xc0000a27e0 sp=0xc0000a2760 pc=0x1001e0054
runtime.goexit()
        /usr/local/opt/go/libexec/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0000a27e8 sp=0xc0000a27e0 pc=0x100066701
created by github.com/marcboeker/go-duckdb.(*stmt).execute in goroutine 1
        /Users/alexandre.ilin/go/pkg/mod/github.com/marcboeker/go-duckdb@v1.6.1/statement.go:210 +0x254

rax    0x0
rbx    0x6
rcx    0x7ff7bfefed68
rdx    0x0
rdi    0x103
rsi    0x6
rbp    0x7ff7bfefed90
rsp    0x7ff7bfefed68
r8     0x7ff7bfefec30
r9     0xaec3fffb
r10    0x0
r11    0x246
r12    0x103
r13    0x3000000008
r14    0x7ff85298c7c0
r15    0x16
rip    0x7ff80f32c7a2
rflags 0x246
cs     0x7
fs     0x0
gs     0x0

Process finished with the exit code 2
JAicewizard commented 3 weeks ago

This looks like a problem within duckdb, they should return an error instead of letting the exception seep through the C api. Would you mind opening an issue there? I think posting the go code there should be good enough, but reproducing this in the REPL would be ideal for them I think.