search

boltdb / bolt

An embedded key/value database for Go.
MIT License
14.22k stars 1.51k forks source link

page already freed #731

Open deepch opened 7 years ago

deepch commented 7 years ago

curent version git i can upload db 

Linux sr7-ipeye 4.4.0-96-generic #119-Ubuntu SMP Tue Sep 12 14:59:54 UTC 2017 x86_64 x86_64 x86_64 GNU/Linux

Description: Ubuntu 16.04.3 LTS

func (this *Database) Clean() {
    a := []string{}
    if this.db != nil {
        this.db.Update(func(tx *bolt.Tx) error {
            start := time.Now().AddDate(0, 0, -7)
            seek := []byte(start.Format(time.RFC3339))
            c := tx.Cursor()
            for k, _ := c.Seek(seek); k != nil; k, _ = c.Next() {
                a = append(a, string(k))
            }
            return nil
        })
        if len(a) > 0 {
            for _, v := range a {
                this.db.Update(func(tx *bolt.Tx) error {
                    start := time.Now().AddDate(0, 0, -7)
                    seek := []byte(start.Format(time.RFC3339))
                    bkt := tx.Bucket([]byte(v))
                    c := bkt.Cursor()
                    for k, _ := c.First(); k != nil && bytes.Compare(k, seek) <= 0; k, _ = c.Next() {
                        c.Delete()
                    }
                    return nil
                })
            }
        }
    }
}
panic: page 1890706 already freed

goroutine 120 [running]:
github.com/boltdb/bolt.(*freelist).free(0xc420459800, 0x187c83, 0x7f01a1992000)
        /home/deepweb/test/src/github.com/boltdb/bolt/freelist.go:121 +0x301
github.com/boltdb/bolt.(*node).spill(0xc819003960, 0xc4c4119960, 0x11342e0)
        /home/deepweb/test/src/github.com/boltdb/bolt/node.go:363 +0x210
github.com/boltdb/bolt.(*node).spill(0xc7be1c0150, 0xc4c4119940, 0x11342e0)
        /home/deepweb/test/src/github.com/boltdb/bolt/node.go:350 +0xbf
github.com/boltdb/bolt.(*node).spill(0xc7be1c00e0, 0xc430fc1500, 0xc6c31b9aa8)
        /home/deepweb/test/src/github.com/boltdb/bolt/node.go:350 +0xbf
github.com/boltdb/bolt.(*Bucket).spill(0xc55f250780, 0xc430fc1400, 0xc6c31b9d20)
        /home/deepweb/test/src/github.com/boltdb/bolt/bucket.go:570 +0x4d3
github.com/boltdb/bolt.(*Bucket).spill(0xc8795ba2b8, 0x20fec8a82ccae, 0x1166ce0)
        /home/deepweb/test/src/github.com/boltdb/bolt/bucket.go:537 +0x417
github.com/boltdb/bolt.(*Tx).Commit(0xc8795ba2a0, 0x0, 0x0)
        /home/deepweb/test/src/github.com/boltdb/bolt/tx.go:163 +0x129
github.com/boltdb/bolt.(*DB).Update(0xc420210000, 0xc6c31b9f80, 0x0, 0x0)
        /home/deepweb/test/src/github.com/boltdb/bolt/db.go:605 +0xf2
main.(*Database).Clean(0xc4203ee1e0)
        /home/deepweb/test/src/github.com/deepch/dd/ModuleDatabase.go:410 +0xf2
main.event_clener()
        /home/deepweb/test/src/github.com/deepch/dd/ModuleDatabase.go:499 +0x2d
created by main.event_init
        /home/deepweb/test/src/github.com/deepch/dd/ModuleDatabase.go:494 +0x99
djadala commented 6 years ago

Same here:

panic: page 162597 already freed

goroutine 17 [running]:
github.com/boltdb/bolt.(*freelist).free(0xc420078330, 0x7b, 0x7efe5cb57000)
    /home/jambo/golang/src/github.com/boltdb/bolt/freelist.go:121 +0x301
github.com/boltdb/bolt.(*node).spill(0xc42381a3f0, 0x0, 0x0)
    /home/jambo/golang/src/github.com/boltdb/bolt/node.go:363 +0x210
github.com/boltdb/bolt.(*node).spill(0xc423d73030, 0x0, 0x0)
    /home/jambo/golang/src/github.com/boltdb/bolt/node.go:350 +0xbf
github.com/boltdb/bolt.(*node).spill(0xc423d72fc0, 0xc426b1a0c0, 0xc420046a80)
    /home/jambo/golang/src/github.com/boltdb/bolt/node.go:350 +0xbf
github.com/boltdb/bolt.(*Bucket).spill(0xc425316100, 0xc426b1a000, 0xc420046cf8)
    /home/jambo/golang/src/github.com/boltdb/bolt/bucket.go:570 +0x4d3
github.com/boltdb/bolt.(*Bucket).spill(0xc422ad40f8, 0x1f3dd3f0d0, 0x59e980)
    /home/jambo/golang/src/github.com/boltdb/bolt/bucket.go:537 +0x417
github.com/boltdb/bolt.(*Tx).Commit(0xc422ad40e0, 0x0, 0x0)
    /home/jambo/golang/src/github.com/boltdb/bolt/tx.go:163 +0x129
github.com/boltdb/bolt.(*DB).Update(0xc422a10000, 0xc42006dfa0, 0x0, 0x0)
    /home/jambo/golang/src/github.com/boltdb/bolt/db.go:605 +0xf2
main.Add(0xc420072060, 0x5087ae, 0x13)

I insert 40Gib bytes to new db file, average slice len=30 bytes,max=30000:

func Add(c chan []byte, bfile string) {
    const bname = `bc41`

    db, err := bolt.Open(bfile, 0600, nil)
    if err != nil {
        log.Fatal(err)
    }
    defer db.Close()

    err = db.Update(func(tx *bolt.Tx) error {
        _, err := tx.CreateBucketIfNotExists([]byte(bname))
        if err != nil {
            return err //fmt.Errorf("create bucket: %s", err)
        }
        return nil
    })
    if err != nil {
        log.Fatal(err)
        panic(err)
    }

    for {
        err := db.Update(func(tx *bolt.Tx) error {
            b := tx.Bucket([]byte(bname))
            for i := 0; i < 100000; i++ {
                v := <-c
                if v == nil {
                    return nil
                }
                err = b.Put(v, []byte("0"))
                if err != nil {
                    return err
                }
            }
            return nil
        })
        if err != nil {
            log.Fatal(err)
            panic(err)
        }
    }
}
djadala commented 6 years ago

reproducer, crash when file is 28 MiB:

package main

import (
    "log"
    "math/rand"

    // "github.com/coreos/bbolt"
    "github.com/boltdb/bolt"
)

var BC = make(chan []byte)

func main() {

    go Add(BC, "/opt/test.db")

    s := make([]byte, 30)
    for i := 0; i < 100000000; i++ {
        rand.Read(s)
        BC <- s
    }

    close(BC)

}

func Add(c chan []byte, bfile string) {
    const bname = `bc41`

    db, err := bolt.Open(bfile, 0600, nil)
    if err != nil {
        log.Fatal(err)
    }
    defer db.Close()

    err = db.Update(func(tx *bolt.Tx) error {
        _, err := tx.CreateBucketIfNotExists([]byte(bname))
        if err != nil {
            return err //fmt.Errorf("create bucket: %s", err)
        }
        return nil
    })
    if err != nil {
        log.Fatal(err)
        panic(err)
    }

    for {
        err := db.Update(func(tx *bolt.Tx) error {
            b := tx.Bucket([]byte(bname))
            for i := 0; i < 100000; i++ {
                v := <-c
                if v == nil {
                    return nil
                }
                err = b.Put(v, []byte("0"))
                if err != nil {
                    return err
                }
            }
            return nil
        })
        if err != nil {
            log.Fatal(err)
            panic(err)
        }
    }
}
panic: page 420 already freed

goroutine 5 [running]:
github.com/boltdb/bolt.(*freelist).free(0xc4200741e0, 0x7, 0x7ff7e4a19000)
    /home/jambo/golang/src/github.com/boltdb/bolt/freelist.go:121 +0x301
github.com/boltdb/bolt.(*node).spill(0xc422ca07e0, 0x0, 0x0)
    /home/jambo/golang/src/github.com/boltdb/bolt/node.go:363 +0x210
github.com/boltdb/bolt.(*node).spill(0xc4202b7650, 0xc4201407e0, 0x55d920)
    /home/jambo/golang/src/github.com/boltdb/bolt/node.go:350 +0xbf
github.com/boltdb/bolt.(*node).spill(0xc420e503f0, 0xc4222d20c0, 0xc420049a80)
    /home/jambo/golang/src/github.com/boltdb/bolt/node.go:350 +0xbf
github.com/boltdb/bolt.(*Bucket).spill(0xc4200523c0, 0xc4222d2000, 0xc420049cf8)
    /home/jambo/golang/src/github.com/boltdb/bolt/bucket.go:570 +0x4d3
github.com/boltdb/bolt.(*Bucket).spill(0xc4200900f8, 0x20140c719, 0x570160)
    /home/jambo/golang/src/github.com/boltdb/bolt/bucket.go:537 +0x417
github.com/boltdb/bolt.(*Tx).Commit(0xc4200900e0, 0x0, 0x0)
    /home/jambo/golang/src/github.com/boltdb/bolt/tx.go:163 +0x129
github.com/boltdb/bolt.(*DB).Update(0xc420088000, 0xc42006dfa0, 0x0, 0x0)
    /home/jambo/golang/src/github.com/boltdb/bolt/db.go:605 +0xf2
main.Add(0xc420072060, 0x4e88fb, 0x12)
    /home/jambo/go/src/jambo/tests/error1/error1.go:49 +0x1d4
created by main.main
    /home/jambo/go/src/jambo/tests/error1/error1.go:15 +0x5a
djadala commented 6 years ago 
go version 
go version go1.9.1 linux/amd64

uname -a
Linux bee 4.9.0-0.bpo.4-amd64 #1 SMP Debian 4.9.51-1~bpo8+1 (2017-10-17) x86_64 GNU/Linux
djadala commented 6 years ago

Opened issue in coreos/bbolt: https://github.com/coreos/bbolt/issues/72

djadala commented 6 years ago

There is race in my example, modified example without race works fine. @deepch , can you run your program under race detector ?