h2non / bimg

Go package for fast high-level image processing powered by libvips C library
https://pkg.go.dev/github.com/h2non/bimg?tab=doc
MIT License
2.72k stars 339 forks source link

runtime: program exceeds 10000-thread limit fatal error: thread exhaustion #445

Open yourchanges opened 1 year ago

yourchanges commented 1 year ago

ENV: bimg 1.1.6 libvips 8.10.6 on centos7

when run in our prod env, the server cashed by "runtime: program exceeds 10000-thread limit fatal error: thread exhaustion"

here are 9927 cgo-goroutine (block cgo goroutine === os-thread)+ default owned os-thread, so our process crashed

here is the cgo-goroutine's detail info

goroutine 2036897 [syscall, 4 minutes]:
runtime.cgocall(0x15ac6c0, 0xc71b719e98)
    /usr/local/src/go/src/runtime/cgocall.go:158 +0x5c fp=0xc71b719e70 sp=0xc71b719e38 pc=0x411c3c
github.com/h2non/bimg._Cfunc_vips_jpegsave_bridge(0x7f5d9ae014d0, 0xc68207acd8, 0xc24372abf0, 0x0, 0x4b, 0x0)
    _cgo_gotypes.go:1020 +0x4c fp=0xc71b719e98 sp=0xc71b719e70 pc=0xa44c6c
github.com/h2non/bimg.vipsSave.func8(0x17eb200?, 0xc0004066c0?, 0x1?, 0x0?, 0x0?, 0x0?)
    /root/go/pkg/mod/github.com/h2non/bimg@v1.1.6/vips.go:528 +0xa5 fp=0xc71b719f00 sp=0xc71b719e98 pc=0xa4b945
github.com/h2non/bimg.vipsSave(0x7f5d98326640, {0x0, 0x4b, 0x6, 0x1, 0x0, 0x0, 0x0, 0x0, {0x0, ...}, ...})
    /root/go/pkg/mod/github.com/h2non/bimg@v1.1.6/vips.go:528 +0x36f fp=0xc71b719ff0 sp=0xc71b719f00 pc=0xa4b4cf
github.com/h2non/bimg.saveImage(_, {0x21f, 0x21f, 0x0, 0x0, 0x0, 0x0, 0x4b, 0x6, 0x0, ...})
    /root/go/pkg/mod/github.com/h2non/bimg@v1.1.6/resizer.go:193 +0x158 fp=0xc71b71a0b8 sp=0xc71b719ff0 pc=0xa46f58
github.com/h2non/bimg.resizer({_, _, _}, {0x21f, 0x21f, 0x0, 0x0, 0x0, 0x0, 0x4b, ...})
    /root/go/pkg/mod/github.com/h2non/bimg@v1.1.6/resizer.go:145 +0xaf8 fp=0xc71b71a948 sp=0xc71b71a0b8 pc=0xa46938
github.com/h2non/bimg.Resize({_, _, _}, {0x0, 0x21f, 0x0, 0x0, 0x0, 0x0, 0x0, ...})
    /root/go/pkg/mod/github.com/h2non/bimg@v1.1.6/resize.go:15 +0xfc fp=0xc71b71ab68 sp=0xc71b71a948 pc=0xa4271c
github.com/h2non/bimg.(*Image).Process(_, {0x0, 0x21f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...})
    /root/go/pkg/mod/github.com/h2non/bimg@v1.1.6/image.go:203 +0x65 fp=0xc71b71ad20 sp=0xc71b71ab68 pc=0xa425a5
github.com/h2non/bimg.(*Image).ForceResize(0x1dfd1e0?, 0xc0af6f5020?, 0x41b6bf?)
    /root/go/pkg/mod/github.com/h2non/bimg@v1.1.6/image.go:30 +0x98 fp=0xc71b71b058 sp=0xc71b71ad20 pc=0xa424f8

goroutine 2036340 [syscall, 2 minutes]:
runtime.cgocall(0x15ac680, 0xc671d59fa0)
    /usr/local/src/go/src/runtime/cgocall.go:158 +0x5c fp=0xc671d59f78 sp=0xc671d59f40 pc=0x411c3c
github.com/h2non/bimg._Cfunc_vips_jpegload_buffer_shrink(0xc6d3c0a000, 0x38106, 0xc9b90aa240, 0x2)
    _cgo_gotypes.go:1004 +0x4c fp=0xc671d59fa0 sp=0xc671d59f78 pc=0xa44b4c
github.com/h2non/bimg.vipsShrinkJpeg.func2(0x7f5fc0827980?, {0xc6d3c0a000?, 0x38106, 0xc671d5a018?}, 0xc671d5a040?, 0x2)
    /root/go/pkg/mod/github.com/h2non/bimg@v1.1.6/vips.go:620 +0x90 fp=0xc671d59fe8 sp=0xc671d59fa0 pc=0xa4d490
github.com/h2non/bimg.vipsShrinkJpeg({0xc6d3c0a000, 0x38106, 0x44000}, 0xc671d5a0a8?, 0xa4835f?)
    /root/go/pkg/mod/github.com/h2non/bimg@v1.1.6/vips.go:620 +0xba fp=0xc671d5a068 sp=0xc671d59fe8 pc=0xa4d2fa
github.com/h2non/bimg.shrinkOnLoad({0xc6d3c0a000?, 0xf0?, 0x0?}, 0x0?, 0x0?, 0x0?, 0x4b?)
    /root/go/pkg/mod/github.com/h2non/bimg@v1.1.6/resizer.go:478 +0x125 fp=0xc671d5a0b8 sp=0xc671d5a068 pc=0xa47da5
github.com/h2non/bimg.resizer({_, _, _}, {0xf0, 0xf0, 0x0, 0x0, 0x0, 0x0, 0x4b, ...})
    /root/go/pkg/mod/github.com/h2non/bimg@v1.1.6/resizer.go:88 +0x575 fp=0xc671d5a948 sp=0xc671d5a0b8 pc=0xa463b5
github.com/h2non/bimg.Resize({_, _, _}, {0x0, 0xf0, 0x0, 0x0, 0x0, 0x0, 0x0, ...})
    /root/go/pkg/mod/github.com/h2non/bimg@v1.1.6/resize.go:15 +0xfc fp=0xc671d5ab68 sp=0xc671d5a948 pc=0xa4271c
github.com/h2non/bimg.(*Image).Process(_, {0x0, 0xf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...})
    /root/go/pkg/mod/github.com/h2non/bimg@v1.1.6/image.go:203 +0x65 fp=0xc671d5ad20 sp=0xc671d5ab68 pc=0xa425a5
github.com/h2non/bimg.(*Image).ForceResize(0x1dfd1e0?, 0xc3fbf83980?, 0x41b6bf?)
    /root/go/pkg/mod/github.com/h2non/bimg@v1.1.6/image.go:30 +0x98 fp=0xc671d5b058 sp=0xc671d5ad20 pc=0xa424f8

goroutine 2035896 [syscall, 6 minutes]:
runtime.cgocall(0x15ac750, 0xc6a5a49e40)
    /usr/local/src/go/src/runtime/cgocall.go:158 +0x5c fp=0xc6a5a49e18 sp=0xc6a5a49de0 pc=0x411c3c
github.com/h2non/bimg._Cfunc_vips_reduce_bridge(0x7f60c78f0960, 0xc11625abe0, 0x4000000000000000, 0x4000000000000000)
    _cgo_gotypes.go:1058 +0x4c fp=0xc6a5a49e40 sp=0xc6a5a49e18 pc=0xa44fac
github.com/h2non/bimg.vipsReduce.func2(0x7f60c78f0960?, 0xc6a5a49eb8?, 0x4000000000000000, 0x4000000000000000)
    /root/go/pkg/mod/github.com/h2non/bimg@v1.1.6/vips.go:657 +0x95 fp=0xc6a5a49e88 sp=0xc6a5a49e40 pc=0xa4de75
github.com/h2non/bimg.vipsReduce(0xc6a5a49f48?, 0x4505f1?, 0x450652?)
    /root/go/pkg/mod/github.com/h2non/bimg@v1.1.6/vips.go:657 +0x8a fp=0xc6a5a49f00 sp=0xc6a5a49e88 pc=0xa4dd0a
github.com/h2non/bimg.transformImage(_, {0x190, 0x190, 0x0, 0x0, 0x0, 0x0, 0x4b, 0x6, 0x0, ...}, ...)
    /root/go/pkg/mod/github.com/h2non/bimg@v1.1.6/resizer.go:230 +0xf7 fp=0xc6a5a4a0b8 sp=0xc6a5a49f00 pc=0xa47097
github.com/h2non/bimg.resizer({_, _, _}, {0x190, 0x190, 0x0, 0x0, 0x0, 0x0, 0x4b, ...})
    /root/go/pkg/mod/github.com/h2non/bimg@v1.1.6/resizer.go:107 +0x7a7 fp=0xc6a5a4a948 sp=0xc6a5a4a0b8 pc=0xa465e7
github.com/h2non/bimg.Resize({_, _, _}, {0x0, 0x190, 0x0, 0x0, 0x0, 0x0, 0x0, ...})
    /root/go/pkg/mod/github.com/h2non/bimg@v1.1.6/resize.go:15 +0xfc fp=0xc6a5a4ab68 sp=0xc6a5a4a948 pc=0xa4271c
github.com/h2non/bimg.(*Image).Process(_, {0x0, 0x190, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...})
    /root/go/pkg/mod/github.com/h2non/bimg@v1.1.6/image.go:203 +0x65 fp=0xc6a5a4ad20 sp=0xc6a5a4ab68 pc=0xa425a5
github.com/h2non/bimg.(*Image).ForceResize(0x1dfd1e0?, 0xc4f88745a0?, 0x41b6bf?)
    /root/go/pkg/mod/github.com/h2non/bimg@v1.1.6/image.go:30 +0x98 fp=0xc6a5a4b058 sp=0xc6a5a4ad20 pc=0xa424f8

it seems the following code not work correctly:

// resizer is used to transform a given image as byte buffer
// with the passed options.
func resizer(buf []byte, o Options) ([]byte, error) {
    defer C.vips_thread_shutdown() 
yourchanges commented 1 year ago

here is the test code:

func ResizedBybimg(ext string, read io.ReadSeeker, width, height int, mode string) (resized io.ReadSeeker, w int, h int) {
    if width == 0 && height == 0 {
        return read, 0, 0
    }
    srcImage, _, err := image.Decode(read)
    if err == nil {
        bounds := srcImage.Bounds()
        if bounds.Dx() > width && width != 0 || bounds.Dy() > height && height != 0 {
            read.Seek(0, 0)
            imageBuf, _ := io.ReadAll(read)
            dstImageBuff, err := bimg.NewImage(imageBuf).ForceResize(width, height)
            if err == nil {
                return bytes.NewReader(dstImageBuff), width, height
            }
            log.Println("bimg resize error", err)
        } else {
            read.Seek(0, 0)
            return read, bounds.Dx(), bounds.Dy()
        }
    } else {
        log.Println("image decode error", err)
    }
    read.Seek(0, 0)
    return read, 0, 0
}