valyala / gozstd

go wrapper for zstd
MIT License
420 stars 60 forks source link

Initialization fails when running go test -race on a module which includes gozstd. #40

Open Villenny opened 2 years ago

Villenny commented 2 years ago

Running any test or benchmark from this file with -race will cause the below error

==17204==ERROR: ThreadSanitizer failed to allocate 0x000000a61000 (10883072) bytes at 0x200dc6f9fc000 (error code: 87)
exit status 66
package crashOnLoad

import (
    "bytes"
    "compress/gzip"
    "encoding/base64"
    "io/ioutil"
    "sync"
    "testing"

    "github.com/stretchr/testify/assert"
    "github.com/valyala/gozstd"
)

func TestThisCantRunEither(t *testing.T) {
}

func BenchmarkCompressValyalaZstd(b *testing.B) {
    b.ResetTimer()
    for i := 0; i < b.N; i += 1 {
        func() {
            compressedBytes := gozstd.Compress(make([]byte, 0, len(SampleVast)), SampleVast)
            compressionRatio := float64(len(SampleVast)) / float64(len(compressedBytes))
            if compressionRatio < 3.8 {
                panic("WTF")
            }
        }()
    }
}

func BenchmarkDecompressValyalaZstd(b *testing.B) {
    compressedBytes := gozstd.Compress(make([]byte, 0, len(SampleVast)), SampleVast)
    compressionRatio := float64(len(SampleVast)) / float64(len(compressedBytes))
    if compressionRatio < 3.8 {
        panic("WTF")
    }

    b.ResetTimer()
    for i := 0; i < b.N; i += 1 {
        func() {
            decompressedBytes, err := gozstd.Decompress(make([]byte, 0, 8192), compressedBytes)
            if err != nil {
                panic("wtf")
            }
            if len(decompressedBytes) <= 0 {
                panic("wtf")
            }
        }()
    }
}
go test -benchmem -run=^$ -coverprofile=go-code-cover -bench ^TestThisCantRunEither$ crashOnLoad -race -v -memprofile memprofile.out -cpuprofile profile.out -benchtime 3s
valyala commented 2 years ago

The error failed to allocate 0x000000a61000 (10883072) bytes at 0x200dc6f9fc000 (error code: 87) suggests that the host system doesn't have enough RAM for completing the test with race detector enabled. Try running the test on a machine with more RAM.