search

bytedance / sonic

A blazingly fast JSON serializing & deserializing library
Apache License 2.0
6.84k stars 335 forks source link

bug: steam IO encode performance is unexpect, freeBytes is not done #542

Closed 201430098137 closed 11 months ago

201430098137 commented 11 months ago

recently, I use stram IO to reduct memory usage, but memory usage is higher then sonic.Marshal, I bench it. I encode map[string]interface{} to json and write it to file,

var cases = []map[string]interface{}{
    {
        "dsds": "dsdds",
        "dsalkjlsd":   "sasasas",
        "dsalkjlsdd":  "sasasas",
        "dsalkjlsds":  "sasasas",
        "dsalkjlasd":  "sasasas",
        "dsalkjlsdsf": "sasasas",
    },
    {
        "dsdsd": "dsdsd",
        "dsalkjalsd":   "sasasas",
        "dsalkjlsdsdd":  "sasasas",
        "dsalkjlsdss":  "sasasas",
        "dsalkjlasdsd":  "sasasas",
        "dsalkjlssfddsf": "sasasas",
        "dsalkjlssdfd":   "sasasas",
        "dsalkjlssdd":  "sasasas",
        "dsalkjlssds":  "sasasas",
        "dsalkjlassd":  "sasasas",
        "dsalkjlscdsf": "sasasas",
        "dsalkjlsdsd":   "sasasas",
        "dsalkjlsdd":  "sasasas",
        "dsalkjlsds":  "sasasas",
        "dsalkjlasd":  "sasasas",
        "dsalkjlsdsf": "sasasas",
    },
    {
        "dsdsd": "dsdsd",
        "dsalkjalsd":   "sasasas",
        "dsalkjlsdsdd":  "sasasas",
        "dsalkjlsdss":  "sasasas",
        "dsalkjlasdsd":  "sasasas",
        "dsalkjlssfddsf": "sasasas",
        "dsalkjlssdfd":   "sasasas",
        "dsalkjlssdd":  "sasasas",
        "dsalkjlssds":  "sasasas",
        "dsalkjlassd":  "sasasas",
        "dsalkjlscdsf": "sasasas",
        "dsalkjlsdsd":   "sasasas",
        "dsalkjlsdd":  "sasasas",
        "dsalkjlsds":  "sasasas",
        "dsalkjlasd":  "sasasas",
        "dsalkjlsdsf": "sasasas",
        "sasdas":"asa",
        "saasdss":"asa",
        "sassdc":"asa",
        "sasvsdsdas":"asa",
        "sasjyja":"asa",
        "sagncs":"asa",
        "sabdfss":"asa",
        "santys":"asa",
        "saabfdss":"asa",
        "sasbdfc":"asa",
        "sasbdfas":"asa",
        "sabdfsa":"asa",
        "sacdbfas":"asa",
        "sasss":"asa",
        "sasaasf":"asa",
        "saasdass":"asa",
        "sasavsasc":"asa",
        "sfda":"asa",
        "saasfsa":"asa",
        "sasafscs":"asa",
        "asf":"asa",
        "vxxz":"asa",
        "zxc":"asa",
        "nrt":"asa",
        "dbf":"asa",
        "sabwdfsa":"asa",
        "bdf":"asa",
        "sazxvss":"asa",
        "sasvzv":"asa",
        "saaczczxss":map[string]interface{}{

            "dsdsd": "dsdsd",
            "dsalkjalsd":   "sasasas",
            "dsalkjlsdsdd":  "sasasas",
            "dsalkjlsdss":  "sasasas",
            "dsalkjlasdsd":  "sasasas",
            "dsalkjlssfddsf": "sasasas",
            "dsalkjlssdfd":   "sasasas",
            "dsalkjlssdd":  "sasasas",
            "dsalkjlssds":  "sasasas",
            "dsalkjlassd":  "sasasas",
            "dsalkjlscdsf": "sasasas",
            "dsalkjlsdsd":   "sasasas",
            "dsalkjlsdd":  "sasasas",
            "dsalkjlsds":  "sasasas",
            "dsalkjlasd":  "sasasas",
            "dsalkjlsdsf": "sasasas",
        },
        "cz":"asa",
        "sasvzxas":map[string]interface{}{

            "dsdsd": "dsdsd",
            "dsalkjalsd":   "sasasas",
            "dsalkjlsdsdd":  "sasasas",
            "dsalkjlsdss":  "sasasas",
            "dsalkjlasdsd":  "sasasas",
            "dsalkjlssfddsf": "sasasas",
            "dsalkjlssdfd":   "sasasas",
            "dsalkjlssdd":  "sasasas",
            "dsalkjlssds":  "sasasas",
            "dsalkjlassd":  "sasasas",
            "dsalkjlscdsf": "sasasas",
            "dsalkjlsdsd":   "sasasas",
            "dsalkjlsdd":  "sasasas",
            "dsalkjlsds":  "sasasas",
            "dsalkjlasd":  "sasasas",
            "dsalkjlsdsf": "sasasas",
        },
        "vz":map[string]interface{}{

            "dsdsd": "dsdsd",
            "dsalkjalsd":   "sasasas",
            "dsalkjlsdsdd":  "sasasas",
            "dsalkjlsdss":  "sasasas",
            "dsalkjlasdsd":  "sasasas",
            "dsalkjlssfddsf": "sasasas",
            "dsalkjlssdfd":   "sasasas",
            "dsalkjlssdd":  "sasasas",
            "dsalkjlssds":  "sasasas",
            "dsalkjlassd":  "sasasas",
            "dsalkjlscdsf": "sasasas",
            "dsalkjlsdsd":   "sasasas",
            "dsalkjlsdd":  "sasasas",
            "dsalkjlsds":  "sasasas",
            "dsalkjlasd":  "sasasas",
            "dsalkjlsdsf": "sasasas",
        },
        "sazxvcs":map[string]interface{}{

            "dsdsd": "dsdsd",
            "dsalkjalsd":   "sasasas",
            "dsalkjlsdsdd":  "sasasas",
            "dsalkjlsdss":  "sasasas",
            "dsalkjlasdsd":  "sasasas",
            "dsalkjlssfddsf": "sasasas",
            "dsalkjlssdfd":   "sasasas",
            "dsalkjlssdd":  "sasasas",
            "dsalkjlssds":  "sasasas",
            "dsalkjlassd":  "sasasas",
            "dsalkjlscdsf": "sasasas",
            "dsalkjlsdsd":   "sasasas",
            "dsalkjlsdd":  "sasasas",
            "dsalkjlsds":  "sasasas",
            "dsalkjlasd":  "sasasas",
            "dsalkjlsdsf": "sasasas",
        },
        "sasvzxs":map[string]interface{}{

            "dsdsd": "dsdsd",
            "dsalkjalsd":   "sasasas",
            "dsalkjlsdsdd":  "sasasas",
            "dsalkjlsdss":  "sasasas",
            "dsalkjlasdsd":  "sasasas",
            "dsalkjlssfddsf": "sasasas",
            "dsalkjlssdfd":   "sasasas",
            "dsalkjlssdd":  "sasasas",
            "dsalkjlssds":  "sasasas",
            "dsalkjlassd":  "sasasas",
            "dsalkjlscdsf": "sasasas",
            "dsalkjlsdsd":   "sasasas",
            "dsalkjlsdd":  "sasasas",
            "dsalkjlsds":  "sasasas",
            "dsalkjlasd":  "sasasas",
            "dsalkjlsdsf": "sasasas",
        },
        "sl;kas":map[string]interface{}{

                "dsdsd": "dsdsd",
                "dsalkjalsd":   "sasasas",
                "dsalkjlsdsdd":  "sasasas",
                "dsalkjlsdss":  "sasasas",
                "dsalkjlasdsd":  "sasasas",
                "dsalkjlssfddsf": "sasasas",
                "dsalkjlssdfd":   "sasasas",
                "dsalkjlssdd":  "sasasas",
                "dsalkjlssds":  "sasasas",
                "dsalkjlassd":  "sasasas",
                "dsalkjlscdsf": "sasasas",
                "dsalkjlsdsd":   "sasasas",
                "dsalkjlsdd":  "sasasas",
                "dsalkjlsds":  "sasasas",
                "dsalkjlasd":  "sasasas",
                "dsalkjlsdsf": "sasasas",
        },

    },
}
func BenchmarkEncode(b *testing.B) {
    fd, err := os.OpenFile("/tmp/aa.json", os.O_RDWR|os.O_CREATE|os.O_APPEND, os.ModePerm)
    if err != nil {
        b.Errorf("打开文件错误：%s", err.Error())
    }
    var enc = sonic.ConfigDefault.NewEncoder(fd)
    for i := 0; i < b.N; i++ {
        err := enc.Encode(cases[i%len(cases)])
        if err != nil {
            b.Errorf("enc.Encode错误：%s", err.Error())
        }
    }
}

func BenchmarkMarshal(b *testing.B) {
    fd, err := os.OpenFile("/tmp/bb.json", os.O_RDWR|os.O_CREATE|os.O_APPEND, os.ModePerm)
    if err != nil {
        b.Errorf("打开文件错误：%s", err.Error())
    }

    for i := 0; i < b.N; i++ {
        bytes, err := sonic.Marshal(cases[i%len(cases)])
        if err != nil {
            b.Errorf("sonic.Marshal错误：%s", err.Error())
        }
        _, err = fd.Write(bytes)
        if err != nil {
            b.Errorf("Write错误：%s", err.Error())
        }
        _, err = fd.Write([]byte{'\n'})
        if err != nil {
            b.Errorf("Write错误：%s", err.Error())
        }
    }

}

profile memory stream IO image sonic.Marshal image

Interestingly, stream IO use more memory then sonic.Marshal. I debugger that code, I discover the issue.

image line 71 out is set out[n:], data of out is unaccess, line 81 freeBytes is useless.
image

201430098137 commented 11 months ago

code: github.com/bytedance/sonic@v1.10.2/internal/encoder/stream.go:70

liuq19 commented 11 months ago

Thanks. we will investigate it. cc @AsterDY

AsterDY commented 11 months ago

fixed by #547