search

ostap / comp

a tool for querying files in various formats
MIT License
43 stars 2 forks source link

out of memory with ~16GB loaded #1

Closed julochrobak closed 11 years ago

julochrobak commented 11 years ago

When trying to load quite some data into the prod comp, I get an out of memory error with about 16GB loaded and the comp crashes. I'm loading in 22 files of total 2.9GB.

    throw: out of memory

    goroutine 740 [running]:
    strconv.atof64(0xfc3ff49a5e, 0x7f7800000000, 0x0, 0x0, 0x0, ...)
        /usr/lib/go/src/pkg/strconv/atof.go:388 +0x13f
    strconv.ParseFloat(0xfc3ff49a5e, 0xfc00000000, 0x40, 0x568ae0, 0x0, ...)
        /usr/lib/go/src/pkg/strconv/atof.go:436 +0x8d
    main.TabDelimParser(0x2200000015, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:129 +0x2de
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 1 [select]:
    main.ReadBody(0xfb58b928a0, 0x7fffe427fce5, 0xfb0000001a, 0x20, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:94 +0x690
    main.main()
        /home/ubuntu/comp/src/comp/main.go:50 +0xba5

    goroutine 2 [syscall]:
    created by runtime.main
        /build/buildd/golang-1.0.2/src/pkg/runtime/proc.c:221

    goroutine 742 [chan receive]:
    main.TabDelimParser(0x2200000017, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 749 [chan receive]:
    main.TabDelimParser(0x220000001e, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 731 [chan receive]:
    main.TabDelimParser(0x220000000c, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 719 [chan receive]:
    main.TabDelimParser(0x2200000000, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 718 [running]:
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:73 +0x1b0

    goroutine 738 [chan receive]:
    main.TabDelimParser(0x2200000013, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 747 [chan receive]:
    main.TabDelimParser(0x220000001c, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 724 [running]:
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 720 [chan receive]:
    main.TabDelimParser(0x2200000001, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 734 [chan receive]:
    main.TabDelimParser(0x220000000f, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 743 [chan receive]:
    main.TabDelimParser(0x2200000018, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 726 [chan receive]:
    main.TabDelimParser(0x2200000007, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 735 [running]:
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 745 [chan receive]:
    main.TabDelimParser(0x220000001a, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 751 [chan receive]:
    main._func_029(0xfb5dea91d0, 0xfb5dea91f8, 0xfb5dea91f0, 0x0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:83 +0x42
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:86 +0x2bf

    goroutine 727 [chan receive]:
    main.TabDelimParser(0x2200000008, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 730 [chan receive]:
    main.TabDelimParser(0x220000000b, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 746 [running]:
    main.TabDelimParser(0x220000001b, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:131 +0x45e
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 729 [chan receive]:
    main.TabDelimParser(0x220000000a, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 748 [chan receive]:
    main.TabDelimParser(0x220000001d, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 728 [chan receive]:
    main.TabDelimParser(0x2200000009, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 737 [chan receive]:
    main.TabDelimParser(0x2200000012, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 732 [running]:
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 725 [chan receive]:
    main.TabDelimParser(0x2200000006, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 721 [chan receive]:
    main.TabDelimParser(0x2200000002, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 741 [chan receive]:
    main.TabDelimParser(0x2200000016, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 750 [running]:
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 744 [chan receive]:
    main.TabDelimParser(0x2200000019, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 739 [running]:
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 723 [chan receive]:
    main.TabDelimParser(0x2200000004, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 733 [running]:
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 722 [chan receive]:
    main.TabDelimParser(0x2200000003, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 736 [chan receive]:
    main.TabDelimParser(0x2200000011, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:115 +0x54
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289

    goroutine 37 [syscall]:
    created by addtimer
        /build/buildd/golang-1.0.2/src/pkg/runtime/ztime_amd64.c:72
    throw: out of memory

    goroutine 750 [running]:
    strings.genSplit(0xf98914a9a0, 0x66, 0x63a334, 0xf900000001, 0x2200000000, ...)
        /usr/lib/go/src/pkg/strings/strings.go:198 +0x10e
    strings.Split(0xf98914a9a0, 0x66, 0x63a334, 0x900000001, 0x66, ...)
        /usr/lib/go/src/pkg/strings/strings.go:236 +0x53
    main.TabDelimParser(0x220000001f, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:116 +0xe1
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289
    throw: out of memory

    goroutine 739 [running]:
    strconv.atof64(0xfc3ff7a2f1, 0x7f7800000000, 0x0, 0x0, 0x0, ...)
        /usr/lib/go/src/pkg/strconv/atof.go:388 +0x13f
    strconv.ParseFloat(0xfc3ff7a2f1, 0xfc00000000, 0x40, 0x568ae0, 0x0, ...)
        /usr/lib/go/src/pkg/strconv/atof.go:436 +0x8d
    main.TabDelimParser(0x2200000014, 0xfb82856100, 0xf9d64f3500, 0xfb58a51fa0, 0x0, ...)
        /home/ubuntu/comp/src/comp/file.go:129 +0x2de
    created by main.ReadBody
        /home/ubuntu/comp/src/comp/file.go:79 +0x289
ostap commented 11 years ago

Go runtime cannot work with heaps above 16G. This is not a problem for the prod version since each instance handles a partition of a data set (just create more partitions and run more comps). However with the new comp it will be a little bit more tricky since each comp needs to be able to "see" all data. I would close this issue since its fairly obvious. Any objections?

julochrobak commented 11 years ago

ok, so 16gb heap is to be solved with go upgrade to 1.1