[fatal error: runtime: out of memory] when parsing events using types.EventRecordsRaw.DecodeEventRecords

[AstaFrode]

When the program is running normally, this error suddenly appears and directly causes the program to be killed.

golang：go 1.19

go-substrate-rpc-client: github.com/centrifuge/go-substrate-rpc-client/v4 v4.0.13

related code:

        ......
        // Do the transfer and track the actual status
    sub, err := c.api.RPC.Author.SubmitAndWatchExtrinsic(ext)
    if err != nil {
        c.SetChainState(false)
        return txhash, errors.Wrap(err, "[SubmitAndWatchExtrinsic]")
    }
    defer sub.Unsubscribe()

    timeout := time.NewTimer(c.timeForBlockOut)
    defer timeout.Stop()

    for {
        select {
        case status := <-sub.Chan():
            if status.IsInBlock {
                events := event.EventRecords{}
                txhash, _ = codec.EncodeToHex(status.AsInBlock)
                h, err := c.api.RPC.State.GetStorageRaw(c.keyEvents, status.AsInBlock)
                if err != nil {
                    return txhash, errors.Wrap(err, "[GetStorageRaw]")
                }
                err = types.EventRecordsRaw(*h).DecodeEventRecords(c.metadata, &events)
                if err != nil || len(events.FileBank_UploadDeclaration) > 0 {
                    return txhash, nil
                }
                return txhash, errors.New(pattern.ERR_Failed)
            }
        case err = <-sub.Err():
            return txhash, errors.Wrap(err, "[sub]")
        case <-timeout.C:
            return txhash, pattern.ERR_RPC_TIMEOUT
        }
    }

system：

[root@centos]# cat /etc/os-release 
NAME="CentOS Stream"
VERSION="8"
ID="centos"
ID_LIKE="rhel fedora"
VERSION_ID="8"
PLATFORM_ID="platform:el8"
PRETTY_NAME="CentOS Stream 8"
ANSI_COLOR="0;31"
CPE_NAME="cpe:/o:centos:centos:8"
HOME_URL="https://centos.org/"
BUG_REPORT_URL="https://bugzilla.redhat.com/"
REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8"
REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream"
[root@centos]# cat /proc/version 
Linux version 4.18.0-408.el8.x86_64 (mockbuild@kbuilder.bsys.centos.org) (gcc version 8.5.0 20210514 (Red Hat 8.5.0-14) (GCC)) #1 SMP Mon Jul 18 17:42:52 UTC 2022

error message:

fatal error: runtime: out of memory

runtime stack:
runtime.throw({0x188f097?, 0x2030?})
    /opt/go/src/runtime/panic.go:1047 +0x5d fp=0x7ff0d7ffeca8 sp=0x7ff0d7ffec78 pc=0x43b09d
runtime.sysMapOS(0xc048400000, 0x754400000?)
    /opt/go/src/runtime/mem_linux.go:187 +0x11b fp=0x7ff0d7ffecf0 sp=0x7ff0d7ffeca8 pc=0x41aa5b
runtime.sysMap(0x29ab260?, 0x7ff107780000?, 0x42fdc0?)
    /opt/go/src/runtime/mem.go:142 +0x35 fp=0x7ff0d7ffed20 sp=0x7ff0d7ffecf0 pc=0x41a435
runtime.(*mheap).grow(0x29ab260, 0x3aa189?)
    /opt/go/src/runtime/mheap.go:1459 +0x23d fp=0x7ff0d7ffed90 sp=0x7ff0d7ffed20 pc=0x42cdbd
runtime.(*mheap).allocSpan(0x29ab260, 0x3aa189, 0x0, 0x1)
    /opt/go/src/runtime/mheap.go:1191 +0x1be fp=0x7ff0d7ffee28 sp=0x7ff0d7ffed90 pc=0x42c51e
runtime.(*mheap).alloc.func1()
    /opt/go/src/runtime/mheap.go:910 +0x65 fp=0x7ff0d7ffee70 sp=0x7ff0d7ffee28 pc=0x42bfa5
runtime.systemstack()
    /opt/go/src/runtime/asm_amd64.s:492 +0x49 fp=0x7ff0d7ffee78 sp=0x7ff0d7ffee70 pc=0x46a0a9

goroutine 269 [running]:
runtime.systemstack_switch()
    /opt/go/src/runtime/asm_amd64.s:459 fp=0xc003230248 sp=0xc003230240 pc=0x46a040
runtime.(*mheap).alloc(0x754312000?, 0x3aa189?, 0xff?)
    /opt/go/src/runtime/mheap.go:904 +0x65 fp=0xc003230290 sp=0xc003230248 pc=0x42bee5
runtime.(*mcache).allocLarge(0x7ff1077a0108?, 0x754310b40, 0x1)
    /opt/go/src/runtime/mcache.go:233 +0x85 fp=0xc0032302e0 sp=0xc003230290 pc=0x4193c5
runtime.mallocgc(0x754310b40, 0x15409e0, 0x1)
    /opt/go/src/runtime/malloc.go:1029 +0x57e fp=0xc003230358 sp=0xc0032302e0 pc=0x40f2de
runtime.newarray(0x15c73a0?, 0xc003230301?)
    /opt/go/src/runtime/malloc.go:1214 +0x52 fp=0xc003230380 sp=0xc003230358 pc=0x40f7d2
reflect.unsafe_NewArray(0xc0047ada00?, 0xc001c4a0cc?)
    /opt/go/src/runtime/malloc.go:1219 +0x19 fp=0xc0032303a0 sp=0xc003230380 pc=0x4656f9
reflect.MakeSlice({0x1bf4850, 0x145cc00}, 0x3aa1885a, 0x3aa1885a)
    /opt/go/src/reflect/value.go:2991 +0xac fp=0xc0032303d0 sp=0xc0032303a0 pc=0x4a7d2c
github.com/centrifuge/go-substrate-rpc-client/v4/scale.Decoder.DecodeIntoReflectValue({{0x1bcb960?, 0xc003edaa20?}}, {0x145cc00?, 0xc004b38608?, 0xc003230678?})
    /root/go/pkg/mod/github.com/centrifuge/go-substrate-rpc-client/v4@v4.0.13/scale/codec.go:436 +0x53d fp=0xc0032305e0 sp=0xc0032303d0 pc=0x54f27d
github.com/centrifuge/go-substrate-rpc-client/v4/scale.Decoder.Decode({{0x1bcb960?, 0xc003edaa20?}}, {0xc0047ada00?, 0xc004b38608})
    /root/go/pkg/mod/github.com/centrifuge/go-substrate-rpc-client/v4@v4.0.13/scale/codec.go:325 +0x28d fp=0xc003230688 sp=0xc0032305e0 pc=0x54ec6d
github.com/centrifuge/go-substrate-rpc-client/v4/types.EventRecordsRaw.DecodeEventRecords({0xc00165ca00, 0x378, 0x700}, 0xd2bed06f13aa81bb?, {0x1428b00?, 0xc0006bcf00})
    /root/go/pkg/mod/github.com/centrifuge/go-substrate-rpc-client/v4@v4.0.13/types/event_record.go:558 +0x1695 fp=0xc003230a08 sp=0xc003230688 pc=0x604615
github.com/CESSProject/cess-go-sdk/chain.(*ChainSDK).UploadDeclaration(0xc00021f100, {0xc00069cabf, 0x40}, {0xc001dd48a0, 0x1, 0x1}, {{0xfa, 0xf, 0x89, 0x99, ...}, ...}, ...)
    /root/go/pkg/mod/github.com/!c!e!s!s!project/cess-go-sdk@v0.3.3/chain/fileBank.go:697 +0x10dd fp=0xc0032315b8 sp=0xc003230a08 pc=0x135891d
github.com/CESSProject/cess-go-sdk/chain.(*ChainSDK).GenerateStorageOrder(0x0?, {0xc00069cabf, 0x40}, {0xc0007a2280, 0x1, 0x0?}, {0xc001bd8990, 0x20, 0x0?}, {0xc0006b3393, ...}, ...)
    /root/go/pkg/mod/github.com/!c!e!s!s!project/cess-go-sdk@v0.3.3/chain/file.go:142 +0x2eb fp=0xc0032316f0 sp=0xc0032315b8 pc=0x134eb2b
github.com/CESSProject/DeOSS/node.(*Node).backupFiles(0xc000694090, {0xc001bd8990, 0x20, 0x21}, {0xc0007a2280, 0x1, 0x10}, {0xc00069cabf, 0x40}, {0xc0006b3393, ...}, ...)
    /go/src/DeOSS/node/tracker.go:217 +0x2d0 fp=0xc0032319c8 sp=0xc0032316f0 pc=0x12a9e90
github.com/CESSProject/DeOSS/node.(*Node).trackFile(0xc000694090, {0xc00069ca80, 0x7f})
    /go/src/DeOSS/node/tracker.go:182 +0x9e5 fp=0xc003231f28 sp=0xc0032319c8 pc=0x12a9045
github.com/CESSProject/DeOSS/node.(*Node).tracker(0xc000694090, 0x8?)
    /go/src/DeOSS/node/tracker.go:65 +0x20d fp=0xc003231fc0 sp=0xc003231f28 pc=0x12a84cd
github.com/CESSProject/DeOSS/node.(*Node).TaskMgt.func1()
    /go/src/DeOSS/node/taskMgt.go:17 +0x2a fp=0xc003231fe0 sp=0xc003231fc0 pc=0x12a828a
runtime.goexit()
    /opt/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003231fe8 sp=0xc003231fe0 pc=0x46c261
created by github.com/CESSProject/DeOSS/node.(*Node).TaskMgt
    /go/src/DeOSS/node/taskMgt.go:17 +0xca

[cdamian]

Hey @AstaFrode, do you know what events are in the block when this issue is encountered? Are there any XCM-related events in there?

[AstaFrode]

@cdamian The blocks that went wrong had the following events，no XCM related events.

You can check it out in the CESS browser. The block hash is 0xbbc3aef1f9040e76ef38793ba75a58c0a76dfea76b57ad8aa0cb9518b7188377

[ygcool]

Hi, I had the same problem, have you solved it yet?

[AstaFrode]

@ygcool I now rely on docker's restart mechanism to keep the service running. @cdamian Is there any progress on this issue?

[cdamian]

We did not find the cause of the memory leak so far. An alternative would be to use the recently added event retriever - https://github.com/centrifuge/go-substrate-rpc-client/blob/master/registry/retriever/event_retriever_live_test.go#L52

[khssnv]

I'm experiencing the same with v4.2.1. But it actually does not fail with OOM in my case, but consumes about 20GB+ RAM and a lot of CPU before returning an error on an attempt to decode events from a runtime where the event definition has one more field than the type in my EventRecords.