search

pkujhd / goloader

load and run golang code at runtime.
Apache License 2.0
497 stars 58 forks source link

[WIP] - JIT compiler + bugfixes #65

Closed eh-steve closed 1 year ago

eh-steve commented 1 year ago

This is mostly a rebased commit history of #62, plus the addition of a new jit package to allow compiling and loading Go files/packages/text, and recursively resolving their dependencies.

This allows users to build and load arbitrary Go files with any number of imports automatically.

The jit package and tests are early on in the commit history to demonstrate the purpose of each subsequent commit in fixing specific tests.

Some tests are still failing randomly (e.g. TestJitCGoCall, TestStackSplit and TestJitPanicRecoveryStackTrace) due to some outstanding bugs in goloader which still need to be fixed.

Specifically:

runtime stack: runtime.throw({0x80b9d1?, 0xc0002c0240?}) /usr/local/go/src/runtime/panic.go:1047 +0x5d fp=0x7f5f277fd790 sp=0x7f5f277fd760 pc=0x439e7d runtime.gentraceback(0x0?, 0x0?, 0xe?, 0xc00058dfd8?, 0x0, 0x0, 0x7fffffff, 0x83c5e0, 0x0?, 0x0) /usr/local/go/src/runtime/traceback.go:258 +0x1cf7 fp=0x7f5f277fdb00 sp=0x7f5f277fd790 pc=0x4616f7 runtime.copystack(0xc001e99860, 0x800000002?) /usr/local/go/src/runtime/stack.go:932 +0x2f5 fp=0x7f5f277fdcb8 sp=0x7f5f277fdb00 pc=0x453bf5 runtime.newstack() /usr/local/go/src/runtime/stack.go:1112 +0x497 fp=0x7f5f277fde70 sp=0x7f5f277fdcb8 pc=0x454177 runtime.morestack() /usr/local/go/src/runtime/asm_amd64.s:570 +0x8b fp=0x7f5f277fde78 sp=0x7f5f277fde70 pc=0x46c34b

goroutine 170 [copystack]: crypto/internal/nistec/fiat.p384ToMontgomery(0xc00a33ca68, 0xc00a33c8e8) $GOROOT/src/crypto/internal/nistec/fiat/p384_fiat64.go:2116 +0x1b2b fp=0xc00a33c8d8 sp=0xc00a33c8d0 pc=0x40b8c0e3 crypto/internal/nistec/fiat.(P384Element).SetBytes(0xc00a33ca68, {0xc000275e29?, 0xb?, 0xc006f79aa0?}) $GOROOT/src/crypto/internal/nistec/fiat/p384.go:96 +0x179 fp=0xc00a33c958 sp=0xc00a33c8d8 pc=0x40b8a0f9 crypto/internal/nistec.(P384Point).SetBytes(0xc004bb0888, {0xc000275e28, 0xc00058cbc8?, 0x122}) $GOROOT/src/crypto/internal/nistec/p384.go:69 +0x1f6 fp=0xc00a33cb70 sp=0xc00a33c958 pc=0x40b8f1d6 crypto/elliptic.(nistCurve[go.shape.uint8_0]).Unmarshal(0x40d7bc00, {0xc000275e28, 0x61, 0x122}) $GOROOT/src/crypto/elliptic/nistec.go:257 +0x84 fp=0xc00a33cbd0 sp=0xc00a33cb70 pc=0x40b7cdfc crypto/elliptic.(nistCurve[crypto/internal/nistec.P384Point]).Unmarshal({0xc000275e28?, 0x40d7bc00?, 0x40b61498?})

:1 +0x3a fp=0xc00a33cc08 sp=0xc00a33cbd0 pc=0x40b90122 crypto/elliptic.Unmarshal({0x40da3f90, 0x40d7bc00}, {0xc000275e28, 0x61, 0x122}) $GOROOT/src/crypto/elliptic/elliptic.go:117 +0x2d6 fp=0xc00a33cc70 sp=0xc00a33cc08 pc=0x40bbf796 crypto/x509.parsePublicKey(0x3, 0xc00058d198) $GOROOT/src/crypto/x509/parser.go:258 +0x805 fp=0xc00a33cd58 sp=0xc00a33cc70 pc=0x40bbf175 crypto/x509.parseCertificate({0xc000275d40, 0x202, 0x20a}) $GOROOT/src/crypto/x509/parser.go:911 +0x8f2 fp=0xc00a33d240 sp=0xc00a33cd58 pc=0x40bbbec2 crypto/x509.ParseCertificate({0xc000275d40?, 0x202, 0x2df5b?}) $GOROOT/src/crypto/x509/parser.go:972 +0x25 fp=0xc00a33d268 sp=0xc00a33d240 pc=0x40bbb545 crypto/x509.(*CertPool).AppendCertsFromPEM(0xc00a3137d0, {0xc00a380000?, 0x40?, 0x4ed8aa4a391c0cb3?}) $GOROOT/src/crypto/x509/cert_pool.go:219 +0x119 fp=0xc00a33d388 sp=0xc00a33d268 pc=0x40bba529 runtime: g 170: unexpected return pc for io/fs.(*FileMode).IsDir called from 0xc00a3137d0 stack: frame={sp:0xc00a33d388, fp:0xc00a33d390} stack=[0xc00a33a000,0xc00a33e000) 0x000000c00a33d288: 0x000000c005666b00 0x000000000000054a 0x000000c00a33d298: 0x0000000000000561 0x00000000004afa57 0x000000c00a33d2a8: 0x010000c00a0f3578 0x0000000000000043 0x000000c00a33d2b8: 0x0000000000000046 0x0000000000000006 0x000000c00a33d2c8: 0x000000000002dc69 0x000000000002dc6a 0x000000c00a33d2d8: 0xfb8df2c40058d350 0xe133ee67d5eba579 0x000000c00a33d2e8: 0xec93b80898182804 0xb28c2f4eddbef9b8 0x000000c00a33d2f8: 0xd5eba579fb8df2c4 0x98182804e133ee67 0x000000c00a33d308: 0xddbef9b8ec93b808 0xfb8df2c4b28c2f4e 0x000000c00a33d318: 0xe133ee67d5eba579 0xec93b80898182804 0x000000c00a33d328: 0xb28c2f4eddbef9b8 0x000000c00507b9f0 0x000000c00a33d338: 0x000000c000209180 0x000000c00a330690 0x000000c00a33d348: 0x000000c00aac4680 0x000000c00507b900 0x000000c00a33d358: 0x000000c004bb07b0 0x000000c00a382e45 0x000000c00a33d368: 0x000000c004bb0798 0x000000c004bb07c8 0x000000c00a33d378: 0x000000c00a33d460 0x0000000040bc6416 0x000000c00a33d388: <0x000000c00a3137d0 >0x000000c00a380000 0x000000c00a33d398: 0x0000000000000040 0x4ed8aa4a391c0cb3 0x000000c00a33d3a8: 0x0000000000000008 0x0099999999999999 0x000000c00a33d3b8: 0x0000000000000038 0x0000000000000008 0x000000c00a33d3c8: 0x0000000000000000 0x0000000000000006 0x000000c00a33d3d8: 0x0009999999999999 0x000000c00058d420 0x000000c00a33d3e8: 0x0000000000000000 0x0000000000000040 0x000000c00a33d3f8: 0x0000000000000020 0x000000c00a3137d0 0x000000c00a33d408: 0x0000000000000000 0x0000000040d7c070 0x000000c00a33d418: 0x000000c009491dc0 0x302b000000000000 0x000000c00a33d428: 0x0000000000000000 0x0000000040d7c070 0x000000c00a33d438: 0x000000c00a3137a0 0x000000c00a313770 0x000000c00a33d448: 0x0000000000000000 0x000000c00058d4c8 0x000000c00a33d458: 0x000000000040f97f 0x000000c00058d470 0x000000c00a33d468: 0x0000000040bc5f53 0x000000c00058d4d0 0x000000c00a33d478: 0x0000000040b613d2 0x000100c0000ddc00 0x000000c00a33d488: 0x03007f5f685930f8 io/fs.(*FileMode).IsDir(0xc00a380000) :1 fp=0xc00a33d390 sp=0xc00a33d388 pc=0x40bc6416 created by net/http.(*persistConn).addTLS $GOROOT/src/net/http/transport.go:1534 +0x345 ``` * `TestJitCGoCall` segfaults after the first `Unload()` (but individual tests run fine one at a time): ``` unexpected fault address 0x40ea3cc0 fatal error: fault [signal SIGSEGV: segmentation violation code=0x1 addr=0x40ea3cc0 pc=0x47857c] goroutine 588 [running]: runtime.throw({0x7f5588?, 0x515eab?}) /usr/local/go/src/runtime/panic.go:1047 +0x5d fp=0xc0053a02a0 sp=0xc0053a0270 pc=0x439e7d runtime.sigpanic() /usr/local/go/src/runtime/signal_unix.go:842 +0x2c5 fp=0xc0053a02f0 sp=0xc0053a02a0 pc=0x450805 sync.poolCleanup() /usr/local/go/src/sync/pool.go:245 +0x3c fp=0xc0053a0300 sp=0xc0053a02f0 pc=0x47857c runtime.clearpools() /usr/local/go/src/runtime/mgc.go:1590 +0x2a fp=0xc0053a0340 sp=0xc0053a0300 pc=0x41eeaa runtime.gcStart({0x44?, 0x6bec35?, 0x1?}) /usr/local/go/src/runtime/mgc.go:673 +0x317 fp=0xc0053a03c8 sp=0xc0053a0340 pc=0x41c797 runtime.mallocgc(0x6bec35, 0x779d00, 0x1) /usr/local/go/src/runtime/malloc.go:1138 +0x80f fp=0xc0053a0440 sp=0xc0053a03c8 pc=0x40faef runtime.makeslice(0xc004f40268?, 0x2?, 0xc002fe9aa0?) /usr/local/go/src/runtime/slice.go:103 +0x52 fp=0xc0053a0468 sp=0xc0053a0440 pc=0x451bd2 cmd/objfile/objfile.openGoFile(0xc004f40268) /usr/local/go/src/cmd/objfile/objfile/goobj.go:42 +0x2a9 fp=0xc0053a0658 sp=0xc0053a0468 pc=0x62d309 cmd/objfile/objfile.Open({0xc000032a50, 0x50}) /usr/local/go/src/cmd/objfile/objfile/objfile.go:75 +0x53 fp=0xc0053a0710 sp=0xc0053a0658 pc=0x630213 github.com/pkujhd/goloader/obj.(*Pkg).Symbols(0xc0053a0e70) /home/git/goloader/obj/readobj.1.16.go:24 +0x8f fp=0xc0053a0b80 sp=0xc0053a0710 pc=0x632da2f github.com/pkujhd/goloader.readObj(0xc0053a0e70, 0xc006918580) /home/git/goloader/readobj.go:28 +0x73 fp=0xc0053a0d48 sp=0xc0053a0b80 pc=0x63ff73 github.com/pkujhd/goloader.ReadObjs({0xc0005d20a0, 0x1, 0x40?}, {0xc0005d2090, 0x1, 0x0?}) /home/git/goloader/readobj.go:99 +0x508 fp=0xc0053a0ed0 sp=0xc0053a0d48 pc=0x640e48 github.com/pkujhd/goloader/jit.buildAndLoadDeps({0x0, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, {0x8a0a18, 0x0}, 0x0}, ...) /home/git/goloader/jit/jit.go:269 +0x8a5 fp=0xc0053a13d8 sp=0xc0053a0ed0 pc=0x6a2ce5 github.com/pkujhd/goloader/jit.resolveDependencies({0x0, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, {0x8a0a18, 0x0}, 0x0}, ...) /home/git/goloader/jit/jit.go:153 +0x58e fp=0xc0053a1910 sp=0xc0053a13d8 pc=0x6a182e github.com/pkujhd/goloader/jit.BuildGoPackage({0x0, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, {0x8a0a18, 0x0}, 0x0}, ...) /home/git/goloader/jit/jit.go:629 +0x1893 fp=0xc0053a1db8 sp=0xc0053a1910 pc=0x6a8453 github.com/pkujhd/goloader/jit_test.buildLoadable(0xc0036371e0, {0x0, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, {0x8a0a18, 0x0}, ...}, ...) /home/git/goloader/jit/jit_test.go:30 +0x25e fp=0xc0053a1e98 sp=0xc0053a1db8 pc=0x749abe github.com/pkujhd/goloader/jit_test.TestJitCGoCall.func1(0xc0036371e0) /home/git/goloader/jit/jit_test.go:207 +0x6f fp=0xc0053a1f70 sp=0xc0053a1e98 pc=0x74b74f testing.tRunner(0xc0036371e0, 0xc00534e000) /usr/local/go/src/testing/testing.go:1446 +0x10b fp=0xc0053a1fc0 sp=0xc0053a1f70 pc=0x515eab testing.(*T).Run.func1() /usr/local/go/src/testing/testing.go:1493 +0x2a fp=0xc0053a1fe0 sp=0xc0053a1fc0 pc=0x516d4a runtime.goexit() /usr/local/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0053a1fe8 sp=0xc0053a1fe0 pc=0x46e421 created by testing.(*T).Run /usr/local/go/src/testing/testing.go:1493 +0x35f ``` So far, testing has only been on linux/amd64. Once the last few `goloader` bugs are ironed out, I will try to backport these changes for earlier Go versions and windows/darwin where possible.