search

tomnomnom / waybackurls

Fetch all the URLs that the Wayback Machine knows about for a domain
3.43k stars 457 forks source link

fatal error: runtime: cannot allocate memory #7

Open joeldeleep opened 5 years ago

joeldeleep commented 5 years ago

While running cat domains.txt | waybackurls it runs for a few seconds and throw an error " fatal error: runtime: cannot allocate memory " then exit the code with go errors

goroutine 5 [running]: runtime.throw(0x6eb0bc, 0x1f) /usr/local/go/src/runtime/panic.go:617 +0x72 fp=0xc0003e63f8 sp=0xc0003e63c8 pc=0x42d202 runtime.newArenaMayUnlock(0x919e80) /usr/local/go/src/runtime/mheap.go:1930 +0xda fp=0xc0003e6430 sp=0xc0003e63f8 pc=0x427eda runtime.newMarkBits(0x200, 0x100) /usr/local/go/src/runtime/mheap.go:1850 +0xc3 fp=0xc0003e6478 sp=0xc0003e6430 pc=0x427ac3 runtime.heapBits.initSpan(0x7f199c0a9600, 0x20300000000000, 0x7f199c103fff, 0x7f199e77dc10) /usr/local/go/src/runtime/mbitmap.go:792 +0x75 fp=0xc0003e64f8 sp=0xc0003e6478 pc=0x4164a5 runtime.(mcentral).grow(0x9205a0, 0x0) /usr/local/go/src/runtime/mcentral.go:264 +0x12e fp=0xc0003e6540 sp=0xc0003e64f8 pc=0x418b5e runtime.(mcentral).cacheSpan(0x9205a0, 0xbf) /usr/local/go/src/runtime/mcentral.go:106 +0x2ff fp=0xc0003e65a0 sp=0xc0003e6540 pc=0x4185cf runtime.(mcache).refill(0x7f199e8c4008, 0x205) /usr/local/go/src/runtime/mcache.go:135 +0x86 fp=0xc0003e65c0 sp=0xc0003e65a0 pc=0x418066 runtime.(mcache).nextFree(0x7f199e8c4008, 0x5, 0xc0003e6638, 0x47dbcd, 0x6778e0) /usr/local/go/src/runtime/malloc.go:786 +0x88 fp=0xc0003e65f8 sp=0xc0003e65c0 pc=0x40ca08 runtime.mallocgc(0x9, 0x0, 0x100, 0x18) /usr/local/go/src/runtime/malloc.go:915 +0x589 fp=0xc0003e6698 sp=0xc0003e65f8 pc=0x40d139 runtime.slicebytetostring(0x0, 0xc0091dbf70, 0x9, 0x2e23e90, 0x0, 0x0) /usr/local/go/src/runtime/string.go:102 +0xa1 fp=0xc0003e66c8 sp=0xc0003e6698 pc=0x446291 encoding/json.(decodeState).literalStore(0xc0000781e0, 0xc0091dbf6f, 0xb, 0x2e23e91, 0x67f320, 0xc0034a82f0, 0x198, 0xc0003e6a00, 0x487fd2, 0x197) /usr/local/go/src/encoding/json/decode.go:947 +0x236d fp=0xc0003e6a28 sp=0xc0003e66c8 pc=0x4ce71d encoding/json.(decodeState).value(0xc0000781e0, 0x67f320, 0xc0034a82f0, 0x198, 0x67f320, 0xc0034a82f0) /usr/local/go/src/encoding/json/decode.go:395 +0x1ef fp=0xc0003e6a90 sp=0xc0003e6a28 pc=0x4c870f encoding/json.(decodeState).array(0xc0000781e0, 0x67bd20, 0xc002cf3f78, 0x197, 0xc000078208, 0x5b) /usr/local/go/src/encoding/json/decode.go:560 +0x1a9 fp=0xc0003e6b78 sp=0xc0003e6a90 pc=0x4c9179 encoding/json.(decodeState).value(0xc0000781e0, 0x67bd20, 0xc002cf3f78, 0x197, 0x67bd20, 0xc002cf3f78) /usr/local/go/src/encoding/json/decode.go:371 +0xff fp=0xc0003e6be0 sp=0xc0003e6b78 pc=0x4c861f encoding/json.(decodeState).array(0xc0000781e0, 0x6705a0, 0xc00000e3e0, 0x16, 0xc000078208, 0x5b) /usr/local/go/src/encoding/json/decode.go:560 +0x1a9 fp=0xc0003e6cc8 sp=0xc0003e6be0 pc=0x4c9179 encoding/json.(decodeState).value(0xc0000781e0, 0x6705a0, 0xc00000e3e0, 0x16, 0xc0003e6d78, 0x4d8ce2) /usr/local/go/src/encoding/json/decode.go:371 +0xff fp=0xc0003e6d30 sp=0xc0003e6cc8 pc=0x4c861f encoding/json.(decodeState).unmarshal(0xc0000781e0, 0x6705a0, 0xc00000e3e0, 0xc000078208, 0x0) /usr/local/go/src/encoding/json/decode.go:179 +0x209 fp=0xc0003e6db8 sp=0xc0003e6d30 pc=0x4c7da9 encoding/json.Unmarshal(0xc008000000, 0x28144b8, 0x3fffe00, 0x6705a0, 0xc00000e3e0, 0x3fffe00, 0x0) /usr/local/go/src/encoding/json/decode.go:106 +0x123 fp=0xc0003e6e00 sp=0xc0003e6db8 pc=0x4c7783 main.getWaybackURLs(0xc000016280, 0x19, 0xc000018300, 0x0, 0x0, 0x0, 0x0, 0x0) /home/rootx/go/src/github.com/tomnomnom/waybackurls/main.go:127 +0x24f fp=0xc0003e6f10 sp=0xc0003e6e00 pc=0x651d4f main.main.func1(0xc000018310, 0x6f7598, 0xc000012bf0, 0xc000018309, 0xc000060120) /home/rootx/go/src/github.com/tomnomnom/waybackurls/main.go:57 +0x90 fp=0xc0003e6fb8 sp=0xc0003e6f10 pc=0x6526f0 runtime.goexit() /usr/local/go/src/runtime/asm_amd64.s:1337 +0x1 fp=0xc0003e6fc0 sp=0xc0003e6fb8 pc=0x458981 created by main.main /home/rootx/go/src/github.com/tomnomnom/waybackurls/main.go:55 +0x4c4 goroutine 1 [chan receive]: main.main() /home/rootx/go/src/github.com/tomnomnom/waybackurls/main.go:76 +0x5b2 goroutine 7 [semacquire]: sync.runtime_Semacquire(0xc000018318) /usr/local/go/src/runtime/sema.go:56 +0x39 sync.(WaitGroup).Wait(0xc000018310) /usr/local/go/src/sync/waitgroup.go:130 +0x65 main.main.func2(0xc000018310, 0xc000060120) /home/rootx/go/src/github.com/tomnomnom/waybackurls/main.go:71 +0x2b created by main.main /home/rootx/go/src/github.com/tomnomnom/waybackurls/main.go:70 +0x50c goroutine 19 [IO wait]: internal/poll.runtime_pollWait(0x7f199e878f08, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:182 +0x56 internal/poll.(pollDesc).wait(0xc0000ca798, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc0000ca780, 0xc0000c9000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b net.(netFD).Read(0xc0000ca780, 0xc0000c9000, 0x1000, 0x1000, 0x1, 0x42f01c, 0xc000047b88) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc0000100d0, 0xc0000c9000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:177 +0x69 net/http.(persistConn).Read(0xc0000806c0, 0xc0000c9000, 0x1000, 0x1000, 0xc000047c88, 0x406845, 0xc000060a20) /usr/local/go/src/net/http/transport.go:1524 +0x7b bufio.(Reader).fill(0xc000057020) /usr/local/go/src/bufio/bufio.go:100 +0x10f bufio.(Reader).Peek(0xc000057020, 0x1, 0x0, 0x0, 0x1, 0xc000060900, 0x0) /usr/local/go/src/bufio/bufio.go:138 +0x4f net/http.(persistConn).readLoop(0xc0000806c0) /usr/local/go/src/net/http/transport.go:1677 +0x1a3 created by net/http.(Transport).dialConn /usr/local/go/src/net/http/transport.go:1357 +0xae8 goroutine 22 [IO wait]: internal/poll.runtime_pollWait(0x7f199e878e38, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:182 +0x56 internal/poll.(pollDesc).wait(0xc0000ca918, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc0000ca900, 0xc000115000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b net.(netFD).Read(0xc0000ca900, 0xc000115000, 0x1000, 0x1000, 0x1, 0x42f01c, 0xc000049b88) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc0000100d8, 0xc000115000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:177 +0x69 net/http.(persistConn).Read(0xc0000807e0, 0xc000115000, 0x1000, 0x1000, 0xc000049c88, 0x406845, 0xc000060c00) /usr/local/go/src/net/http/transport.go:1524 +0x7b bufio.(Reader).fill(0xc0000570e0) /usr/local/go/src/bufio/bufio.go:100 +0x10f bufio.(Reader).Peek(0xc0000570e0, 0x1, 0x0, 0x0, 0x1, 0xc000060b00, 0x0) /usr/local/go/src/bufio/bufio.go:138 +0x4f net/http.(persistConn).readLoop(0xc0000807e0) /usr/local/go/src/net/http/transport.go:1677 +0x1a3 created by net/http.(Transport).dialConn /usr/local/go/src/net/http/transport.go:1357 +0xae8 goroutine 20 [select]: net/http.(persistConn).writeLoop(0xc0000806c0) /usr/local/go/src/net/http/transport.go:1976 +0x113 created by net/http.(Transport).dialConn /usr/local/go/src/net/http/transport.go:1358 +0xb0d goroutine 23 [select]: net/http.(persistConn).writeLoop(0xc0000807e0) /usr/local/go/src/net/http/transport.go:1976 +0x113 created by net/http.(Transport).dialConn /usr/local/go/src/net/http/transport.go:1358 +0xb0d

tomnomnom commented 5 years ago

Hey! Thanks for raising an issue!

Is there any chance you're able to provide me with the contents of your domains.txt to help me debug? Or perhaps a single domain that's causing the issue?

Thanks!

joeldeleep commented 5 years ago

https://raw.githubusercontent.com/nahamsec/SundayStreams/master/alive.txt , it exited with the error i mentioned in my server with 1gb ram , but it was stuck for about 2 minutes at my local system and resumed , i think it is the problem with ram , my local system was running with 8gb ram , not sure about it