search

lukasmartinelli / pgfutter

Import CSV and JSON into PostgreSQL the easy way
MIT License
1.33k stars 128 forks source link

fatal error: MSpanList_Insert #42

Open arjunbanker opened 7 years ago

arjunbanker commented 7 years ago

using https://fallingfruit.org/locations.csv.bz2

./pgfutter csv locations.csv

64.14 MB / 293.78 MB [==========================>---------------------------------------------------------------------------------------------] 21.83% 12sfailed MSpanList_Insert 0x5e50a8 0x11e98c9a501b0 0x0 fatal error: MSpanList_Insert

runtime stack: runtime.MSpanList_Insert(0x578850, 0x5e50a8) /goroot/src/runtime/mheap.c:692 +0x8f runtime.MHeap_Free(0x5787e0, 0x5e50a8, 0x0) /goroot/src/runtime/mheap.c:500 +0x5b runtime.MCentral_FreeSpan(0x580438, 0x5e50a8, 0x20, 0xc208090000, 0xc208091f00, 0x0, 0x64) /goroot/src/runtime/mcentral.c:181 +0x1bb runtime.MSpan_Sweep(0x5e50a8, 0xa2900000a00, 0x1) /goroot/src/runtime/mgc0.c:1099 +0x477 runtime.MHeap_Alloc(0x5787e0, 0x1, 0x10000000008, 0x15159) /goroot/src/runtime/mheap.c:240 +0x66 runtime.MCentral_CacheSpan(0x57fa18, 0xc2080ec708) /goroot/src/runtime/mcentral.c:85 +0x167 runtime.MCache_Refill(0x5e0000, 0x8, 0x10005e6490) /goroot/src/runtime/mcache.c:90 +0xa0

goroutine 1 [running]: runtime.switchtoM() /goroot/src/runtime/asm_amd64.s:198 fp=0xc2080ec7c8 sp=0xc2080ec7c0 runtime.mallocgc(0x70, 0x0, 0x3, 0x0) /goroot/src/runtime/malloc.go:178 +0x849 fp=0xc2080ec878 sp=0xc2080ec7c8 runtime.rawstring(0x6c, 0x0, 0x0, 0x0, 0x0, 0x0) /goroot/src/runtime/string.go:195 +0x93 fp=0xc2080ec8a8 sp=0xc2080ec878 runtime.slicebytetostring(0xc208011200, 0x6c, 0x81f, 0x0, 0x0) /goroot/src/runtime/string.go:69 +0x4b fp=0xc2080ec938 sp=0xc2080ec8a8 encoding/csv.(Reader).parseRecord(0xc2080383c0, 0xc208123400, 0x5, 0x8, 0x0, 0x0) /goroot/src/encoding/csv/reader.go:235 +0x449 fp=0xc2080ec9f8 sp=0xc2080ec938 encoding/csv.(Reader).Read(0xc2080383c0, 0x0, 0x0, 0x0, 0x0, 0x0) /goroot/src/encoding/csv/reader.go:136 +0x5e fp=0xc2080eca60 sp=0xc2080ec9f8 main.copyCSVRows(0xc208031e10, 0xc2080383c0, 0x0, 0xc208031795, 0x1, 0xc2080c6000, 0x10, 0x10, 0x0, 0x0, ...) /usr/src/pgfutter/csv.go:57 +0xa9 fp=0xc2080ecc08 sp=0xc2080eca60 main.importCSV(0x7fff5fbffb91, 0xd, 0xc2080e62a0, 0x68, 0xc2080315b6, 0x6, 0x7fff5fbffb91, 0x9, 0x0, 0x571590, ...) /usr/src/pgfutter/csv.go:144 +0x530 fp=0xc2080ece40 sp=0xc2080ecc08 main.func·003(0xc208056a00) /usr/src/pgfutter/pgfutter.go:172 +0x332 fp=0xc2080ecf60 sp=0xc2080ece40 runtime.call16(0x440268, 0xc2080340b0, 0x800000008) /goroot/src/runtime/asm_amd64.s:401 +0x45 fp=0xc2080ecf78 sp=0xc2080ecf60 reflect.Value.call(0x29e880, 0x440268, 0x13, 0x382790, 0x4, 0xc2080ed3d0, 0x1, 0x1, 0x0, 0x0, ...) /goroot/src/reflect/value.go:419 +0x10e5 fp=0xc2080ed290 sp=0xc2080ecf78 reflect.Value.Call(0x29e880, 0x440268, 0x13, 0xc2080ed3d0, 0x1, 0x1, 0x0, 0x0, 0x0) /goroot/src/reflect/value.go:296 +0xbc fp=0xc2080ed2f0 sp=0xc2080ed290 github.com/codegangsta/cli.HandleAction(0x29e880, 0x440268, 0xc208056a00, 0x0, 0x0) /gopath/src/github.com/codegangsta/cli/app.go:487 +0x2b1 fp=0xc2080ed420 sp=0xc2080ed2f0 github.com/codegangsta/cli.Command.Run(0x38eb70, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3dfcf0, 0x18, 0x0, ...) /gopath/src/github.com/codegangsta/cli/command.go:191 +0x10a5 fp=0xc2080ed998 sp=0xc2080ed420 github.com/codegangsta/cli.(*App).Run(0xc2080e8000, 0xc20800a000, 0x7, 0x7, 0x0, 0x0) /gopath/src/github.com/codegangsta/cli/app.go:240 +0xa76 fp=0xc2080edc10 sp=0xc2080ed998 main.main() /usr/src/pgfutter/pgfutter.go:178 +0xcd3 fp=0xc2080edf98 sp=0xc2080edc10 runtime.main() /goroot/src/runtime/proc.go:63 +0xf3 fp=0xc2080edfe0 sp=0xc2080edf98 runtime.goexit() /goroot/src/runtime/asm_amd64.s:2232 +0x1 fp=0xc2080edfe8 sp=0xc2080edfe0

goroutine 5 [syscall]: os/signal.loop() /goroot/src/os/signal/signal_unix.go:21 +0x1f created by os/signal.init·1 /goroot/src/os/signal/signal_unix.go:27 +0x35

goroutine 6 [chan receive]: database/sql.(*DB).connectionOpener(0xc208039860) /goroot/src/database/sql/sql.go:589 +0x4c created by database/sql.Open /goroot/src/database/sql/sql.go:452 +0x31c

goroutine 9 [IO wait]: net.(pollDesc).Wait(0xc2080e6370, 0x72, 0x0, 0x0) /goroot/src/net/fd_poll_runtime.go:84 +0x47 net.(pollDesc).WaitRead(0xc2080e6370, 0x0, 0x0) /goroot/src/net/fd_poll_runtime.go:89 +0x43 net.(netFD).Read(0xc2080e6310, 0xc20810e000, 0x1000, 0x1000, 0x0, 0x5f1cc0, 0xc208144b50) /goroot/src/net/fd_unix.go:242 +0x40f net.(conn).Read(0xc208034108, 0xc20810e000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /goroot/src/net/net.go:121 +0xdc bufio.(Reader).fill(0xc208092840) /goroot/src/bufio/bufio.go:97 +0x1ce bufio.(Reader).Read(0xc208092840, 0xc208058f20, 0x5, 0x200, 0x0, 0x0, 0x0) /goroot/src/bufio/bufio.go:174 +0x26c io.ReadAtLeast(0x5f3220, 0xc208092840, 0xc208058f20, 0x5, 0x200, 0x5, 0x0, 0x0, 0x0) /goroot/src/io/io.go:298 +0xf1 io.ReadFull(0x5f3220, 0xc208092840, 0xc208058f20, 0x5, 0x200, 0x0, 0x0, 0x0) /goroot/src/io/io.go:316 +0x6d github.com/lib/pq.(conn).recvMessage(0xc208058f00, 0xc20814a6c0, 0x0, 0x0, 0x0) /gopath/src/github.com/lib/pq/conn.go:892 +0x17a github.com/lib/pq.(copyin).resploop(0xc208036320) /gopath/src/github.com/lib/pq/copy.go:143 +0x55 created by github.com/lib/pq.(*conn).prepareCopyIn /gopath/src/github.com/lib/pq/copy.go:90 +0xae5

goroutine 10 [select]: github.com/cheggaaa/pb.(ProgressBar).refresher(0xc208056000) /gopath/src/github.com/cheggaaa/pb/pb.go:423 +0xff created by github.com/cheggaaa/pb.(ProgressBar).Start /gopath/src/github.com/cheggaaa/pb/pb.go:123 +0xcd

arjunbanker commented 7 years ago

fwiw, after the empty table was created i manually tried to load the csv using psql and it worked

querytree=# \copy import.locations FROM 'locations.csv' DELIMITER ',' CSV COPY 1267404

Ruin0x11 commented 7 years ago

Also have a similar error, using the data here:

> pgfutter --table bb csv -d "|" all_billboard_data.txt                                                       
 8.95 MB / 22.54 MB [======================================================================>----------------------------------------------------------------------------------------------------------]  39.68% 1sfailed MSpanList_Insert 0x5e6a40 0x9c471a3df6c 0x0
fatal error: MSpanList_Insert

runtime stack:
runtime.throw(0x552fab)
    /goroot/src/runtime/panic.go:491 +0xad fp=0xc20802fca0 sp=0xc20802fc70
runtime.MSpanList_Insert(0x578850, 0x5e6a40)
    /goroot/src/runtime/mheap.c:692 +0x8f fp=0xc20802fcc8 sp=0xc20802fca0
MHeap_FreeSpanLocked(0x5787e0, 0x5e6a40, 0x570101)
    /goroot/src/runtime/mheap.c:581 +0x147 fp=0xc20802fd08 sp=0xc20802fcc8
mheap_free(0x5787e0, 0x5e6a40, 0xc200000000)
    /goroot/src/runtime/mheap.c:476 +0x135 fp=0xc20802fd28 sp=0xc20802fd08
runtime.MHeap_Free(0x5787e0, 0x5e6a40, 0x0)
    /goroot/src/runtime/mheap.c:500 +0x5b fp=0xc20802fd50 sp=0xc20802fd28
runtime.MCentral_FreeSpan(0x57fe98, 0x5e6a40, 0x2e, 0xc20816a000, 0xc20816bef0, 0x0, 0x64)
    /goroot/src/runtime/mcentral.c:181 +0x1bb fp=0xc20802fd78 sp=0xc20802fd50
runtime.MSpan_Sweep(0x5e6a40, 0x32700000300, 0x1)
    /goroot/src/runtime/mgc0.c:1099 +0x477 fp=0xc20802fe48 sp=0xc20802fd78
runtime.sweepone(0x0)
    /goroot/src/runtime/mgc0.c:1148 +0x148 fp=0xc20802fe80 sp=0xc20802fe48
gc(0xc20802ffb8)
    /goroot/src/runtime/mgc0.c:1407 +0xbd fp=0xc20802ff98 sp=0xc20802fe80
runtime.gc_m()
    /goroot/src/runtime/mgc0.c:1371 +0xe0 fp=0xc20802ffd0 sp=0xc20802ff98
runtime.onM(0xc208010000)
    /goroot/src/runtime/asm_amd64.s:257 +0x68 fp=0xc20802ffd8 sp=0xc20802ffd0
runtime.mstart()
    /goroot/src/runtime/proc.c:818 fp=0xc20802ffe0 sp=0xc20802ffd8
kaplanov commented 7 years ago

same error for me. Its a shame because otherwise library worked like charm

saschagoebel commented 6 years ago

Same error here. Problem occurs on macOS Sierra 10.12.6

gmile commented 6 years ago

Same error here. What can I do to help aid this?

andrewking0207 commented 6 years ago

Same here.

quinlanj commented 6 years ago

I fixed this by upgrading go and building pgfutter from source. (ie) brew upgrade go then go get github.com/lukasmartinelli/pgfutter.

This error is due to older versions of go not playing well with macOS Sierra https://github.com/golang/go/issues/17335

lukasmartinelli commented 6 years ago

Maybe fixed with https://github.com/lukasmartinelli/pgfutter/releases/tag/v1.2?