search

omniscale / imposm3

Imposm imports OpenStreetMap data into PostGIS
http://imposm.org/docs/imposm3/latest/
Apache License 2.0
723 stars 158 forks source link

fatal error: runtime: out of memory when updating a continent #237

Closed NyakudyaA closed 3 years ago

NyakudyaA commented 4 years ago

Problem

Whilst running an update for a region - north america I experience the following

fatal error: runtime: out of memory

runtime stack:
runtime.throw(0x9945f2, 0x16)
    /usr/local/go/src/runtime/panic.go:616 +0x81
runtime.sysMap(0xc6579f0000, 0x1a9000000, 0xffffff00, 0xe91038)
    /usr/local/go/src/runtime/mem_linux.go:216 +0x20a
runtime.(*mheap).sysAlloc(0xe77c20, 0x1a9000000, 0x0)
    /usr/local/go/src/runtime/malloc.go:470 +0xd4
runtime.(*mheap).grow(0xe77c20, 0xd4800, 0x0)
    /usr/local/go/src/runtime/mheap.go:907 +0x60
runtime.(*mheap).allocSpanLocked(0xe77c20, 0xd4800, 0xe91048, 0x747808ea18d27800)
    /usr/local/go/src/runtime/mheap.go:820 +0x301
runtime.(*mheap).alloc_m(0xe77c20, 0xd4800, 0x100, 0x7fab0375b4e3)
    /usr/local/go/src/runtime/mheap.go:686 +0x118
runtime.(*mheap).alloc.func1()
    /usr/local/go/src/runtime/mheap.go:753 +0x4d
runtime.(*mheap).alloc(0xe77c20, 0xd4800, 0x10100, 0xc420240738)
    /usr/local/go/src/runtime/mheap.go:752 +0x8a
runtime.largeAlloc(0x1a9000000, 0x1, 0x1)
    /usr/local/go/src/runtime/malloc.go:826 +0x94
runtime.mallocgc.func1()
    /usr/local/go/src/runtime/malloc.go:721 +0x46
runtime.systemstack(0x0)
    /usr/local/go/src/runtime/asm_amd64.s:409 +0x79
runtime.mstart()
    /usr/local/go/src/runtime/proc.go:1175

goroutine 1 [running]:
runtime.systemstack_switch()
    /usr/local/go/src/runtime/asm_amd64.s:363 fp=0xc4202406c0 sp=0xc4202406b8 pc=0x45d680
runtime.mallocgc(0x1a9000000, 0x92cf80, 0x1, 0xc55bf796d8)
    /usr/local/go/src/runtime/malloc.go:720 +0x8a2 fp=0xc420240760 sp=0xc4202406c0 pc=0x416c32
runtime.newarray(0x92cf80, 0x1100000, 0xc44b8b9ea0)
    /usr/local/go/src/runtime/malloc.go:855 +0x6a fp=0xc420240790 sp=0xc420240760 pc=0x416fba
runtime.makeBucketArray(0x8ec060, 0x9b0d18, 0xc420240820, 0x18)
    /usr/local/go/src/runtime/hashmap.go:881 +0xe2 fp=0xc4202407c8 sp=0xc420240790 pc=0x40f212
runtime.hashGrow(0x8ec060, 0xc420252030)
    /usr/local/go/src/runtime/hashmap.go:905 +0x80 fp=0xc420240818 sp=0xc4202407c8 pc=0x40f390
runtime.mapassign_fast64(0x8ec060, 0xc420252030, 0x1abf26f78, 0x0)
    /usr/local/go/src/runtime/hashmap_fast.go:582 +0x162 fp=0xc420240860 sp=0xc420240818 pc=0x411672
github.com/omniscale/imposm3/update.(*Deleter).deleteNode(0xc4202415e8, 0x1abf26f78, 0x1abf26f77, 0xc420240940)
    /go/src/github.com/omniscale/imposm3/update/deleter.go:215 +0x17a fp=0xc420240910 sp=0xc420240860 pc=0x86015a
github.com/omniscale/imposm3/update.(*Deleter).Delete(0xc4202415e8, 0x1, 0xc57c03cae0, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/omniscale/imposm3/update/deleter.go:271 +0x92 fp=0xc4202409a8 sp=0xc420240910 pc=0x860422
github.com/omniscale/imposm3/update.Update(0x7ffe5b7c9d63, 0x20, 0x7ffe5b7c9cb8, 0xb, 0x7ffe5b7c9d24, 0xe, 0x7ffe5b7c9d3c, 0x1a, 0x10e6, 0x7ffe5b7c9d8d, ...)
    /go/src/github.com/omniscale/imposm3/update/process.go:256 +0x138d fp=0xc420241750 sp=0xc4202409a8 pc=0x86289d
github.com/omniscale/imposm3/update.Diff(0x7ffe5b7c9d63, 0x20, 0x7ffe5b7c9cb8, 0xb, 0x7ffe5b7c9d24, 0xe, 0x7ffe5b7c9d3c, 0x1a, 0x10e6, 0x7ffe5b7c9d8d, ...)
    /go/src/github.com/omniscale/imposm3/update/process.go:75 +0x37e fp=0xc4202419d8 sp=0xc420241750 pc=0x86113e
main.Main(0x9b0458)
    /go/src/github.com/omniscale/imposm3/cmd/imposm/main.go:55 +0x366 fp=0xc420241f70 sp=0xc4202419d8 pc=0x8698f6
main.main()
    /go/src/github.com/omniscale/imposm3/cmd/imposm/main.go:77 +0x2d fp=0xc420241f88 sp=0xc420241f70 pc=0x869dad
runtime.main()
    /usr/local/go/src/runtime/proc.go:198 +0x212 fp=0xc420241fe0 sp=0xc420241f88 pc=0x432912
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:2361 +0x1 fp=0xc420241fe8 sp=0xc420241fe0 pc=0x460211

goroutine 6 [syscall, 103 minutes]:
os/signal.signal_recv(0xc42003bfc8)
    /usr/local/go/src/runtime/sigqueue.go:139 +0xa6
os/signal.loop()
    /usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.0
    /usr/local/go/src/os/signal/signal_unix.go:28 +0x41

goroutine 34 [select, 103 minutes]:
database/sql.(*DB).connectionOpener(0xc42009f360, 0x9f4ac0, 0xc4201800c0)
    /usr/local/go/src/database/sql/sql.go:935 +0x119
created by database/sql.OpenDB
    /usr/local/go/src/database/sql/sql.go:634 +0x178

goroutine 35 [select, 103 minutes]:
database/sql.(*DB).connectionResetter(0xc42009f360, 0x9f4ac0, 0xc4201800c0)
    /usr/local/go/src/database/sql/sql.go:948 +0x12a
created by database/sql.OpenDB
    /usr/local/go/src/database/sql/sql.go:635 +0x1ae

goroutine 40 [chan receive, 103 minutes]:
database/sql.(*Tx).awaitDone(0xc4200c6a00)
    /usr/local/go/src/database/sql/sql.go:1839 +0x4b
created by database/sql.(*DB).beginDC
    /usr/local/go/src/database/sql/sql.go:1595 +0x1ce

goroutine 19 [select, 2 minutes]:
github.com/omniscale/imposm3/stats.(*Statistics).loop(0xc4202d8260)
    /go/src/github.com/omniscale/imposm3/stats/stats.go:109 +0x11c
created by github.com/omniscale/imposm3/stats.NewStatsReporter
    /go/src/github.com/omniscale/imposm3/stats/stats.go:89 +0xa8

goroutine 20 [chan receive, 103 minutes]:
github.com/omniscale/imposm3/writer.(*RelationWriter).loop(0xc420098210)
    /go/src/github.com/omniscale/imposm3/writer/relations.go:78 +0x289
created by github.com/omniscale/imposm3/writer.(*OsmElemWriter).Start
    /go/src/github.com/omniscale/imposm3/writer/writer.go:52 +0x82

goroutine 21 [chan receive, 103 minutes]:
github.com/omniscale/imposm3/writer.(*WayWriter).loop(0xc4203286e0)
    /go/src/github.com/omniscale/imposm3/writer/ways.go:71 +0xcb
created by github.com/omniscale/imposm3/writer.(*OsmElemWriter).Start
    /go/src/github.com/omniscale/imposm3/writer/writer.go:52 +0x82

goroutine 22 [chan receive, 103 minutes]:
github.com/omniscale/imposm3/writer.(*NodeWriter).loop(0xc42032c200)
    /go/src/github.com/omniscale/imposm3/writer/nodes.go:51 +0xc5
created by github.com/omniscale/imposm3/writer.(*OsmElemWriter).Start
    /go/src/github.com/omniscale/imposm3/writer/writer.go:52 +0x82

goroutine 23 [select]:
github.com/omniscale/imposm3/vendor/github.com/omniscale/go-osm/parser/diff.(*Parser).Parse(0xc42006a280, 0x9f4ac0, 0xc420194440, 0x0, 0x0)
    /go/src/github.com/omniscale/imposm3/vendor/github.com/omniscale/go-osm/parser/diff/parser.go:220 +0x3e1
github.com/omniscale/imposm3/update.Update.func1(0xc4202ce1e0, 0xc42006a280, 0x9f4ac0, 0xc420194440)
    /go/src/github.com/omniscale/imposm3/update/process.go:236 +0x3f
created by github.com/omniscale/imposm3/update.Update
    /go/src/github.com/omniscale/imposm3/update/process.go:235 +0x12a5
39
An error occured in imposm with a diff.Detect SRID: 4326
olt commented 4 years ago

Your server likely ran out of memory. Make sure your server is has enough RAM.

NyakudyaA commented 4 years ago

Your server likely ran out of memory. Make sure your server is has enough RAM.

Unfortunately, my server has plenty of memory. The update should not use a lot of memory more than the initial import

NyakudyaA commented 3 years ago

Not valid anymore, managed to fine tune some PostgreSQL conf