x/net/idna: ToASCII bug causes memory and cpu leak

[ma6174]

What version of Go are you using (go version)?

go version go1.9.1 darwin/amd64

Does this issue reproduce with the latest release?

Yes

What operating system and processor architecture are you using (go env)?

GOARCH="amd64"
GOBIN=""
GOEXE=""
GOHOSTARCH="amd64"
GOHOSTOS="darwin"
GOOS="darwin"
GOPATH="/Users/ma6174/gopath"
GORACE=""
GOROOT="/Users/ma6174/go1.9.1"
GOTOOLDIR="/Users/ma6174/go1.9.1/pkg/tool/darwin_amd64"
GCCGO="gccgo"
CC="clang"
GOGCCFLAGS="-fPIC -m64 -pthread -fno-caret-diagnostics -Qunused-arguments -fmessage-length=0 -fdebug-prefix-map=/var/folders/zz/wrwm9_1n7rq094rkczx_pm280000gn/T/go-build805739314=/tmp/go-build -gno-record-gcc-switches -fno-common"
CXX="clang++"
CGO_ENABLED="1"
CGO_CFLAGS="-g -O2"
CGO_CPPFLAGS=""
CGO_CXXFLAGS="-g -O2"
CGO_FFLAGS="-g -O2"
CGO_LDFLAGS="-g -O2"
PKG_CONFIG="pkg-config"

What did you do?

package main

import (
    "fmt"
    "net/http"
    "runtime"
)

func main() {
    fmt.Println("start", runtime.Version(), runtime.GOOS, runtime.GOARCH)
    u := "http://\xed/"
    _, err := http.Get(u)
    fmt.Printf("==> %v", err)
}

What did you expect to see?

program exit with an error of no such host or invalid domain name

in go1.8

start go1.8.1 linux amd64
==> Get http://%ED/: dial tcp: lookup xn--zn7c on 192.168.1.1:53: no such host

in go1.7 or before

start go1.7.5 linux amd64
==> Get http://%ED/: dial tcp: lookup �: invalid domain name

What did you see instead?

program fataled until oom, and more than 100% cpu usage.

start go1.9.1 linux amd64
fatal error: runtime: out of memory

runtime stack:
runtime.throw(0x66cb10, 0x16)
    /Users/ma6174/go1.9.1/src/runtime/panic.go:605 +0x95
runtime.sysMap(0xc4247c0000, 0x2970000, 0xc42050a400, 0x7d2df8)
    /Users/ma6174/go1.9.1/src/runtime/mem_linux.go:216 +0x1d0
runtime.(*mheap).sysAlloc(0x7b9da0, 0x2970000, 0xc4204f3e20)
    /Users/ma6174/go1.9.1/src/runtime/malloc.go:470 +0xd7
runtime.(*mheap).grow(0x7b9da0, 0x14b2, 0x0)
    /Users/ma6174/go1.9.1/src/runtime/mheap.go:887 +0x60
runtime.(*mheap).allocSpanLocked(0x7b9da0, 0x14b2, 0x7d2e08, 0xc4204f3eb0)
    /Users/ma6174/go1.9.1/src/runtime/mheap.go:800 +0x334
runtime.(*mheap).alloc_m(0x7b9da0, 0x14b2, 0xffffffffffff0101, 0xc42050a480)
    /Users/ma6174/go1.9.1/src/runtime/mheap.go:666 +0x118
runtime.(*mheap).alloc.func1()
    /Users/ma6174/go1.9.1/src/runtime/mheap.go:733 +0x4d
runtime.systemstack(0xc4204f3f08)
    /Users/ma6174/go1.9.1/src/runtime/asm_amd64.s:360 +0xab
runtime.(*mheap).alloc(0x7b9da0, 0x14b2, 0x101, 0xc420000180)
    /Users/ma6174/go1.9.1/src/runtime/mheap.go:732 +0xa1
runtime.largeAlloc(0x2964000, 0x450100, 0x10000c420001d28)
    /Users/ma6174/go1.9.1/src/runtime/malloc.go:827 +0x98
runtime.mallocgc.func1()
    /Users/ma6174/go1.9.1/src/runtime/malloc.go:722 +0x46
runtime.systemstack(0xc42050a4b8)
    /Users/ma6174/go1.9.1/src/runtime/asm_amd64.s:344 +0x79
runtime.mstart()
    /Users/ma6174/go1.9.1/src/runtime/proc.go:1125

goroutine 1 [running]:
runtime.systemstack_switch()
    /Users/ma6174/go1.9.1/src/runtime/asm_amd64.s:298 fp=0xc4200593a8 sp=0xc4200593a0 pc=0x454370
runtime.mallocgc(0x2964000, 0x0, 0x1a7c000, 0xc421fac000)
    /Users/ma6174/go1.9.1/src/runtime/malloc.go:721 +0x7b8 fp=0xc420059450 sp=0xc4200593a8 pc=0x410608
runtime.growslice(0x610840, 0xc421fac000, 0x211bfff, 0x211c000, 0x211c002, 0xc421fac000, 0x1a7c000, 0x211c000)
    /Users/ma6174/go1.9.1/src/runtime/slice.go:140 +0x20c fp=0xc4200594b8 sp=0xc420059450 pc=0x44057c
vendor/golang_org/x/net/idna.validateAndMap(0x7a9720, 0x6693b8, 0x1, 0x7f89364e4728, 0x0, 0x30, 0x7b2880)
    /Users/ma6174/go1.9.1/src/vendor/golang_org/x/net/idna/idna.go:423 +0x480 fp=0xc420059580 sp=0xc4200594b8 pc=0x59e0b0
vendor/golang_org/x/net/idna.(*Profile).process(0x7a9720, 0x6693b8, 0x1, 0x580001, 0x6693b8, 0x1, 0xc42008ec3a, 0xffffffffffffffff)
    /Users/ma6174/go1.9.1/src/vendor/golang_org/x/net/idna/idna.go:293 +0xb0a fp=0xc420059680 sp=0xc420059580 pc=0x59dbda
vendor/golang_org/x/net/idna.(*Profile).ToASCII(0x7a9720, 0x6693b8, 0x1, 0x6693b8, 0x1, 0x6693b8, 0x1)
    /Users/ma6174/go1.9.1/src/vendor/golang_org/x/net/idna/idna.go:189 +0x44 fp=0xc4200596d0 sp=0xc420059680 pc=0x59cdb4
net/http.idnaASCII(0x6693b8, 0x1, 0x1, 0x580028, 0x6693b8, 0x1)
    /Users/ma6174/go1.9.1/src/net/http/request.go:650 +0x96 fp=0xc420059718 sp=0xc4200596d0 pc=0x5c65c6
net/http.canonicalAddr(0xc420116000, 0x0, 0x1)
    /Users/ma6174/go1.9.1/src/net/http/transport.go:2081 +0x51 fp=0xc420059768 sp=0xc420059718 pc=0x5d8ab1
net/http.(*Transport).connectMethodForRequest(0x7ae100, 0xc42008ed20, 0x0, 0x6693b1, 0x4, 0x0, 0x0, 0x100, 0xc0)
    /Users/ma6174/go1.9.1/src/net/http/transport.go:614 +0xe8 fp=0xc420059820 sp=0xc420059768 pc=0x5cf5f8
net/http.(*Transport).RoundTrip(0x7ae100, 0xc420118000, 0x7ae100, 0x0, 0x0)
    /Users/ma6174/go1.9.1/src/net/http/transport.go:390 +0x5dd fp=0xc420059a88 sp=0xc420059820 pc=0x5ce19d
net/http.send(0xc420118000, 0x77c9c0, 0x7ae100, 0x0, 0x0, 0x0, 0xc42000c058, 0x100, 0xc420059c58, 0x1)
    /Users/ma6174/go1.9.1/src/net/http/client.go:249 +0x1a9 fp=0xc420059be8 sp=0xc420059a88 pc=0x5a5fd9
net/http.(*Client).send(0x7b1c00, 0xc420118000, 0x0, 0x0, 0x0, 0xc42000c058, 0x0, 0x1, 0xc420059ce0)
    /Users/ma6174/go1.9.1/src/net/http/client.go:173 +0xfd fp=0xc420059c68 sp=0xc420059be8 pc=0x5a5c6d
net/http.(*Client).Do(0x7b1c00, 0xc420118000, 0x6693b1, 0x9, 0x0)
    /Users/ma6174/go1.9.1/src/net/http/client.go:602 +0x28d fp=0xc420059e38 sp=0xc420059c68 pc=0x5a737d
net/http.(*Client).Get(0x7b1c00, 0x6693b1, 0x9, 0x1a, 0x0, 0x0)
    /Users/ma6174/go1.9.1/src/net/http/client.go:393 +0xa7 fp=0xc420059e90 sp=0xc420059e38 pc=0x5a6e97
net/http.Get(0x6693b1, 0x9, 0x4, 0x1a, 0x0)
    /Users/ma6174/go1.9.1/src/net/http/client.go:367 +0x41 fp=0xc420059ed0 sp=0xc420059e90 pc=0x5a6dc1
main.main()
    /Users/ma6174/url_get.go:12 +0x10c fp=0xc420059f80 sp=0xc420059ed0 pc=0x5e715c
runtime.main()
    /Users/ma6174/go1.9.1/src/runtime/proc.go:185 +0x20d fp=0xc420059fe0 sp=0xc420059f80 pc=0x42b11d
runtime.goexit()
    /Users/ma6174/go1.9.1/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc420059fe8 sp=0xc420059fe0 pc=0x456ef1

the problem code is:

https://github.com/golang/go/blob/release-branch.go1.9/src/vendor/golang_org/x/net/idna/tables.go#L484-L487

https://github.com/golang/go/blob/release-branch.go1.9/src/vendor/golang_org/x/net/idna/idna.go#L401-L404

https://github.com/golang/go/blob/release-branch.go1.9/src/vendor/golang_org/x/net/idna/idna.go#L423

len(s) == 1 and lookupString() return 0, 0; in validateAndMap, sz=0, i+=0, i not changed, so the for loop cannot break, this is a cpu leak; v=0 hit select unknown branch, call b = append(b, "\ufffd"...), this is a memory leak.

[odeke-em]

/cc @mpvl

[ghost]

Reproduces on linux/amd64 (x/net/idna master).

package idna

import (
    "testing"

    "golang.org/x/net/idna"
)

func TestToASCII(t *testing.T) {
    idna.Lookup.ToASCII("\xed")
}

[SunRunAway]

any updates here?

[ALTree]

So this is an issue in the x/net/idna master; setting the milestone to 1.10 since the faulty idna version is also currently vendored in the 1.9 tree.

[corona10]

@ALTree Can I take a look this issue?

[ALTree]

@corona10 sure! I'm not working on this.

[gopherbot]

Change https://golang.org/cl/73610 mentions this issue: idna: Fix ToASCII bug causes memory and cpu leak

[corona10]

@ALTree I submitted the patch PTAL!

[gopherbot]

Change https://golang.org/cl/73630 mentions this issue: idna: Fix ToASCII bug causes memory and cpu leak

[gopherbot]

Change https://golang.org/cl/73730 mentions this issue: idna: Fix functions can causes memory and cpu leak.