search

cockroachdb / cockroach

CockroachDB - the open source, cloud-native distributed SQL database.
https://www.cockroachlabs.com
Other
29.52k stars 3.7k forks source link

security: TestUseCerts failed #119052

Open cockroach-teamcity opened 5 months ago

cockroach-teamcity commented 5 months ago

security.TestUseCerts failed on master @ 2420e5c067c23eb5323f298a5f11d6d86d1d45b1:

        net/http.(*Transport).dialConnFor(0x6590dd0?, 0xc0047b9a20)
            GOROOT/src/net/http/transport.go:1467 +0x9f
        created by net/http.(*Transport).queueForDial in goroutine 10540
            GOROOT/src/net/http/transport.go:1436 +0x3cb
        Leaked goroutine: goroutine 12788 [syscall]:
        syscall.Syscall(0x9b23c68?, 0x45d245?, 0x800000?, 0x7ffff800000?)
            GOROOT/src/syscall/syscall_linux.go:69 +0x25
        syscall.read(0xc002e881e0?, {0xc003254c00?, 0xc004c84ac8?, 0x0?})
            GOROOT/src/syscall/zsyscall_linux_amd64.go:721 +0x38
        syscall.Read(...)
            GOROOT/src/syscall/syscall_unix.go:181
        internal/poll.ignoringEINTRIO(...)
            GOROOT/src/internal/poll/fd_unix.go:736
        internal/poll.(*FD).Read(0xc002e881e0, {0xc003254c00, 0x591, 0x591})
            GOROOT/src/internal/poll/fd_unix.go:160 +0x2ae
        os.(*File).read(...)
            GOROOT/src/os/file_posix.go:29
        os.(*File).Read(0xc001ad74d0, {0xc003254c00?, 0x0?, 0x51ca6b7?})
            GOROOT/src/os/file.go:118 +0x52
        os.ReadFile({0xc002a9e140?, 0x51ca6b7?})
            GOROOT/src/os/file.go:744 +0x1d5
        crypto/x509.loadSystemRoots()
            GOROOT/src/crypto/x509/root_unix.go:70 +0x3d4
        crypto/x509.initSystemRoots()
            GOROOT/src/crypto/x509/root.go:30 +0x5c
        sync.(*Once).doSlow(0x454f8b?, 0xc0024f34c8?)
            GOROOT/src/sync/once.go:74 +0xbf
        sync.(*Once).Do(...)
            GOROOT/src/sync/once.go:65
        crypto/x509.systemRootsPool()
            GOROOT/src/crypto/x509/root.go:21 +0x45
        crypto/x509.(*Certificate).Verify(0xc0022d3700, {{0xc003c883f0, 0x9}, 0xc0014fd8f0, 0x0, {0xc169e25f0845e61f, 0x229c61f3e, 0x9add4e0}, {0x0, 0x0, ...}, ...})
            GOROOT/src/crypto/x509/verify.go:784 +0x132
        crypto/tls.(*Conn).verifyServerCertificate(0xc00309bc00, {0xc002dcf3b0, 0x1, 0x1})
            GOROOT/src/crypto/tls/handshake_client.go:993 +0x819
        crypto/tls.(*clientHandshakeStateTLS13).readServerCertificate(0xc002dffc08)
            GOROOT/src/crypto/tls/handshake_client_tls13.go:531 +0x2b3
        crypto/tls.(*clientHandshakeStateTLS13).handshake(0xc002dffc08)
            GOROOT/src/crypto/tls/handshake_client_tls13.go:96 +0x258
        crypto/tls.(*Conn).clientHandshake(0xc00309bc00, {0x65bd3a0?, 0xc005b430e0})
            GOROOT/src/crypto/tls/handshake_client.go:263 +0x59a
        crypto/tls.(*Conn).handshakeContext(0xc00309bc00, {0x65bd448, 0xc002d7d880})
            GOROOT/src/crypto/tls/conn.go:1552 +0x3d3
        crypto/tls.(*Conn).HandshakeContext(...)
            GOROOT/src/crypto/tls/conn.go:1492
        net/http.(*persistConn).addTLS.func2()
            GOROOT/src/net/http/transport.go:1555 +0x6d
        created by net/http.(*persistConn).addTLS in goroutine 12783
            GOROOT/src/net/http/transport.go:1551 +0x31b
--- FAIL: TestUseCerts (17.46s)

Parameters:

See also: How To Investigate a Go Test Failure (internal)

/cc @cockroachdb/prodsec @cockroachdb/server

This test on roachdash | Improve this report!

Jira issue: CRDB-36214

cockroach-teamcity commented 4 months ago

security.TestUseCerts failed on master @ cc6ca026319024800395293b0fb18f05dd8eb50e:

        net/http.(*Transport).dialConnFor(0xc00238d680?, 0xc0045cb760)
            GOROOT/src/net/http/transport.go:1467 +0x9f
        created by net/http.(*Transport).queueForDial in goroutine 10008
            GOROOT/src/net/http/transport.go:1436 +0x3cb
        Leaked goroutine: goroutine 11639 [syscall]:
        syscall.Syscall(0x5?, 0x7?, 0x800000?, 0x7ffff800000?)
            GOROOT/src/syscall/syscall_linux.go:69 +0x25
        syscall.read(0xc0024de8a0?, {0xc001854d00?, 0xc004577ca8?, 0xc001833180?})
            GOROOT/src/syscall/zsyscall_linux_amd64.go:721 +0x38
        syscall.Read(...)
            GOROOT/src/syscall/syscall_unix.go:181
        internal/poll.ignoringEINTRIO(...)
            GOROOT/src/internal/poll/fd_unix.go:736
        internal/poll.(*FD).Read(0xc0024de8a0, {0xc001854d00, 0x82f, 0x82f})
            GOROOT/src/internal/poll/fd_unix.go:160 +0x2ae
        os.(*File).read(...)
            GOROOT/src/os/file_posix.go:29
        os.(*File).Read(0xc005bee0c0, {0xc001854d00?, 0x0?, 0x51c709e?})
            GOROOT/src/os/file.go:118 +0x52
        os.ReadFile({0xc002f50740?, 0x51c709e?})
            GOROOT/src/os/file.go:744 +0x1d5
        crypto/x509.loadSystemRoots()
            GOROOT/src/crypto/x509/root_unix.go:70 +0x3d4
        crypto/x509.initSystemRoots()
            GOROOT/src/crypto/x509/root.go:30 +0x5c
        sync.(*Once).doSlow(0x454f8b?, 0xc003f1d4c8?)
            GOROOT/src/sync/once.go:74 +0xbf
        sync.(*Once).Do(...)
            GOROOT/src/sync/once.go:65
        crypto/x509.systemRootsPool()
            GOROOT/src/crypto/x509/root.go:21 +0x45
        crypto/x509.(*Certificate).Verify(0xc00301e100, {{0xc003c305f0, 0x9}, 0xc004756a80, 0x0, {0xc16b8a571668bf28, 0x2267eeb12, 0x9ae63e0}, {0x0, 0x0, ...}, ...})
            GOROOT/src/crypto/x509/verify.go:784 +0x132
        crypto/tls.(*Conn).verifyServerCertificate(0xc0046e8e00, {0xc0047621c8, 0x1, 0x1})
            GOROOT/src/crypto/tls/handshake_client.go:993 +0x819
        crypto/tls.(*clientHandshakeStateTLS13).readServerCertificate(0xc001833c08)
            GOROOT/src/crypto/tls/handshake_client_tls13.go:531 +0x2b3
        crypto/tls.(*clientHandshakeStateTLS13).handshake(0xc001833c08)
            GOROOT/src/crypto/tls/handshake_client_tls13.go:96 +0x258
        crypto/tls.(*Conn).clientHandshake(0xc0046e8e00, {0x65c33e0?, 0xc004764000})
            GOROOT/src/crypto/tls/handshake_client.go:263 +0x59a
        crypto/tls.(*Conn).handshakeContext(0xc0046e8e00, {0x65c3488, 0xc00413f7a0})
            GOROOT/src/crypto/tls/conn.go:1552 +0x3d3
        crypto/tls.(*Conn).HandshakeContext(...)
            GOROOT/src/crypto/tls/conn.go:1492
        net/http.(*persistConn).addTLS.func2()
            GOROOT/src/net/http/transport.go:1555 +0x6d
        created by net/http.(*persistConn).addTLS in goroutine 11636
            GOROOT/src/net/http/transport.go:1551 +0x31b
--- FAIL: TestUseCerts (16.82s)

Parameters:

See also: How To Investigate a Go Test Failure (internal)

This test on roachdash | Improve this report!

BabuSrithar commented 4 months ago

The failure is not consistently reproducible.

The functionality itself is working fine. But in these 2 failure instances there were leaky goroutines found. We suspect that somehow the system is overloaded by other workloads and it just takes more than 30 secs to read files from disk, which causes the failure. What we were not sure is if engflow is running all 30 instances of the test concurrently on same environment or just sequential. While we are still investigating this, removing the release blocker on this.

cockroach-teamcity commented 4 months ago

security.TestUseCerts failed on master @ f4dc2b59541e11cf06e2b948c7c37bf1aed21868:

        net/http.(*Transport).dialConnFor(0xc0012e1b80, 0xc004537a20)
            GOROOT/src/net/http/transport.go:1485 +0xcd
        created by net/http.(*Transport).queueForDial in goroutine 9785
            GOROOT/src/net/http/transport.go:1449 +0x3c9
        Leaked goroutine: goroutine 11396 [syscall]:
        syscall.Syscall(0x0, 0x9, 0xc00284c700, 0x34c)
            GOROOT/src/syscall/syscall_linux.go:69 +0x25
        syscall.read(0xc0041301e0?, {0xc00284c700?, 0xc005dc4e08?, 0xc00264f210?})
            GOROOT/src/syscall/zsyscall_linux_amd64.go:736 +0x38
        syscall.Read(...)
            GOROOT/src/syscall/syscall_unix.go:181
        internal/poll.ignoringEINTRIO(...)
            GOROOT/src/internal/poll/fd_unix.go:736
        internal/poll.(*FD).Read(0xc0041301e0, {0xc00284c700, 0x34c, 0x34c})
            GOROOT/src/internal/poll/fd_unix.go:160 +0x2ae
        os.(*File).read(...)
            GOROOT/src/os/file_posix.go:29
        os.(*File).Read(0xc001c14070, {0xc00284c700?, 0x0?, 0x0?})
            GOROOT/src/os/file.go:118 +0x52
        os.ReadFile({0xc003ff6090?, 0x529cb32?})
            GOROOT/src/os/file.go:793 +0x167
        crypto/x509.loadSystemRoots()
            GOROOT/src/crypto/x509/root_unix.go:70 +0x396
        crypto/x509.initSystemRoots()
            GOROOT/src/crypto/x509/root.go:30 +0x5c
        sync.(*Once).doSlow(0xc002a6bf80?, 0x10?)
            GOROOT/src/sync/once.go:74 +0xc2
        sync.(*Once).Do(...)
            GOROOT/src/sync/once.go:65
        crypto/x509.systemRootsPool()
            GOROOT/src/crypto/x509/root.go:21 +0x45
        crypto/x509.(*Certificate).Verify(0xc0025b1b88, {{0xc00202ad60, 0x9}, 0xc002a6bef0, 0x0, {0xc172736e8921484e, 0x12de0d00b, 0x9c5f9c0}, {0x0, 0x0, ...}, ...})
            GOROOT/src/crypto/x509/verify.go:784 +0x15d
        crypto/tls.(*Conn).verifyServerCertificate(0xc002ec6a88, {0xc000e47230, 0x1, 0x1})
            GOROOT/src/crypto/tls/handshake_client.go:997 +0x819
        crypto/tls.(*clientHandshakeStateTLS13).readServerCertificate(0xc00264fbd0)
            GOROOT/src/crypto/tls/handshake_client_tls13.go:531 +0x273
        crypto/tls.(*clientHandshakeStateTLS13).handshake(0xc00264fbd0)
            GOROOT/src/crypto/tls/handshake_client_tls13.go:96 +0x29a
        crypto/tls.(*Conn).clientHandshake(0xc002ec6a88, {0x6700060, 0xc003318c80})
            GOROOT/src/crypto/tls/handshake_client.go:263 +0x594
        crypto/tls.(*Conn).handshakeContext(0xc002ec6a88, {0x6700108, 0xc002820700})
            GOROOT/src/crypto/tls/conn.go:1553 +0x3cb
        crypto/tls.(*Conn).HandshakeContext(...)
            GOROOT/src/crypto/tls/conn.go:1493
        net/http.(*persistConn).addTLS.func2()
            GOROOT/src/net/http/transport.go:1573 +0x6e
        created by net/http.(*persistConn).addTLS in goroutine 11392
            GOROOT/src/net/http/transport.go:1569 +0x309
--- FAIL: TestUseCerts (16.83s)

Parameters:

See also: How To Investigate a Go Test Failure (internal)

This test on roachdash | Improve this report!

cockroach-teamcity commented 3 months ago

security.TestUseCerts failed on master @ 7488e090daa588c4d7c0f828c8006bb9b13a90f6:

        net/http.(*Transport).dialConnFor(0xc001489900, 0xc000b73760)
            GOROOT/src/net/http/transport.go:1485 +0xcd
        created by net/http.(*Transport).queueForDial in goroutine 9975
            GOROOT/src/net/http/transport.go:1449 +0x3c9
        Leaked goroutine: goroutine 11667 [syscall]:
        syscall.Syscall(0x0, 0x7, 0xc002aa5000, 0x7f2)
            GOROOT/src/syscall/syscall_linux.go:69 +0x25
        syscall.read(0xc0029f8720?, {0xc002aa5000?, 0xc003e08ed8?, 0xc0063a9210?})
            GOROOT/src/syscall/zsyscall_linux_amd64.go:736 +0x38
        syscall.Read(...)
            GOROOT/src/syscall/syscall_unix.go:181
        internal/poll.ignoringEINTRIO(...)
            GOROOT/src/internal/poll/fd_unix.go:736
        internal/poll.(*FD).Read(0xc0029f8720, {0xc002aa5000, 0x7f2, 0x7f2})
            GOROOT/src/internal/poll/fd_unix.go:160 +0x2ae
        os.(*File).read(...)
            GOROOT/src/os/file_posix.go:29
        os.(*File).Read(0xc003780170, {0xc002aa5000?, 0x0?, 0x0?})
            GOROOT/src/os/file.go:118 +0x52
        os.ReadFile({0xc00385d000?, 0x534e4d0?})
            GOROOT/src/os/file.go:793 +0x167
        crypto/x509.loadSystemRoots()
            GOROOT/src/crypto/x509/root_unix.go:70 +0x396
        crypto/x509.initSystemRoots()
            GOROOT/src/crypto/x509/root.go:30 +0x5c
        sync.(*Once).doSlow(0xc006131f80?, 0x10?)
            GOROOT/src/sync/once.go:74 +0xc2
        sync.(*Once).Do(...)
            GOROOT/src/sync/once.go:65
        crypto/x509.systemRootsPool()
            GOROOT/src/crypto/x509/root.go:21 +0x45
        crypto/x509.(*Certificate).Verify(0xc0054f0008, {{0xc0059faa80, 0x9}, 0xc006130ae0, 0x0, {0xc178b10b7abc5033, 0x1a6984428, 0x9d7c000}, {0x0, 0x0, ...}, ...})
            GOROOT/src/crypto/x509/verify.go:784 +0x15d
        crypto/tls.(*Conn).verifyServerCertificate(0xc001a5ca88, {0xc003ca2ac8, 0x1, 0x1})
            GOROOT/src/crypto/tls/handshake_client.go:997 +0x819
        crypto/tls.(*clientHandshakeStateTLS13).readServerCertificate(0xc0063a9bd0)
            GOROOT/src/crypto/tls/handshake_client_tls13.go:531 +0x273
        crypto/tls.(*clientHandshakeStateTLS13).handshake(0xc0063a9bd0)
            GOROOT/src/crypto/tls/handshake_client_tls13.go:96 +0x29a
        crypto/tls.(*Conn).clientHandshake(0xc001a5ca88, {0x67d7600, 0xc005493c20})
            GOROOT/src/crypto/tls/handshake_client.go:263 +0x594
        crypto/tls.(*Conn).handshakeContext(0xc001a5ca88, {0x67d76a8, 0xc005a79180})
            GOROOT/src/crypto/tls/conn.go:1553 +0x3cb
        crypto/tls.(*Conn).HandshakeContext(...)
            GOROOT/src/crypto/tls/conn.go:1493
        net/http.(*persistConn).addTLS.func2()
            GOROOT/src/net/http/transport.go:1573 +0x6e
        created by net/http.(*persistConn).addTLS in goroutine 11662
            GOROOT/src/net/http/transport.go:1569 +0x309
--- FAIL: TestUseCerts (17.27s)

Parameters:

See also: How To Investigate a Go Test Failure (internal)

This test on roachdash | Improve this report!

github-actions[bot] commented 2 months ago

security.TestUseCerts failed on master @ bf7788e0102bd5ae3213d810055a813ba03d29fd:

        net/http.(*Transport).dialConnFor(0xc006762dc0, 0xc001f4fce0)
            GOROOT/src/net/http/transport.go:1485 +0xcd
        created by net/http.(*Transport).queueForDial in goroutine 9976
            GOROOT/src/net/http/transport.go:1449 +0x3c9
        Leaked goroutine: goroutine 11666 [syscall]:
        syscall.Syscall(0x0, 0x7, 0xc001a55b00, 0x82f)
            GOROOT/src/syscall/syscall_linux.go:69 +0x25
        syscall.read(0xc004b64a80?, {0xc001a55b00?, 0xc0062692e8?, 0xc001fd1210?})
            GOROOT/src/syscall/zsyscall_linux_amd64.go:736 +0x38
        syscall.Read(...)
            GOROOT/src/syscall/syscall_unix.go:181
        internal/poll.ignoringEINTRIO(...)
            GOROOT/src/internal/poll/fd_unix.go:736
        internal/poll.(*FD).Read(0xc004b64a80, {0xc001a55b00, 0x82f, 0x82f})
            GOROOT/src/internal/poll/fd_unix.go:160 +0x2ae
        os.(*File).read(...)
            GOROOT/src/os/file_posix.go:29
        os.(*File).Read(0xc003c241d0, {0xc001a55b00?, 0x0?, 0x0?})
            GOROOT/src/os/file.go:118 +0x52
        os.ReadFile({0xc0018a3200?, 0x536ad7c?})
            GOROOT/src/os/file.go:793 +0x167
        crypto/x509.loadSystemRoots()
            GOROOT/src/crypto/x509/root_unix.go:70 +0x396
        crypto/x509.initSystemRoots()
            GOROOT/src/crypto/x509/root.go:30 +0x5c
        sync.(*Once).doSlow(0xc006615f80?, 0x10?)
            GOROOT/src/sync/once.go:74 +0xc2
        sync.(*Once).Do(...)
            GOROOT/src/sync/once.go:65
        crypto/x509.systemRootsPool()
            GOROOT/src/crypto/x509/root.go:21 +0x45
        crypto/x509.(*Certificate).Verify(0xc004a77088, {{0xc001c2e4b0, 0x9}, 0xc006615260, 0x0, {0xc18764f56a9e9e6c, 0x190ba2397, 0x9e402e0}, {0x0, 0x0, ...}, ...})
            GOROOT/src/crypto/x509/verify.go:784 +0x15d
        crypto/tls.(*Conn).verifyServerCertificate(0xc0026dd508, {0xc003531530, 0x1, 0x1})
            GOROOT/src/crypto/tls/handshake_client.go:997 +0x819
        crypto/tls.(*clientHandshakeStateTLS13).readServerCertificate(0xc001fd1bd0)
            GOROOT/src/crypto/tls/handshake_client_tls13.go:531 +0x273
        crypto/tls.(*clientHandshakeStateTLS13).handshake(0xc001fd1bd0)
            GOROOT/src/crypto/tls/handshake_client_tls13.go:96 +0x29a
        crypto/tls.(*Conn).clientHandshake(0xc0026dd508, {0x68383a0, 0xc004eb1180})
            GOROOT/src/crypto/tls/handshake_client.go:263 +0x594
        crypto/tls.(*Conn).handshakeContext(0xc0026dd508, {0x6838448, 0xc0023eb500})
            GOROOT/src/crypto/tls/conn.go:1553 +0x3cb
        crypto/tls.(*Conn).HandshakeContext(...)
            GOROOT/src/crypto/tls/conn.go:1493
        net/http.(*persistConn).addTLS.func2()
            GOROOT/src/net/http/transport.go:1573 +0x6e
        created by net/http.(*persistConn).addTLS in goroutine 11663
            GOROOT/src/net/http/transport.go:1569 +0x309
--- FAIL: TestUseCerts (16.62s)

Parameters:

See also: How To Investigate a Go Test Failure (internal)

This test on roachdash | Improve this report!