"Unhealthy Readiness probe failed: Get "http://xx.xx.xx.xx:8080/ping": dial tcp xx.xx.xx.xx:8080: connect: connection refused" #bug

[Puvendhan]

I am getting the below error in the devlake-lake pod, the UI is up , but after clicking on "Proceed Database Migration" , the devlake pod is going down and giving the below error.

Pod is having this event "Unhealthy Readiness probe failed: Get "http://xx.xx.xx.xx:8080/ping": dial tcp xx.xx.xx.xx:8080: connect: connection refused"

I am using gcp Cloudsql mysql instance and used the below arguments to enable external mysql server

mysql: useExternal: true externalServer: xxxxxxxx externalPort: 3306 username: testuser password: xxxxxx database: devlake option: autoCreateSecret: true

time="2024-04-25 15:43:38" level=info msg=" [migrator] applying migration script modify component type to varchar(500):20231007145127"
time="2024-04-25 15:43:39" level=info msg="\x1b[31;1m/app/impls/dalgorm/dalgorm.go:146 \x1b[35;1mError 1054 (42S22): Unknown column 'component' in '_tool_sonarqube_issues'\n\x1b[0m\x1b[33m[0.838ms] \x1b[34;1m[rows:0]\x1b[0m ALTER TABLE `_tool_sonarqube_issues` RENAME COLUMN `component` TO `component_FC59F7072BF9A4479E6A1AA0AFFDB630`"
fatal error: sync: unlock of unlocked mutex

goroutine 84 [running]:
sync.fatal({0x18c5963?, 0xc0001e3608?})
        /usr/local/go/src/runtime/panic.go:1031 +0x1e
sync.(*Mutex).unlockSlow(0x2c0b430, 0xffffffff)
        /usr/local/go/src/sync/mutex.go:229 +0x3c
sync.(*Mutex).Unlock(0xc0006de120?)
        /usr/local/go/src/sync/mutex.go:223 +0x29
github.com/apache/incubator-devlake/server/services.ExecuteMigration()
        /app/server/services/init.go:138 +0x2fa
github.com/apache/incubator-devlake/server/api.SetupApiServer.func1(0xc0000bf400?)
        /app/server/api/api.go:100 +0x25
github.com/gin-gonic/gin.(*Context).Next(...)
        /go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/context.go:174
github.com/apache/incubator-devlake/server/api.OAuth2ProxyAuthentication.func1(0xc0000bf400)
        /app/server/api/middlewares.go:95 +0x143
github.com/gin-gonic/gin.(*Context).Next(...)
        /go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/context.go:174
github.com/apache/incubator-devlake/server/api.RestAuthentication.func1(0xc0000bf400)
        /app/server/api/middlewares.go:117 +0x2bb
github.com/gin-gonic/gin.(*Context).Next(...)
        /go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/context.go:174
github.com/gin-gonic/gin.CustomRecoveryWithWriter.func1(0xc0000bf400)
        /go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/recovery.go:102 +0x82
github.com/gin-gonic/gin.(*Context).Next(...)
        /go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/context.go:174
github.com/gin-gonic/gin.LoggerWithConfig.func1(0xc0000bf400)
        /go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/logger.go:240 +0xe7
github.com/gin-gonic/gin.(*Context).Next(...)
        /go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/context.go:174
github.com/gin-gonic/gin.(*Engine).handleHTTPRequest(0xc000605520, 0xc0000bf400)
        /go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/gin.go:620 +0x66b
github.com/gin-gonic/gin.(*Engine).ServeHTTP(0xc000605520, {0x1b34640?, 0xc0006f20e0}, 0xc0006e4100)
        /go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/gin.go:576 +0x1dd
net/http.serverHandler.ServeHTTP({0xc0006e0120?}, {0x1b34640, 0xc0006f20e0}, 0xc0006e4100)
        /usr/local/go/src/net/http/server.go:2936 +0x316
net/http.(*conn).serve(0xc002f3e360, {0x1b35600, 0xc002f14c90})
        /usr/local/go/src/net/http/server.go:1995 +0x612
created by net/http.(*Server).Serve
        /usr/local/go/src/net/http/server.go:3089 +0x5ed

goroutine 1 [IO wait]:
internal/poll.runtime_pollWait(0x7eaf6c986978, 0x72)
        /usr/local/go/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0000b1b00?, 0x4?, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc0000b1b00)
        /usr/local/go/src/internal/poll/fd_unix.go:614 +0x2bd
net.(*netFD).accept(0xc0000b1b00)
        /usr/local/go/src/net/fd_unix.go:172 +0x35
net.(*TCPListener).accept(0xc0000173b0)
        /usr/local/go/src/net/tcpsock_posix.go:148 +0x25
net.(*TCPListener).Accept(0xc0000173b0)
        /usr/local/go/src/net/tcpsock.go:297 +0x3d
net/http.(*Server).Serve(0xc002d21ef0, {0x1b34430, 0xc0000173b0})
        /usr/local/go/src/net/http/server.go:3059 +0x385
net/http.(*Server).ListenAndServe(0xc002d21ef0)
        /usr/local/go/src/net/http/server.go:2988 +0x7d
net/http.ListenAndServe(...)
        /usr/local/go/src/net/http/server.go:3242
github.com/gin-gonic/gin.(*Engine).Run(0xc000605520, {0xc001087ee8, 0x1, 0x1})
        /go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/gin.go:386 +0x20e
github.com/apache/incubator-devlake/server/api.RunApiServer(0xc000605520?)
        /app/server/api/api.go:172 +0xdb
github.com/apache/incubator-devlake/server/api.CreateAndRunApiServer()
        /app/server/api/api.go:73 +0x67
main.main()
        /app/server/main.go:33 +0x36

goroutine 8 [chan receive]:
github.com/panjf2000/ants/v2.(*Pool).purgePeriodically(0xc0001f8620)
        /go/pkg/mod/github.com/panjf2000/ants/v2@v2.4.6/pool.go:69 +0x8b
created by github.com/panjf2000/ants/v2.NewPool
        /go/pkg/mod/github.com/panjf2000/ants/v2@v2.4.6/pool.go:137 +0x365

goroutine 9 [select, 1 minutes]:
database/sql.(*DB).connectionOpener(0xc0001a11e0, {0x1b35558, 0xc0004e47d0})
        /usr/local/go/src/database/sql/sql.go:1218 +0x8d
created by database/sql.OpenDB
        /usr/local/go/src/database/sql/sql.go:791 +0x18d

goroutine 35 [select, 1 minutes]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
        /go/pkg/mod/github.com/go-sql-driver/mysql@v1.7.1/connection.go:614 +0xaa
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
        /go/pkg/mod/github.com/go-sql-driver/mysql@v1.7.1/connection.go:611 +0x10a

goroutine 10 [select, 1 minutes]:
database/sql.(*DB).connectionCleaner(0xc0001a11e0, 0x0?)
        /usr/local/go/src/database/sql/sql.go:1061 +0xa7
created by database/sql.(*DB).startCleanerLocked
        /usr/local/go/src/database/sql/sql.go:1048 +0x105

goroutine 11 [chan receive, 1 minutes]:
database/sql.(*Tx).awaitDone(0xc0004f1d80)
        /usr/local/go/src/database/sql/sql.go:2167 +0x31
created by database/sql.(*DB).beginDC
        /usr/local/go/src/database/sql/sql.go:1887 +0x23b

goroutine 67 [IO wait]:
internal/poll.runtime_pollWait(0x7eaf6c986d38, 0x72)
        /usr/local/go/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0000b1f80?, 0xc0006e0131?, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0000b1f80, {0xc0006e0131, 0x1, 0x1})
        /usr/local/go/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc0000b1f80, {0xc0006e0131?, 0x173c920?, 0xc00060cf48?})
        /usr/local/go/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc002f1a0e8, {0xc0006e0131?, 0xc002f11590?, 0xc00009aff0?})
        /usr/local/go/src/net/net.go:183 +0x45
net/http.(*connReader).backgroundRead(0xc0006e0120)
        /usr/local/go/src/net/http/server.go:674 +0x3f
created by net/http.(*connReader).startBackgroundRead
        /usr/local/go/src/net/http/server.go:670 +0xca

goroutine 13 [select, 1 minutes]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
        /go/pkg/mod/github.com/go-sql-driver/mysql@v1.7.1/connection.go:614 +0xaa
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
        /go/pkg/mod/github.com/go-sql-driver/mysql@v1.7.1/connection.go:611 +0x10a

[d4x1]

@Puvendhan what's the version you are using?

[Puvendhan]

Here's the version 1.0.0-beta5

[Puvendhan]

Hi, Any update on this?

[klesh]

Seems like a compatibility problem of the gcp Cloudsql mysql, try using mysql or mariadb instead.