search

lightningnetwork / lnd

Lightning Network Daemon ⚡️
MIT License
7.65k stars 2.07k forks source link

[bug]: Race in SQLITE dependency ? #8666

Open ziggie1984 opened 5 months ago

ziggie1984 commented 5 months ago 
==================
WARNING: DATA RACE
Write at 0x0000022aea6a by goroutine 89:
  modernc.org/sqlite/lib.Xsqlite3_initialize()
      /home/runner/go/pkg/mod/modernc.org/sqlite@v1.29.5/lib/sqlite_linux_amd64.go:163309 +0x65b
  modernc.org/sqlite/lib._openDatabase()
      /home/runner/go/pkg/mod/modernc.org/sqlite@v1.29.5/lib/sqlite_linux_amd64.go:165949 +0x24f
  modernc.org/sqlite/lib.Xsqlite3_open_v2()
      /home/runner/go/pkg/mod/modernc.org/sqlite@v1.29.5/lib/sqlite_linux_amd64.go:166169 +0x1ad
  modernc.org/sqlite.(*conn).openV2()
      /home/runner/go/pkg/mod/modernc.org/sqlite@v1.29.5/sqlite.go:1395 +0x1[74](https://github.com/lightningnetwork/lnd/actions/runs/8752151693/job/24019207768?pr=8613#step:7:75)
  modernc.org/sqlite.newConn()
      /home/runner/go/pkg/mod/modernc.org/sqlite@v1.29.5/sqlite.go:839 +0x22e
  modernc.org/sqlite.(*Driver).Open()
      /home/runner/go/pkg/mod/modernc.org/sqlite@v1.29.5/sqlite.go:1965 +0x59
  database/sql.dsnConnector.Connect()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/database/sql/sql.go:[75](https://github.com/lightningnetwork/lnd/actions/runs/8752151693/job/24019207768?pr=8613#step:7:76)8 +0x64
  database/sql.(*dsnConnector).Connect()
      <autogenerated>:1 +0x1f
  database/sql.(*DB).conn()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/database/sql/sql.go:1387 +0xacc
  database/sql.(*DB).PingContext.func1()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/database/sql/sql.go:850 +0x65
  database/sql.(*DB).retry()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/database/sql/sql.go:1538 +0x4a
  database/sql.(*DB).PingContext()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/database/sql/sql.go:849 +0xbc
  database/sql.(*DB).Ping()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/database/sql/sql.go:867 +0x4b
  github.com/golang-migrate/migrate/v4/database/sqlite.WithInstance()
      /home/runner/go/pkg/mod/github.com/golang-migrate/migrate/v4@v4.17.0/database/sqlite/sqlite.go:48 +0x4c
  github.com/lightningnetwork/lnd/sqldb.NewSqliteStore()
      /home/runner/go/pkg/mod/github.com/lightningnetwork/lnd/sqldb@v1.0.1/sqlite.go:106 +0x5e7
  github.com/lightningnetwork/lnd/sqldb.NewTestSqliteDB()
      /home/runner/go/pkg/mod/github.com/lightningnetwork/lnd/sqldb@v1.0.1/sqlite.go:150 +0x124
  github.com/lightningnetwork/lnd/invoices_test.TestInvoiceRegistry.func3()
      /home/runner/work/lnd/lnd/invoices/invoiceregistry_test.go:133 +0x57
  github.com/lightningnetwork/lnd/invoices_test.TestInvoiceRegistry.func5.1()
      /home/runner/work/lnd/lnd/invoices/invoiceregistry_test.go:161 +0x39
  github.com/lightningnetwork/lnd/invoices_test.newTestContext()
      /home/runner/work/lnd/lnd/invoices/test_utils_test.go:156 +0x68
  github.com/lightningnetwork/lnd/invoices_test.testMppPaymentWithOverpayment.func1()
      /home/runner/work/lnd/lnd/invoices/invoiceregistry_test.go:1123 +0xa4
  runtime.call16()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/runtime/asm_amd64.s:747 +0x42
  reflect.Value.Call()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/reflect/value.go:3[80](https://github.com/lightningnetwork/lnd/actions/runs/8752151693/job/24019207768?pr=8613#step:7:81) +0xb5
  testing/quick.Check()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/testing/quick/quick.go:290 +0x2fc
  github.com/lightningnetwork/lnd/invoices_test.testMppPaymentWithOverpayment()
      /home/runner/work/lnd/lnd/invoices/invoiceregistry_test.go:1187 +0x124
  github.com/lightningnetwork/lnd/invoices_test.TestInvoiceRegistry.func5()
      /home/runner/work/lnd/lnd/invoices/invoiceregistry_test.go:157 +0xa6
  testing.tRunner()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/testing/testing.go:1595 +0x238
  testing.(*T).Run.func1()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/testing/testing.go:1648 +0x44

Previous write at 0x0000022aea6a by goroutine 104:
  modernc.org/sqlite/lib.Xsqlite3_initialize()
      /home/runner/go/pkg/mod/modernc.org/sqlite@v1.29.5/lib/sqlite_linux_amd64.go:163309 +0x65b
  modernc.org/sqlite/lib._openDatabase()
      /home/runner/go/pkg/mod/modernc.org/sqlite@v1.29.5/lib/sqlite_linux_amd64.go:165949 +0x24f
  modernc.org/sqlite/lib.Xsqlite3_open_v2()
      /home/runner/go/pkg/mod/modernc.org/sqlite@v1.29.5/lib/sqlite_linux_amd64.go:166169 +0x1ad
  modernc.org/sqlite.(*conn).openV2()
      /home/runner/go/pkg/mod/modernc.org/sqlite@v1.29.5/sqlite.go:1395 +0x174
  modernc.org/sqlite.newConn()
      /home/runner/go/pkg/mod/modernc.org/sqlite@v1.29.5/sqlite.go:839 +0x22e
  modernc.org/sqlite.(*Driver).Open()
      /home/runner/go/pkg/mod/modernc.org/sqlite@v1.29.5/sqlite.go:1965 +0x59
  database/sql.dsnConnector.Connect()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/database/sql/sql.go:758 +0x64
  database/sql.(*dsnConnector).Connect()
      <autogenerated>:1 +0x1f
  database/sql.(*DB).conn()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/database/sql/sql.go:1387 +0xacc
  database/sql.(*DB).PingContext.func1()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/database/sql/sql.go:850 +0x65
  database/sql.(*DB).retry()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/database/sql/sql.go:1538 +0x4a
  database/sql.(*DB).PingContext()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/database/sql/sql.go:849 +0xbc
  database/sql.(*DB).Ping()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/database/sql/sql.go:867 +0x4b
  github.com/golang-migrate/migrate/v4/database/sqlite.WithInstance()
      /home/runner/go/pkg/mod/github.com/golang-migrate/migrate/v4@v4.17.0/database/sqlite/sqlite.go:48 +0x4c
  github.com/lightningnetwork/lnd/sqldb.NewSqliteStore()
      /home/runner/go/pkg/mod/github.com/lightningnetwork/lnd/sqldb@v1.0.1/sqlite.go:106 +0x5e7
  github.com/lightningnetwork/lnd/sqldb.NewTestSqliteDB()
      /home/runner/go/pkg/mod/github.com/lightningnetwork/lnd/sqldb@v1.0.1/sqlite.go:150 +0x124
  github.com/lightningnetwork/lnd/invoices_test.TestInvoiceRegistry.func3()
      /home/runner/work/lnd/lnd/invoices/invoiceregistry_test.go:133 +0x57
  github.com/lightningnetwork/lnd/invoices_test.TestInvoiceRegistry.func5.1()
      /home/runner/work/lnd/lnd/invoices/invoiceregistry_test.go:161 +0x39
  github.com/lightningnetwork/lnd/invoices_test.newTestContext()
      /home/runner/work/lnd/lnd/invoices/test_utils_test.go:156 +0x68
  github.com/lightningnetwork/lnd/invoices_test.testSettleInvoicePaymentAddrRequired()
      /home/runner/work/lnd/lnd/invoices/invoiceregistry_test.go:1619 +0x5d
  github.com/lightningnetwork/lnd/invoices_test.TestInvoiceRegistry.func5()
      /home/runner/work/lnd/lnd/invoices/invoiceregistry_test.go:157 +0xa6
  testing.tRunner()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/testing/testing.go:1595 +0x238
  testing.(*T).Run.func1()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/testing/testing.go:1648 +0x44

Goroutine 89 (running) created at:
  testing.(*T).Run()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/testing/testing.go:1648 +0x[82](https://github.com/lightningnetwork/lnd/actions/runs/8752151693/job/24019207768?pr=8613#step:7:83)a
  github.com/lightningnetwork/lnd/invoices_test.TestInvoiceRegistry()
      /home/runner/work/lnd/lnd/invoices/invoiceregistry_test.go:156 +0x55c
  github.com/cenkalti/backoff/v4.RetryNotifyWithTimer()
      /home/runner/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.1.3/retry.go:55 +0x193
  database/sql.(*DB).Ping()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/database/sql/sql.go:[86](https://github.com/lightningnetwork/lnd/actions/runs/8752151693/job/24019207768?pr=8613#step:7:87)7 +0x159
  github.com/lightningnetwork/lnd/sqldb.NewTestPgFixture.func2()
      /home/runner/go/pkg/mod/github.com/lightningnetwork/lnd/sqldb@v1.0.1/postgres_fixture.go:99 +0x135
  github.com/cenkalti/backoff/v4.RetryNotifyWithTimer()
      /home/runner/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.1.3/retry.go:55 +0x193
  github.com/cenkalti/backoff/v4.RetryNotify()
      /home/runner/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.1.3/retry.go:34 +0xfd
  github.com/cenkalti/backoff/v4.Retry()
      /home/runner/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.1.3/retry.go:28 +0xe5
  github.com/ory/dockertest/v3.(*Pool).Retry()
      /home/runner/go/pkg/mod/github.com/ory/dockertest/v3@v3.10.0/dockertest.go:574 +0x100
  github.com/lightningnetwork/lnd/sqldb.NewTestPgFixture()
      /home/runner/go/pkg/mod/github.com/lightningnetwork/lnd/sqldb@v1.0.1/postgres_fixture.go:93 +0xb7e
  github.com/lightningnetwork/lnd/invoices_test.TestInvoiceRegistry()
      /home/runner/work/lnd/lnd/invoices/invoiceregistry_test.go:121 +0x1[91](https://github.com/lightningnetwork/lnd/actions/runs/8752151693/job/24019207768?pr=8613#step:7:92)
  testing.tRunner()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/testing/testing.go:1595 +0x238
  testing.(*T).Run.func1()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/testing/testing.go:1648 +0x44

Goroutine 104 (running) created at:
  testing.(*T).Run()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/testing/testing.go:1648 +0x82a
  github.com/lightningnetwork/lnd/invoices_test.TestInvoiceRegistry()
      /home/runner/work/lnd/lnd/invoices/invoiceregistry_test.go:156 +0x55c
  github.com/cenkalti/backoff/v4.RetryNotifyWithTimer()
      /home/runner/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.1.3/retry.go:55 +0x1[93](https://github.com/lightningnetwork/lnd/actions/runs/8752151693/job/24019207768?pr=8613#step:7:94)
  database/sql.(*DB).Ping()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/database/sql/sql.go:867 +0x159
  github.com/lightningnetwork/lnd/sqldb.NewTestPgFixture.func2()
      /home/runner/go/pkg/mod/github.com/lightningnetwork/lnd/sqldb@v1.0.1/postgres_fixture.go:99 +0x135
  github.com/cenkalti/backoff/v4.RetryNotifyWithTimer()
      /home/runner/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.1.3/retry.go:55 +0x193
  github.com/cenkalti/backoff/v4.RetryNotify()
      /home/runner/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.1.3/retry.go:34 +0xfd
  github.com/cenkalti/backoff/v4.Retry()
      /home/runner/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.1.3/retry.go:28 +0xe5
  github.com/ory/dockertest/v3.(*Pool).Retry()
      /home/runner/go/pkg/mod/github.com/ory/dockertest/v3@v3.10.0/dockertest.go:574 +0x100
  github.com/lightningnetwork/lnd/sqldb.NewTestPgFixture()
      /home/runner/go/pkg/mod/github.com/lightningnetwork/lnd/sqldb@v1.0.1/postgres_fixture.go:93 +0xb7e
  github.com/lightningnetwork/lnd/invoices_test.TestInvoiceRegistry()
      /home/runner/work/lnd/lnd/invoices/invoiceregistry_test.go:121 +0x191
  testing.tRunner()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/testing/testing.go:15[95](https://github.com/lightningnetwork/lnd/actions/runs/8752151693/job/24019207768?pr=8613#step:7:96) +0x238
  testing.(*T).Run.func1()
      /home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.21.4.linux-amd64/src/testing/testing.go:1648 +0x44

source: https://github.com/lightningnetwork/lnd/actions/runs/8752151693/job/24019207768?pr=8613

Hmm seems like a race in the sqlite library, I think we had something similar recently @ellemouton ?

bhandras commented 5 months ago

Hi @ziggie1984, thanks for the report. We're aware and this is a known issue that is being worked on: https://gitlab.com/cznic/sqlite/-/issues/180

Roasbeef commented 5 months ago

Looks like there's two newer versions we could try: https://gitlab.com/cznic/sqlite/-/tags

Roasbeef commented 5 months ago

Made #8696