search

opensearch-project / OpenSearch

🔎 Open source distributed and RESTful search engine.
https://opensearch.org/docs/latest/opensearch/index/
Apache License 2.0
9.79k stars 1.82k forks source link

[BUG] org.opensearch.indices.replication.SegmentReplicationIT.classMethod test failure due to file handle leaks #11034

Closed dreamer-89 closed 6 months ago

dreamer-89 commented 1 year ago

https://build.ci.opensearch.org/job/gradle-check/29346/testReport/org.opensearch.indices.replication/SegmentReplicationIT/classMethod/

java.lang.RuntimeException: file handle leaks: [FileChannel(/var/jenkins/workspace/gradle-check/search/server/build/testrun/internalClusterTest/temp/org.opensearch.indices.replication.SegmentReplicationIT_C533551D8FDBD2D9-001/tempDir-013/node_t1/nodes/0/indices/qUl2w4isRVuQATN5BR4O8w/0/index/_11_Lucene90_0.pos), FileChannel(/var/jenkins/workspace/gradle-check/search/server/build/testrun/internalClusterTest/temp/org.opensearch.indices.replication.SegmentReplicationIT_C533551D8FDBD2D9-001/tempDir-013/node_t1/nodes/0/indices/qUl2w4isRVuQATN5BR4O8w/0/index/_11.fdt), FileChannel(/var/jenkins/workspace/gradle-check/search/server/build/testrun/internalClusterTest/temp/org.opensearch.indices.replication.SegmentReplicationIT_C533551D8FDBD2D9-001/tempDir-013/node_t1/nodes/0/indices/qUl2w4isRVuQATN5BR4O8w/0/index/_11.fdx)]
    at __randomizedtesting.SeedInfo.seed([C533551D8FDBD2D9]:0)
    at org.apache.lucene.tests.mockfile.LeakFS.onClose(LeakFS.java:63)
    at org.apache.lucene.tests.mockfile.FilterFileSystem.close(FilterFileSystem.java:69)
    at org.apache.lucene.tests.mockfile.FilterFileSystem.close(FilterFileSystem.java:70)
    at org.apache.lucene.tests.util.TestRuleTemporaryFilesCleanup.afterAlways(TestRuleTemporaryFilesCleanup.java:223)
    at com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterAlways(TestRuleAdapter.java:31)
    at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
    at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
    at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
    at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
    at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
    at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
    at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
    at org.junit.rules.RunRules.evaluate(RunRules.java:20)
    at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
    at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
    at java.base/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.Exception
    at org.apache.lucene.tests.mockfile.LeakFS.onOpen(LeakFS.java:46)
    at org.apache.lucene.tests.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:82)
    at org.apache.lucene.tests.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:202)
    at org.apache.lucene.tests.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:171)
    at java.base/java.nio.channels.FileChannel.open(FileChannel.java:298)
    at java.base/java.nio.channels.FileChannel.open(FileChannel.java:357)
    at org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:78)
    at org.opensearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:183)
    at org.apache.lucene.tests.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:816)
    at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:101)
    at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:101)
    at org.apache.lucene.codecs.lucene90.Lucene90PostingsReader.<init>(Lucene90PostingsReader.java:93)
    at org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.fieldsProducer(Lucene90PostingsFormat.java:424)
    at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:330)
    at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:392)
    at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:118)
    at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:92)
    at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:180)
    at org.apache.lucene.index.ReadersAndUpdates.getLatestReader(ReadersAndUpdates.java:245)
    at org.apache.lucene.index.PendingSoftDeletes.ensureInitialized(PendingSoftDeletes.java:209)
    at org.apache.lucene.index.PendingSoftDeletes.isFullyDeleted(PendingSoftDeletes.java:220)
    at org.apache.lucene.index.ReadersAndUpdates.isFullyDeleted(ReadersAndUpdates.java:823)
    at org.apache.lucene.index.IndexWriter.isFullyDeleted(IndexWriter.java:5992)
    at org.apache.lucene.index.IndexWriter.commitMerge(IndexWriter.java:4563)
    at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:5337)
    at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4698)
    at org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:6450)
    at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:639)
    at org.opensearch.index.engine.OpenSearchConcurrentMergeScheduler.doMerge(OpenSearchConcurrentMergeScheduler.java:120)
    at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:700)
reta commented 8 months ago

Same for org.opensearch.indexing.IndexActionIT.classMethod:

java.lang.RuntimeException: file handle leaks: [FileChannel(/var/jenkins/workspace/gradle-check/search/server/build/testrun/internalClusterTest/temp/org.opensearch.indexing.IndexActionIT_D1A88E4DE260012C-001/tempDir-002/node_s5-shared/QjrsfkZmyo/0/-kSdWg8vRKyiGHUfLGy1Hg/8/index/_b.fdx), FileChannel(/var/jenkins/workspace/gradle-check/search/server/build/testrun/internalClusterTest/temp/org.opensearch.indexing.IndexActionIT_D1A88E4DE260012C-001/tempDir-002/node_s5-shared/QjrsfkZmyo/0/-kSdWg8vRKyiGHUfLGy1Hg/8/index/_b_Lucene99_1.pos), FileChannel(/var/jenkins/workspace/gradle-check/search/server/build/testrun/internalClusterTest/temp/org.opensearch.indexing.IndexActionIT_D1A88E4DE260012C-001/tempDir-002/node_s5-shared/QjrsfkZmyo/0/-kSdWg8vRKyiGHUfLGy1Hg/8/index/_b.fdt), FileChannel(/var/jenkins/workspace/gradle-check/search/server/build/testrun/internalClusterTest/temp/org.opensearch.indexing.IndexActionIT_D1A88E4DE260012C-001/tempDir-002/node_s5-shared/QjrsfkZmyo/0/-kSdWg8vRKyiGHUfLGy1Hg/8/index/_b_Lucene99_0.pos)]
    at __randomizedtesting.SeedInfo.seed([D1A88E4DE260012C]:0)
    at org.apache.lucene.tests.mockfile.LeakFS.onClose(LeakFS.java:63)
    at org.apache.lucene.tests.mockfile.FilterFileSystem.close(FilterFileSystem.java:69)
    at org.apache.lucene.tests.mockfile.FilterFileSystem.close(FilterFileSystem.java:70)
    at org.apache.lucene.tests.util.TestRuleTemporaryFilesCleanup.afterAlways(TestRuleTemporaryFilesCleanup.java:223)
    at com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterAlways(TestRuleAdapter.java:31)
    at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
    at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
    at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
    at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
    at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
    at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
    at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
    at org.junit.rules.RunRules.evaluate(RunRules.java:20)
    at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
    at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
    at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.Exception
    at org.apache.lucene.tests.mockfile.LeakFS.onOpen(LeakFS.java:46)
    at org.apache.lucene.tests.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:82)
    at org.apache.lucene.tests.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:202)
    at org.apache.lucene.tests.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:171)
    at java.base/java.nio.channels.FileChannel.open(FileChannel.java:309)
    at java.base/java.nio.channels.FileChannel.open(FileChannel.java:369)
    at org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:78)
    at org.opensearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:183)
    at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:101)
    at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:101)
    at org.apache.lucene.codecs.lucene90.compressing.FieldsIndexReader.<init>(FieldsIndexReader.java:68)
    at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsReader.<init>(Lucene90CompressingStoredFieldsReader.java:166)
    at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsFormat.fieldsReader(Lucene90CompressingStoredFieldsFormat.java:133)
    at org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat.fieldsReader(Lucene90StoredFieldsFormat.java:136)
    at org.apache.lucene.tests.codecs.asserting.AssertingStoredFieldsFormat.fieldsReader(AssertingStoredFieldsFormat.java:43)
    at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:139)
    at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:96)
    at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:178)
    at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:220)
    at org.apache.lucene.index.IndexWriter.lambda$getReader$0(IndexWriter.java:542)
    at org.apache.lucene.index.IndexWriter.lambda$getReader$1(IndexWriter.java:632)
    at org.apache.lucene.index.IndexWriter$2.onMergeComplete(IndexWriter.java:3893)
    at org.apache.lucene.index.IndexWriter.commitMerge(IndexWriter.java:4552)
    at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:5432)
    at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4740)
    at org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:6541)
    at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:639)
    at org.opensearch.index.engine.OpenSearchConcurrentMergeScheduler.doMerge(OpenSearchConcurrentMergeScheduler.java:120)
    at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:700)
mch2 commented 6 months ago

I believe this has been fixed with the changes here to node-node replication. Will close this for now please re-open if this occurs again.