search

smallrye / smallrye-fault-tolerance

SmallRye implementation of MicroProfile Fault Tolerance: bulkheads, circuit breakers, fallbacks, rate limits, retries, timeouts, and more
Apache License 2.0
90 stars 37 forks source link

Intermittent failure in Quarkus TCK #233

Open stuartwdouglas opened 4 years ago

stuartwdouglas commented 4 years ago

There is the following failure in a Quarkus PR (unrelated to FT): https://github.com/quarkusio/quarkus/pull/9556/checks?check_run_id=703890388

[ERROR] Failures: 
[ERROR]   TimeoutUninterruptableTest>Arquillian.run:138->testTimeoutAsyncBulkhead:199 Execution count after completing all tasks expected [1] but found [2]

Looking at the test logic the only way I can see for this to happen is if a thread continues to execute after throwing an exception (e.g. the caller receives the TimeoutException, but then the actual thread runs anyway).

Ladicek commented 4 years ago

Full stack trace of the failing assert: https://github.com/quarkusio/quarkus/pull/9556#issuecomment-633299576

Ladicek commented 3 years ago

It may also fail in a different way: https://github.com/quarkusio/quarkus/pull/18387

Full stacktrace:

java.lang.AssertionError: Unexpected exception thrown from Future
    at org.testng.Assert.fail(Assert.java:85)
    at org.eclipse.microprofile.fault.tolerance.tck.util.Exceptions.expect(Exceptions.java:98)
    at org.eclipse.microprofile.fault.tolerance.tck.TimeoutUninterruptableTest.testTimeoutAsyncBulkhead(TimeoutUninterruptableTest.java:190)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:566)
    at io.quarkus.arquillian.QuarkusProtocol$QuarkusMethodExecutor$1.invoke(QuarkusProtocol.java:87)
    at org.jboss.arquillian.container.test.impl.execution.LocalTestExecuter.execute(LocalTestExecuter.java:57)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:566)
    at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:86)
    at org.jboss.arquillian.core.impl.EventContextImpl.invokeObservers(EventContextImpl.java:103)
    at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:90)
    at org.jboss.arquillian.core.impl.ManagerImpl.fire(ManagerImpl.java:133)
    at org.jboss.arquillian.core.impl.ManagerImpl.fire(ManagerImpl.java:105)
    at org.jboss.arquillian.core.impl.EventImpl.fire(EventImpl.java:62)
    at io.quarkus.arquillian.QuarkusProtocol$QuarkusMethodExecutor.invoke(QuarkusProtocol.java:67)
    at org.jboss.arquillian.container.test.impl.execution.RemoteTestExecuter.execute(RemoteTestExecuter.java:103)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:566)
    at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:86)
    at org.jboss.arquillian.core.impl.EventContextImpl.invokeObservers(EventContextImpl.java:103)
    at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:90)
    at org.jboss.arquillian.core.impl.ManagerImpl.fire(ManagerImpl.java:133)
    at org.jboss.arquillian.core.impl.ManagerImpl.fire(ManagerImpl.java:105)
    at org.jboss.arquillian.core.impl.EventImpl.fire(EventImpl.java:62)
    at org.jboss.arquillian.container.test.impl.execution.ClientTestExecuter.execute(ClientTestExecuter.java:52)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:566)
    at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:86)
    at org.jboss.arquillian.core.impl.EventContextImpl.invokeObservers(EventContextImpl.java:103)
    at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:90)
    at org.jboss.arquillian.container.test.impl.client.ContainerEventController.createContext(ContainerEventController.java:128)
    at org.jboss.arquillian.container.test.impl.client.ContainerEventController.createTestContext(ContainerEventController.java:118)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:566)
    at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:86)
    at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:95)
    at org.jboss.arquillian.test.impl.TestContextHandler.createTestContext(TestContextHandler.java:116)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:566)
    at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:86)
    at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:95)
    at org.jboss.arquillian.test.impl.TestContextHandler.createClassContext(TestContextHandler.java:83)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:566)
    at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:86)
    at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:95)
    at org.jboss.arquillian.test.impl.TestContextHandler.createSuiteContext(TestContextHandler.java:69)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:566)
    at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:86)
    at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:95)
    at org.jboss.arquillian.core.impl.ManagerImpl.fire(ManagerImpl.java:133)
    at org.jboss.arquillian.test.impl.EventTestRunnerAdaptor.test(EventTestRunnerAdaptor.java:139)
    at org.jboss.arquillian.testng.Arquillian.run(Arquillian.java:138)
    at org.testng.internal.MethodInvocationHelper.invokeHookable(MethodInvocationHelper.java:242)
    at org.testng.internal.Invoker.invokeMethod(Invoker.java:576)
    at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:716)
    at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:988)
    at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:125)
    at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:109)
    at org.testng.TestRunner.privateRun(TestRunner.java:648)
    at org.testng.TestRunner.run(TestRunner.java:505)
    at org.testng.SuiteRunner.runTest(SuiteRunner.java:455)
    at org.testng.SuiteRunner.runSequentially(SuiteRunner.java:450)
    at org.testng.SuiteRunner.privateRun(SuiteRunner.java:415)
    at org.testng.SuiteRunner.run(SuiteRunner.java:364)
    at org.testng.SuiteRunnerWorker.runSuite(SuiteRunnerWorker.java:52)
    at org.testng.SuiteRunnerWorker.run(SuiteRunnerWorker.java:84)
    at org.testng.TestNG.runSuitesSequentially(TestNG.java:1208)
    at org.testng.TestNG.runSuitesLocally(TestNG.java:1137)
    at org.testng.TestNG.runSuites(TestNG.java:1049)
    at org.testng.TestNG.run(TestNG.java:1017)
    at org.apache.maven.surefire.testng.TestNGExecutor.run(TestNGExecutor.java:136)
    at org.apache.maven.surefire.testng.TestNGDirectoryTestSuite.executeSingleClass(TestNGDirectoryTestSuite.java:112)
    at org.apache.maven.surefire.testng.TestNGDirectoryTestSuite.execute(TestNGDirectoryTestSuite.java:99)
    at org.apache.maven.surefire.testng.TestNGProvider.invoke(TestNGProvider.java:145)
    at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:428)
    at org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:162)
    at org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:562)
    at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:548)
Caused by: org.eclipse.microprofile.faulttolerance.exceptions.BulkheadException: Bulkhead[org.eclipse.microprofile.fault.tolerance.tck.timeout.clientserver.UninterruptableTimeoutClient#serviceTimeoutAsyncBulkhead] rejected from bulkhead
    at io.smallrye.faulttolerance.core.bulkhead.BulkheadBase.bulkheadRejected(BulkheadBase.java:17)
    at io.smallrye.faulttolerance.core.bulkhead.FutureThreadPoolBulkhead.doApply(FutureThreadPoolBulkhead.java:103)
    at io.smallrye.faulttolerance.core.bulkhead.FutureThreadPoolBulkhead.apply(FutureThreadPoolBulkhead.java:40)
    at io.smallrye.faulttolerance.core.bulkhead.FutureThreadPoolBulkhead.apply(FutureThreadPoolBulkhead.java:24)
    at io.smallrye.faulttolerance.core.timeout.Timeout.doApply(Timeout.java:55)
    at io.smallrye.faulttolerance.core.timeout.Timeout.apply(Timeout.java:30)
    at io.smallrye.faulttolerance.core.timeout.AsyncTimeout.lambda$doApply$0(AsyncTimeout.java:47)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at io.smallrye.context.impl.wrappers.SlowContextualRunnable.run(SlowContextualRunnable.java:19)
    at io.quarkus.vertx.core.runtime.VertxCoreRecorder$14.runWith(VertxCoreRecorder.java:486)
    at org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2442)
    at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1476)
    at org.jboss.threads.DelegatingRunnable.run(DelegatingRunnable.java:29)
    at org.jboss.threads.ThreadLocalResettingRunnable.run(ThreadLocalResettingRunnable.java:29)
    at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
    at java.base/java.lang.Thread.run(Thread.java:829)