search

testng-team / testng

TestNG testing framework
https://testng.org
Apache License 2.0
1.99k stars 1.02k forks source link

Parallel Dataproviders & retries causes test result count to be skewed #2934

Closed krmahadevan closed 1 year ago

krmahadevan commented 1 year ago

TestNG Version

Note: only the latest version is supported

7.8.0

Expected behavior

A test method that can be retried and which is powered by a data provider which can run in parallel is executed, then the count of the test results (total and individual count for passed, failed and skipped) should match.

Actual behavior

The count is skewed

Is the issue reproducible on runner?

Test case sample

Please, share the test case (as small as possible) which shows the issue

Sample testcase ```java import static org.assertj.core.api.Assertions.assertThat; import java.lang.reflect.Method; import java.util.Arrays; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import org.testng.IAnnotationTransformer; import org.testng.IRetryAnalyzer; import org.testng.ITestContext; import org.testng.ITestListener; import org.testng.ITestResult; import org.testng.annotations.DataProvider; import org.testng.annotations.IDataProviderAnnotation; import org.testng.annotations.Test; public class MyTestCase { @Test(dataProvider = "testData", retryAnalyzer = RetryAnalyzer.class) public void API_dummyTest(int number, String text) { System.out.println( "[" + Thread.currentThread().getId() + "] This is for " + text + " " + number); assertThat(number % 2).isEqualTo(0); } @DataProvider(name = "testData") public Object[][] getData() { return new Object[][]{ {1, "Test One"}, {2, "Test Two"}, {3, "Test Three"} }; } public static class ToggleDataProvider implements IAnnotationTransformer { private final boolean isParallel; public ToggleDataProvider(boolean isParallel) { this.isParallel = isParallel; } @Override public void transform(IDataProviderAnnotation annotation, Method method) { if (isParallel) { annotation.setParallel(true); } } } public static class RetryAnalyzer implements IRetryAnalyzer { private final AtomicInteger counter = new AtomicInteger(0); @Override public boolean retry(ITestResult iTestResult) { return counter.getAndIncrement() < 3; } } public static class CoreListener implements ITestListener { private ITestContext context; @Override public void onTestFailure(ITestResult result) { printer(result, "FAILED"); } @Override public void onTestSkipped(ITestResult result) { printer(result, "SKIPPED"); } @Override public void onTestSuccess(ITestResult result) { printer(result, "passed"); } private static void printer(ITestResult result, String prefix) { System.err.println(prefix + " for values " + Arrays.toString(result.getParameters())); } @Override public void onFinish(ITestContext context) { this.context = context; display("PASSED", context.getPassedTests().getAllResults()); display("FAILED", context.getFailedTests().getAllResults()); display("SKIPPED", context.getSkippedTests().getAllResults()); } public int totalTests() { return context.getPassedTests().size() + context.getFailedTests().size() + context.getSkippedTests().size(); } public Set getPassedTests() { return context.getPassedTests().getAllResults(); } public Set getFailedTests() { return context.getFailedTests().getAllResults(); } public Set getSkippedTests() { return context.getSkippedTests().getAllResults(); } } private static void display(String prefix, Set results) { System.err.println(prefix + " combos"); results .stream() .peek(each -> System.err.println( "Retried ? " + each.wasRetried() + " test data " + Arrays.toString( each.getParameters()))) .map(each -> Arrays.toString(each.getParameters())) .forEach(System.err::println); } } ```
Sample Test runner to reproduce the bug ```java import static org.assertj.core.api.Assertions.assertThat; import com.rationaleemotions.dp.MyTestCase.CoreListener; import com.rationaleemotions.dp.MyTestCase.ToggleDataProvider; import java.util.Arrays; import org.assertj.core.api.SoftAssertions; import org.testng.ITestResult; import org.testng.TestNG; import org.testng.annotations.Test; public class TestRunner { @Test public void runTestWithDataProviderInParallel() { runTest(true); } @Test public void runTestWithDataProviderInSequential() { runTest(false); } private static void runTest(boolean isParallel) { TestNG testng = new TestNG(); testng.setTestClasses(new Class[]{MyTestCase.class}); CoreListener listener = new CoreListener(); ToggleDataProvider transformer = new ToggleDataProvider(isParallel); testng.addListener(listener); testng.addListener(transformer); testng.setVerbose(2); testng.run(); SoftAssertions softly = new SoftAssertions(); softly.assertThat(listener.totalTests()) .withFailMessage("We should have had 9 test results in total.") .isEqualTo(9); //Assert passed tests softly.assertThat(listener.getPassedTests()) .withFailMessage("We should have had ONLY 1 passed test.") .hasSize(1); listener.getPassedTests() .forEach(each -> softly.assertThat(each.wasRetried()) .withFailMessage("Passed test " + stringify(each) + " should NOT have been retried") .isFalse()); //Assert failed tests assertThat(listener.getFailedTests()) .withFailMessage("We should have had 2 failed tests.") .hasSize(2); listener.getFailedTests() .forEach(each -> softly.assertThat(each.wasRetried()) .withFailMessage("Failed test " + stringify(each) + " should NOT have been retried") .isFalse()); //Assert skipped tests assertThat(listener.getSkippedTests()) .withFailMessage("We should have had 6 skipped tests due to retries.") .hasSize(6); listener.getSkippedTests() .forEach(each -> softly.assertThat(each.wasRetried()) .withFailMessage("Skipped test " + stringify(each) + " should have been retried") .isTrue()); softly.assertAll(); } private static String stringify(ITestResult itr) { return "(" + Arrays.toString(itr.getParameters()) + ")"; } } ```
console output ```bash [15] This is for Test One 1 [17] This is for Test Three 3 [16] This is for Test Two 2 passed for values [2, Test Two] SKIPPED for values [1, Test One] SKIPPED for values [3, Test Three] [17] This is for Test Three 3 [15] This is for Test One 1 SKIPPED for values [3, Test Three] FAILED for values [1, Test One] [17] This is for Test Three 3 FAILED for values [3, Test Three] PASSED combos Retried ? false test data [2, Test Two] [2, Test Two] FAILED combos Retried ? false test data [1, Test One] [1, Test One] Retried ? false test data [3, Test Three] [3, Test Three] SKIPPED combos Retried ? true test data [3, Test Three] [3, Test Three] Retried ? true test data [3, Test Three] [3, Test Three] Retried ? true test data [1, Test One] [1, Test One] PASSED: com.rationaleemotions.dp.MyTestCase.API_dummyTest(2, "Test Two") FAILED: com.rationaleemotions.dp.MyTestCase.API_dummyTest(1, "Test One") java.lang.AssertionError: expected: 0 but was: 1 at com.rationaleemotions.dp.MyTestCase.API_dummyTest(MyTestCase.java:24) at java.base/java.lang.Thread.run(Thread.java:829) FAILED: com.rationaleemotions.dp.MyTestCase.API_dummyTest(3, "Test Three") java.lang.AssertionError: expected: 0 but was: 1 at com.rationaleemotions.dp.MyTestCase.API_dummyTest(MyTestCase.java:24) at java.base/java.lang.Thread.run(Thread.java:829) RETRIED: com.rationaleemotions.dp.MyTestCase.API_dummyTest(3, "Test Three") java.lang.AssertionError: expected: 0 but was: 1 at com.rationaleemotions.dp.MyTestCase.API_dummyTest(MyTestCase.java:24) at java.base/java.lang.Thread.run(Thread.java:829) RETRIED: com.rationaleemotions.dp.MyTestCase.API_dummyTest(3, "Test Three") java.lang.AssertionError: expected: 0 but was: 1 at com.rationaleemotions.dp.MyTestCase.API_dummyTest(MyTestCase.java:24) at java.base/java.lang.Thread.run(Thread.java:829) RETRIED: com.rationaleemotions.dp.MyTestCase.API_dummyTest(1, "Test One") java.lang.AssertionError: expected: 0 but was: 1 at com.rationaleemotions.dp.MyTestCase.API_dummyTest(MyTestCase.java:24) at java.base/java.lang.Thread.run(Thread.java:829) =============================================== Command line test Tests run: 1, Failures: 2, Skips: 0, Retries: 3 =============================================== =============================================== Command line suite Total tests run: 6, Passes: 1, Failures: 2, Skips: 0, Retries: 3 =============================================== java.lang.AssertionError: We should have had 6 skipped tests due to retries. at com.rationaleemotions.dp.TestRunner.runTest(TestRunner.java:61) at com.rationaleemotions.dp.TestRunner.runTestWithDataProviderInParallel(TestRunner.java:17) [1] This is for Test One 1 SKIPPED for values [1, Test One] [1] This is for Test One 1 SKIPPED for values [1, Test One] [1] This is for Test One 1 SKIPPED for values [1, Test One] [1] This is for Test One 1 FAILED for values [1, Test One] [1] This is for Test Two 2 passed for values [2, Test Two] [1] This is for Test Three 3 SKIPPED for values [3, Test Three] [1] This is for Test Three 3 SKIPPED for values [3, Test Three] [1] This is for Test Three 3 SKIPPED for values [3, Test Three] [1] This is for Test Three 3 FAILED for values [3, Test Three] PASSED combos Retried ? false test data [2, Test Two] [2, Test Two] FAILED combos Retried ? false test data [1, Test One] [1, Test One] Retried ? false test data [3, Test Three] [3, Test Three] SKIPPED combos Retried ? true test data [3, Test Three] [3, Test Three] Retried ? true test data [3, Test Three] [3, Test Three] Retried ? true test data [3, Test Three] [3, Test Three] Retried ? true test data [1, Test One] [1, Test One] Retried ? true test data [1, Test One] [1, Test One] Retried ? true test data [1, Test One] [1, Test One] PASSED: com.rationaleemotions.dp.MyTestCase.API_dummyTest(2, "Test Two") FAILED: com.rationaleemotions.dp.MyTestCase.API_dummyTest(1, "Test One") java.lang.AssertionError: expected: 0 but was: 1 at com.rationaleemotions.dp.MyTestCase.API_dummyTest(MyTestCase.java:24) at com.rationaleemotions.dp.TestRunner.runTest(TestRunner.java:33) at com.rationaleemotions.dp.TestRunner.runTestWithDataProviderInSequential(TestRunner.java:22) FAILED: com.rationaleemotions.dp.MyTestCase.API_dummyTest(3, "Test Three") java.lang.AssertionError: expected: 0 but was: 1 at com.rationaleemotions.dp.MyTestCase.API_dummyTest(MyTestCase.java:24) at com.rationaleemotions.dp.TestRunner.runTest(TestRunner.java:33) at com.rationaleemotions.dp.TestRunner.runTestWithDataProviderInSequential(TestRunner.java:22) RETRIED: com.rationaleemotions.dp.MyTestCase.API_dummyTest(3, "Test Three") java.lang.AssertionError: expected: 0 but was: 1 at com.rationaleemotions.dp.MyTestCase.API_dummyTest(MyTestCase.java:24) at com.rationaleemotions.dp.TestRunner.runTest(TestRunner.java:33) at com.rationaleemotions.dp.TestRunner.runTestWithDataProviderInSequential(TestRunner.java:22) RETRIED: com.rationaleemotions.dp.MyTestCase.API_dummyTest(3, "Test Three") java.lang.AssertionError: expected: 0 but was: 1 at com.rationaleemotions.dp.MyTestCase.API_dummyTest(MyTestCase.java:24) at com.rationaleemotions.dp.TestRunner.runTest(TestRunner.java:33) at com.rationaleemotions.dp.TestRunner.runTestWithDataProviderInSequential(TestRunner.java:22) RETRIED: com.rationaleemotions.dp.MyTestCase.API_dummyTest(3, "Test Three") java.lang.AssertionError: expected: 0 but was: 1 at com.rationaleemotions.dp.MyTestCase.API_dummyTest(MyTestCase.java:24) at com.rationaleemotions.dp.TestRunner.runTest(TestRunner.java:33) at com.rationaleemotions.dp.TestRunner.runTestWithDataProviderInSequential(TestRunner.java:22) RETRIED: com.rationaleemotions.dp.MyTestCase.API_dummyTest(1, "Test One") java.lang.AssertionError: expected: 0 but was: 1 at com.rationaleemotions.dp.MyTestCase.API_dummyTest(MyTestCase.java:24) at com.rationaleemotions.dp.TestRunner.runTest(TestRunner.java:33) at com.rationaleemotions.dp.TestRunner.runTestWithDataProviderInSequential(TestRunner.java:22) RETRIED: com.rationaleemotions.dp.MyTestCase.API_dummyTest(1, "Test One") java.lang.AssertionError: expected: 0 but was: 1 at com.rationaleemotions.dp.MyTestCase.API_dummyTest(MyTestCase.java:24) at com.rationaleemotions.dp.TestRunner.runTest(TestRunner.java:33) at com.rationaleemotions.dp.TestRunner.runTestWithDataProviderInSequential(TestRunner.java:22) RETRIED: com.rationaleemotions.dp.MyTestCase.API_dummyTest(1, "Test One") java.lang.AssertionError: expected: 0 but was: 1 at com.rationaleemotions.dp.MyTestCase.API_dummyTest(MyTestCase.java:24) at com.rationaleemotions.dp.TestRunner.runTest(TestRunner.java:33) at com.rationaleemotions.dp.TestRunner.runTestWithDataProviderInSequential(TestRunner.java:22) =============================================== Command line test Tests run: 1, Failures: 2, Skips: 0, Retries: 6 =============================================== =============================================== Command line suite Total tests run: 9, Passes: 1, Failures: 2, Skips: 0, Retries: 6 =============================================== =============================================== Default Suite Total tests run: 2, Passes: 1, Failures: 1, Skips: 0 =============================================== Process finished with exit code 0 ```

Contribution guidelines

Incase you plan to raise a pull request to fix this issue, please make sure you refer our Contributing section for detailed set of steps.

krmahadevan commented 1 year ago

More details are available in this thread https://groups.google.com/g/testng-users/c/wn00nIBwdaM/m/vMDBquUAAgAJ