search

serenity-bdd / serenity-core

Serenity BDD is a test automation library designed to make writing automated acceptance tests easier, and more fun.
http://serenity-bdd.info
Other
715 stars 513 forks source link

The log is not detecting the failure (locally)after upgrading to serenityCoreVersion = '4.0.1' #3252

Open jerrin89 opened 10 months ago

jerrin89 commented 10 months ago

When I upgraded to the latest version (4.0.1) of Serenity, I am not able to see the step-by-step console log locally when a step actually fails (I intentionally used the wrong element locator in one step to make it fail). The log is not detecting the failure, and it shows that all steps passed.

Below are the config slf4jVersion = '2.0.9' serenityCoreVersion = '4.0.1' serenityCucumberVersion = '4.0.1' serenityRestAssuredVersion = '4.0.1' junitVersion = '4.13.1' assertJVersion = '3.8.0' logbackVersion = '1.4.11'

wakaleo commented 10 months ago

Can you provide a sample project to reproduce the issue?

jerrin89 commented 10 months ago

@wakaleo defaultTasks 'clean', 'test', 'aggregate'

File bddLogger = new File(projectDir, "failed-tests.log") File retryLogger = new File(projectDir, "retried-tests.log")

// Default to false if not provided def includePassedTests = project.findProperty('includePassedTests')?.toBoolean() ?: false repositories { flatDir { dirs './dependencies' } mavenLocal() mavenCentral() maven { url = uri('https://nexus.internal-unilink.co.uk/repository/UCase-Maven') allowInsecureProtocol = true } }

buildscript { repositories { mavenLocal() mavenCentral() maven { url "https://plugins.gradle.org/m2/" } } dependencies { classpath "net.serenity-bdd:serenity-gradle-plugin:4.0.0-beta-3" classpath "net.serenity-bdd:serenity-single-page-report:4.0.0-beta-3" } }

apply plugin: 'java' apply plugin: 'eclipse' apply plugin: 'idea' apply plugin: "net.serenity-bdd.serenity-gradle-plugin" //apply plugin: 'net.serenity-bdd.aggregator' //apply from: "$rootDir/gradle/libraries.gradle"

configurations { nexusCompile }

ext { slf4jVersion = '2.0.9' serenityCoreVersion = '4.0.1' serenityCucumberVersion = '4.0.1' serenityRestAssuredVersion = '4.0.1' junitVersion = '4.13.1' assertJVersion = '3.8.0' logbackVersion = '1.4.11'

// serenityJiraVersion = '1.12.0'

sql2oVersion = '1.6.0'
mssqlJdbcVersion = '9.4.0.jre11'
lombokVersion = '1.18.28'

jacksonJsrVersion = '2.12.5'
jsonSchemaValidatorVersion = '4.1.0'
ucaseApiCommonsVersion = "0.5.55"
jsonVersion = '20201115'
hikariCpVersion = '5.0.0'

rabbitMQClientversion = '5.10.0'
springKafkaVersion = '2.8.2'

}

dependencies { testImplementation "ch.qos.logback:logback-classic:${logbackVersion}", "org.springframework.kafka:spring-kafka:${springKafkaVersion}", "org.springframework.kafka:spring-kafka-test:${springKafkaVersion}", "net.serenity-bdd:serenity-core:${serenityCoreVersion}", "net.serenity-bdd:serenity-cucumber:${serenityCucumberVersion}", "net.serenity-bdd:serenity-screenplay:${serenityCoreVersion}", "net.serenity-bdd:serenity-screenplay-webdriver:${serenityCoreVersion}", "net.serenity-bdd:serenity-ensure:${serenityCoreVersion}", "net.serenity-bdd:serenity-rest-assured:${serenityRestAssuredVersion}", "junit:junit:${junitVersion}", "org.assertj:assertj-core:${assertJVersion}", "org.json:json:${jsonVersion}", "com.rabbitmq:amqp-client:${rabbitMQClientversion}", "io.rest-assured:json-schema-validator:${jsonSchemaValidatorVersion}", "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonJsrVersion}", "org.slf4j:slf4j-api:$slf4jVersion", "org.sql2o:sql2o:${sql2oVersion}", "com.microsoft.sqlserver:mssql-jdbc:${mssqlJdbcVersion}", "org.projectlombok:lombok:${lombokVersion}", "com.fasterxml.jackson.module:jackson-module-jsonSchema:2.9.0", "com.zaxxer:HikariCP:${hikariCpVersion}", "com.unilink.ucase:api-commons:${ucaseApiCommonsVersion}", "com.unilink.ucase.api-commons-models:authorisation-service-model:latest" // "com.unilink.ucase.api-commons-models:party-event-service-model:0.0.16", // "net.serenity-bdd:serenity-jira-plugin:${serenityJiraVersion}" ,

testAnnotationProcessor "org.projectlombok:lombok:${lombokVersion}"

} def isLocalEnv = System.getProperty("environment") == "local" ? true : false

test { testLogging.showStandardStreams = true systemProperties System.getProperties() failFast = false

maxHeapSize = "8192m"

}

gradle.startParameter.continueOnFailure = true

tasks.withType(Test) { if (System.getenv("FORK_COUNT") != null) { println 'Setting maxParallelForks...' maxParallelForks = System.getenv("FORK_COUNT").toInteger() } else { maxParallelForks = (Runtime.runtime.availableProcessors().intdiv(2) ?: 1) - 1 } println 'maxParallelForks set to: ' + maxParallelForks

def retry = System.getenv('CRON') != "true" && System.getenv('GRADLE_TEST_RETRY') != "false"
println("test retry = " + retry)
ext.failedTests = []
if (retry) {
    if (name.endsWith("Rerun")) {
        return
    }

    def rerunTask = tasks.register("${name}Rerun", Test) {
        // Enabled only when there are failures
        enabled = false
        failFast = true // ¯\_(ツ)_/¯
        outputs.upToDateWhen { false }
    }

    ignoreFailures = true

    finalizedBy(rerunTask)
    afterTest { desc, result ->
        if (TestResult.ResultType.FAILURE == result.resultType) {

            def standardOutput = new ByteArrayOutputStream()
            println "*** Running API Healthcheck"

            def code =  exec {
                if(isLocalEnv) {
                    commandLine "nginx-docker", "api"
                    ignoreExitValue(true)
                }else {
                    commandLine  "bash", "/opt/docker-nginx-compose/nginx-docker", "api"
                    ignoreExitValue(true)
                }
            }

            ext.output = {
                return standardOutput.toString()
            }

            if (code.exitValue == 1) {
                throw new GradleException("API Healthcheck FAILED")
            }
            println 're-running = ' + "${desc.name}"
            retryLogger.append("${desc.className}::${desc.name}\n")
            rerunTask.configure {
                enabled = true
                filter.includeTestsMatching("*${desc.name}*")

            }
        }
    }
}

afterTest { desc, result ->
    if (result.resultType == TestResult.ResultType.FAILURE) {

        String failedTest = "${desc.className}::${desc.name}"
        failedTests << failedTest
    }
}

gradle.buildFinished { buildResult ->
    if (buildResult.failure) {
        if (!failedTests.empty) {
            failedTests.each { failedTest ->
                bddLogger.append(failedTest + "\n")
            }
        }
        if (!includePassedTests) {
            file('target/site/serenity').eachFile { testcase ->
                if (testcase.name.contains(".json")) {
                    String testId = testcase.name.replace(".json", "");
                    String fileContents = new File(projectDir, 'target/site/serenity/' + testcase.name).text

                    // Remove files that do not contain a failure for the report
                    if (!fileContents.contains('"FAILURE"') && !fileContents.contains('"BROKEN"') && !fileContents.contains('"ERROR"')) {
                        delete "target/site/serenity/" + testId + ".json"
                        delete "target/site/serenity/" + testId + ".html"
                        delete "target/site/serenity/SERENITY-JUNIT-" + testId + ".xml"
                    }
                }
            }
        }
    }
}

}

serenity { reports = ["single-page-html"] }

jerrin89 commented 10 months ago

<?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

4.0.0 
<groupId>net.serenitybdd.starter</groupId>
<artifactId>cucumber-starter</artifactId>
<version>1.0.0-SNAPSHOT</version>
<packaging>jar</packaging>

<name>Sample Serenity BDD project using Cucumber</name>

<properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <serenity.version>4.0.1</serenity.version>
    <encoding>UTF-8</encoding>
    <tags></tags>
    <webdriver.base.url></webdriver.base.url>
</properties>
<dependencies>
    <dependency>
        <groupId>net.serenity-bdd</groupId>
        <artifactId>serenity-core</artifactId>
        <version>${serenity.version}</version>
        <scope>test</scope>
    </dependency>
    <dependency>
        <groupId>net.serenity-bdd</groupId>
        <artifactId>serenity-cucumber</artifactId>
        <version>${serenity.version}</version>
        <scope>test</scope>
    </dependency>
    <dependency>
        <groupId>net.serenity-bdd</groupId>
        <artifactId>serenity-screenplay</artifactId>
        <version>${serenity.version}</version>
        <scope>test</scope>
    </dependency>
    <dependency>
        <groupId>net.serenity-bdd</groupId>
        <artifactId>serenity-screenplay-webdriver</artifactId>
        <version>${serenity.version}</version>
        <scope>test</scope>
    </dependency>
    <dependency>
        <groupId>net.serenity-bdd</groupId>
        <artifactId>serenity-ensure</artifactId>
        <version>${serenity.version}</version>
        <scope>test</scope>
    </dependency>
    <dependency>
        <groupId>org.junit.platform</groupId>
        <artifactId>junit-platform-launcher</artifactId>
        <version>1.8.1</version>
        <scope>test</scope>
    </dependency>
    <dependency>
        <groupId>org.junit.jupiter</groupId>
        <artifactId>junit-jupiter-engine</artifactId>
        <version>5.8.1</version>
        <scope>test</scope>
    </dependency>
    <dependency>
        <groupId>org.junit.vintage</groupId>
        <artifactId>junit-vintage-engine</artifactId>
        <version>5.8.1</version>
        <scope>test</scope>
    </dependency>
    <dependency>
        <groupId>ch.qos.logback</groupId>
        <artifactId>logback-classic</artifactId>
        <version>1.0.13</version>
    </dependency>
    <dependency>
        <groupId>org.assertj</groupId>
        <artifactId>assertj-core</artifactId>
        <version>3.6.2</version>
        <scope>test</scope>
    </dependency>
</dependencies>
<build>
    <plugins>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-surefire-plugin</artifactId>
            <version>3.0.0-M5</version>
            <configuration>
                <skip>true</skip>
            </configuration>
        </plugin>
        <plugin>
            <artifactId>maven-failsafe-plugin</artifactId>
            <version>3.0.0-M5</version>
            <configuration>
                <includes>
                    <include>**/*Test.java</include>
                    <include>**/Test*.java</include>
                    <include>**/*TestSuite.java</include>
                    <include>**/When*.java</include>
                </includes>
                <systemPropertyVariables>
                    <webdriver.base.url>${webdriver.base.url}</webdriver.base.url>
                </systemPropertyVariables>
                <parallel>classes</parallel>
                <parallel>methods</parallel>
                <useUnlimitedThreads>true</useUnlimitedThreads>
            </configuration>
            <executions>
                <execution>
                    <goals>
                        <goal>integration-test</goal>
                        <goal>verify</goal>
                    </goals>
                </execution>
            </executions>
        </plugin>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-compiler-plugin</artifactId>
            <version>3.8.1</version>
            <configuration>
                <source>1.8</source>
                <target>1.8</target>
            </configuration>
        </plugin>
        <plugin>
            <groupId>net.serenity-bdd.maven.plugins</groupId>
            <artifactId>serenity-maven-plugin</artifactId>
            <version>${serenity.version}</version>
            <configuration>
                <tags>${tags}</tags>
            </configuration>
            <executions>
                <execution>
                    <id>serenity-reports</id>
                    <phase>post-integration-test</phase>
                    <goals>
                        <goal>aggregate</goal>
                    </goals>
                </execution>
            </executions>
        </plugin>
    </plugins>
</build>

jerrin89 commented 10 months ago

@wakaleo the above are the confiqs is there anything you can do with out the sample project??

wakaleo commented 10 months ago

If you can provide a sample project that reproduces the issue, someone might be able to pick it up and take a look. If you cannot do that and need project-specific support, we can do that as part of the commercial support packages (https://www.serenity-dojo.com/serenity-bdd-enterprise-support).