nose2 teamcity test reporter

jredl-va commented 9 years ago

The current team city reporters do not yet include a plugin that supports nose2: https://github.com/nose-devs/nose2

jredl-va commented 9 years ago

Here is a first cut implementation of one, that is loosely modelled after the nose_report plugin.

# coding=utf-8
import sys
import datetime

from teamcity import is_running_under_teamcity
from teamcity.common import is_string, split_output, limit_output, get_class_fullname, convert_error_to_string
from teamcity.messages import TeamcityServiceMessages

from nose2 import events, result

# from nose.util.ln
def _ln(label):
    label_len = len(label) + 2
    chunk = (70 - label_len) // 2
    out = '%s %s %s' % ('-' * chunk, label, '-' * chunk)
    pad = 70 - len(out)
    if pad > 0:
        out = out + ('-' * pad)
    return out

_captured_output_start_marker = _ln('>> begin captured stdout <<') + "\n"
_captured_output_end_marker = "\n" + _ln('>> end captured stdout <<')

_real_stdout = sys.stdout

class TeamcityReport(events.Plugin):
    alwaysOn = is_running_under_teamcity()
    configSection = 'teamcity-report'
    commandLineSwitch = (None, 'teamcity-report', 'Enable Team City test reporting.')

    def __init__(self):
        self.messages = TeamcityServiceMessages(_real_stdout)
        self.test_started_datetime_map = {}
        self.enabled = False

    def get_test_id(self, test):
        """
        Get a test identifier for the current test
        """
        if is_string(test):
            return test

        test_id = test.id()
        real_test = getattr(test, "test", test)
        real_test_class_name = get_class_fullname(real_test)

        test_arg = getattr(real_test, "arg", tuple())
        if (type(test_arg) is tuple or type(test_arg) is list) and len(test_arg) > 0:
            # As written in nose.case.FunctionTestCase#__str__ or nose.case.MethodTestCase#__str__
            test_arg_str = "%s" % (test_arg,)
            if test_id.endswith(test_arg_str):
                # Replace '.' in test args with '_' to preserve test hierarchy on TeamCity
                test_id = test_id[:len(test_id) - len(test_arg_str)] + test_arg_str.replace('.', '_')

        # Force test_id for doctests
        if real_test_class_name != "doctest.DocTestCase" and real_test_class_name != "nose.plugins.doctests.DocTestCase":
            desc = test.shortDescription()
            if desc and desc != test.id():
                return "%s (%s)" % (test_id, desc.replace('.', '_'))

        return test_id

    def report_fail(self, test, fail_type, err):
        """
        Report a test failure to teamcity
        """
        if is_string(err[2]):
            details = err[2]
        else:
            details = convert_error_to_string(err)

        test_id = self.get_test_id(test)
        start_index = details.find(_captured_output_start_marker)
        end_index = details.find(_captured_output_end_marker)

        if 0 <= start_index < end_index:
            captured_output = details[start_index + len(_captured_output_start_marker):end_index]
            details = details[:start_index] + details[end_index + len(_captured_output_end_marker):]

            for chunk in split_output(limit_output(captured_output)):
                self.messages.testStdOut(test_id, chunk, flowId=test_id)

        self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id)

    def reportError(self, event):
        """
        :param event: https://nose2.readthedocs.org/en/latest/dev/event_reference.html#nose2.events.TestOutcomeEvent
        Notify teamcity that an error was encountered running a test.
        """
        test_outcome = event.testEvent
        test_id = self.get_test_id(test_outcome.test)

        if test_outcome.outcome == result.SKIP:
            self.messages.testIgnored(test_id, message="Skipped", flowId=test_id)
        else:
            self.report_fail(test_outcome.test, 'Error', test_outcome.exc_info)

    def reportFailure(self, event):
        """
        :param event: https://nose2.readthedocs.org/en/latest/dev/event_reference.html#nose2.events.TestOutcomeEvent
        Notify teamcity that a test assertion failed.
        """
        self.report_fail(event.testEvent.test, 'Failure', event.testEvent.exc_info)

    def startTest(self, event):
        """
        :param event: https://nose2.readthedocs.org/en/latest/dev/event_reference.html#nose2.events.StartTestRunEvent
        Notify teamcity that a test has started.
        """
        test_id = self.get_test_id(event.test)
        self.test_started_datetime_map[test_id] = event.startTime
        self.messages.testStarted(test_id, captureStandardOutput='true', flowId=test_id)

    def stopTest(self, event):
        """
        :param event: https://nose2.readthedocs.org/en/latest/dev/event_reference.html#nose2.events.StopTestEvent
        Notify teamcity that a test has finished.
        """
        test_id = self.get_test_id(event.test)
        stop_time = datetime.datetime.fromtimestamp(event.stopTime)
        start_time = datetime.datetime.fromtimestamp(self.test_started_datetime_map[test_id])
        self.messages.testFinished(test_id, testDuration=(stop_time - start_time), flowId=test_id)

shalupov commented 9 years ago

Thank you!

jredl-va commented 9 years ago

Updated, slightly as we encountered a slight error with the plugin when running in a parallel test run.

kaneda commented 8 years ago

@shalupov any plan to resolve this?

shalupov commented 8 years ago

@kaneda Unfortunately no immediate plan. for proper nose2 support (besides the code above) I need to port/write test suite too and remove code duplication + test it on all supported Python/OS versions. You're the second person who are interested in this, so chances are higher now :)