taverntesting / tavern

A command-line tool and Python library and Pytest plugin for automated testing of RESTful APIs, with a simple, concise and flexible YAML-based syntax
https://taverntesting.github.io/
MIT License
1.02k stars 193 forks source link

Version 2.0.x broke tavern tests which were working with version 1.25.2 #842

Closed alexbigkid closed 1 year ago

alexbigkid commented 1 year ago

Looks like there was a breaking change with the 2.0 release. Could you please take a look what is breaking in the new release? I am getting following error when running tests based on tavern test framework.

=========================================================================================================== test session starts ============================================================================================================
platform darwin -- Python 3.9.16, pytest-7.2.1, pluggy-1.0.0
rootdir: /
plugins: tavern-2.0.1
collected 1 item                                                                                                                                                                                                                           

test_accept_valid_fleet_provision.tavern.yml F                                                                                                                                                                                       [100%]

================================================================================================================= FAILURES =================================================================================================================
_______________________________ /Users/alexberger/dev/aai/cloud-infrastructure/integration-tests/fleet-provisioning/test_accept_valid_fleet_provision.tavern.yml::Test accept valid fleet provision request ________________________________

cls = <class '_pytest.runner.CallInfo'>, func = <function call_runtest_hook.<locals>.<lambda> at 0x10bf38b80>, when = 'call', reraise = (<class '_pytest.outcomes.Exit'>, <class 'KeyboardInterrupt'>)

    @classmethod
    def from_call(
        cls,
        func: "Callable[[], TResult]",
        when: "Literal['collect', 'setup', 'call', 'teardown']",
        reraise: Optional[
            Union[Type[BaseException], Tuple[Type[BaseException], ...]]
        ] = None,
    ) -> "CallInfo[TResult]":
        """Call func, wrapping the result in a CallInfo.

        :param func:
            The function to call. Called without arguments.
        :param when:
            The phase in which the function is called.
        :param reraise:
            Exception or exceptions that shall propagate if raised by the
            function, instead of being wrapped in the CallInfo.
        """
        excinfo = None
        start = timing.time()
        precise_start = timing.perf_counter()
        try:
>           result: Optional[TResult] = func()

../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/_pytest/runner.py:339: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

>       lambda: ihook(item=item, **kwds), when=when, reraise=reraise
    )

../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/_pytest/runner.py:260: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <_HookCaller 'pytest_runtest_call'>, args = (), kwargs = {'item': <YamlItem Test accept valid fleet provision request>}, argname = 'item', firstresult = False

    def __call__(self, *args, **kwargs):
        if args:
            raise TypeError("hook calling supports only keyword arguments")
        assert not self.is_historic()

        # This is written to avoid expensive operations when not needed.
        if self.spec:
            for argname in self.spec.argnames:
                if argname not in kwargs:
                    notincall = tuple(set(self.spec.argnames) - kwargs.keys())
                    warnings.warn(
                        "Argument(s) {} which are declared in the hookspec "
                        "can not be found in this hook call".format(notincall),
                        stacklevel=2,
                    )
                    break

            firstresult = self.spec.opts.get("firstresult")
        else:
            firstresult = False

>       return self._hookexec(self.name, self.get_hookimpls(), kwargs, firstresult)

../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/pluggy/_hooks.py:265: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <_pytest.config.PytestPluginManager object at 0x10bde3b80>, hook_name = 'pytest_runtest_call'
methods = [<HookImpl plugin_name='runner', plugin=<module '_pytest.runner' from '/Users/alexberger/.pyenv/versions/fp/lib/python....threadexception' from '/Users/alexberger/.pyenv/versions/fp/lib/python3.9/site-packages/_pytest/threadexception.py'>>]
kwargs = {'item': <YamlItem Test accept valid fleet provision request>}, firstresult = False

    def _hookexec(self, hook_name, methods, kwargs, firstresult):
        # called from all hookcaller instances.
        # enable_tracing will set its own wrapping function at self._inner_hookexec
>       return self._inner_hookexec(hook_name, methods, kwargs, firstresult)

../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/pluggy/_manager.py:80: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

hook_name = 'pytest_runtest_call'
hook_impls = [<HookImpl plugin_name='runner', plugin=<module '_pytest.runner' from '/Users/alexberger/.pyenv/versions/fp/lib/python....threadexception' from '/Users/alexberger/.pyenv/versions/fp/lib/python3.9/site-packages/_pytest/threadexception.py'>>]
caller_kwargs = {'item': <YamlItem Test accept valid fleet provision request>}, firstresult = False

    def _multicall(hook_name, hook_impls, caller_kwargs, firstresult):
        """Execute a call into multiple python functions/methods and return the
        result(s).

        ``caller_kwargs`` comes from _HookCaller.__call__().
        """
        __tracebackhide__ = True
        results = []
        excinfo = None
        try:  # run impl and wrapper setup functions in a loop
            teardowns = []
            try:
                for hook_impl in reversed(hook_impls):
                    try:
                        args = [caller_kwargs[argname] for argname in hook_impl.argnames]
                    except KeyError:
                        for argname in hook_impl.argnames:
                            if argname not in caller_kwargs:
                                raise HookCallError(
                                    f"hook call must provide argument {argname!r}"
                                )

                    if hook_impl.hookwrapper:
                        try:
                            gen = hook_impl.function(*args)
                            next(gen)  # first yield
                            teardowns.append(gen)
                        except StopIteration:
                            _raise_wrapfail(gen, "did not yield")
                    else:
                        res = hook_impl.function(*args)
                        if res is not None:
                            results.append(res)
                            if firstresult:  # halt further impl calls
                                break
            except BaseException:
                excinfo = sys.exc_info()
        finally:
            if firstresult:  # first result hooks return a single value
                outcome = _Result(results[0] if results else None, excinfo)
            else:
                outcome = _Result(results, excinfo)

            # run all wrapper post-yield blocks
            for gen in reversed(teardowns):
                try:
                    gen.send(outcome)
                    _raise_wrapfail(gen, "has second yield")
                except StopIteration:
                    pass

>           return outcome.get_result()

../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/pluggy/_callers.py:60: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <pluggy._result._Result object at 0x10beea370>

    def get_result(self):
        """Get the result(s) for this hook call.

        If the hook was marked as a ``firstresult`` only a single value
        will be returned otherwise a list of results.
        """
        __tracebackhide__ = True
        if self._excinfo is None:
            return self._result
        else:
            ex = self._excinfo
>           raise ex[1].with_traceback(ex[2])

../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/pluggy/_result.py:60: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

hook_name = 'pytest_runtest_call'
hook_impls = [<HookImpl plugin_name='runner', plugin=<module '_pytest.runner' from '/Users/alexberger/.pyenv/versions/fp/lib/python....threadexception' from '/Users/alexberger/.pyenv/versions/fp/lib/python3.9/site-packages/_pytest/threadexception.py'>>]
caller_kwargs = {'item': <YamlItem Test accept valid fleet provision request>}, firstresult = False

    def _multicall(hook_name, hook_impls, caller_kwargs, firstresult):
        """Execute a call into multiple python functions/methods and return the
        result(s).

        ``caller_kwargs`` comes from _HookCaller.__call__().
        """
        __tracebackhide__ = True
        results = []
        excinfo = None
        try:  # run impl and wrapper setup functions in a loop
            teardowns = []
            try:
                for hook_impl in reversed(hook_impls):
                    try:
                        args = [caller_kwargs[argname] for argname in hook_impl.argnames]
                    except KeyError:
                        for argname in hook_impl.argnames:
                            if argname not in caller_kwargs:
                                raise HookCallError(
                                    f"hook call must provide argument {argname!r}"
                                )

                    if hook_impl.hookwrapper:
                        try:
                            gen = hook_impl.function(*args)
                            next(gen)  # first yield
                            teardowns.append(gen)
                        except StopIteration:
                            _raise_wrapfail(gen, "did not yield")
                    else:
>                       res = hook_impl.function(*args)

../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/pluggy/_callers.py:39: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

item = <YamlItem Test accept valid fleet provision request>

    def pytest_runtest_call(item: Item) -> None:
        _update_current_test_var(item, "call")
        try:
            del sys.last_type
            del sys.last_value
            del sys.last_traceback
        except AttributeError:
            pass
        try:
            item.runtest()
        except Exception as e:
            # Store trace info to allow postmortem debugging
            sys.last_type = type(e)
            sys.last_value = e
            assert e.__traceback__ is not None
            # Skip *this* frame
            sys.last_traceback = e.__traceback__.tb_next
>           raise e

../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/_pytest/runner.py:175: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

item = <YamlItem Test accept valid fleet provision request>

    def pytest_runtest_call(item: Item) -> None:
        _update_current_test_var(item, "call")
        try:
            del sys.last_type
            del sys.last_value
            del sys.last_traceback
        except AttributeError:
            pass
        try:
>           item.runtest()

../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/_pytest/runner.py:167: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <YamlItem Test accept valid fleet provision request>

    def runtest(self):
        self.global_cfg = load_global_cfg(self.config)

        load_plugins(self.global_cfg)

        # INTERNAL
        xfail = self.spec.get("_xfail", False)

        try:
            fixture_values = self._load_fixture_values()
            self.global_cfg.variables.update(fixture_values)

            call_hook(
                self.global_cfg,
                "pytest_tavern_beta_before_every_test_run",
                test_dict=self.spec,
                variables=self.global_cfg.variables,
            )

>           verify_tests(self.spec)

../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/tavern/_core/pytest/item.py:194: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

test_spec = {'test_name': 'Test accept valid fleet provision request', 'includes': [{'name': 'device test data valid', 'descriptio...emplate/provision/json/accepted', 'json': {'deviceConfiguration': {}, 'thingName': '{deviceUuid:s}'}, 'timeout': 10}}]}
with_plugins = True

    def verify_tests(test_spec, with_plugins=True):
        """Verify that a specific test block is correct

        Todo:
            Load schema file once. Requires some caching of the file

        Args:
            test_spec (dict): Test in dictionary form

        Raises:
            BadSchemaError: Schema did not match
        """
        here = os.path.dirname(os.path.abspath(__file__))

        schema_filename = os.path.join(here, "tests.jsonschema.yaml")
        schema = load_schema_file(schema_filename, with_plugins)

>       verify_jsonschema(test_spec, schema)

../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/tavern/_core/schema/files.py:148: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

to_verify = {'test_name': 'Test accept valid fleet provision request', 'includes': [{'name': 'device test data valid', 'descriptio...emplate/provision/json/accepted', 'json': {'deviceConfiguration': {}, 'thingName': '{deviceUuid:s}'}, 'timeout': 10}}]}
schema = {'$schema': 'http://json-schema.org/draft-07/schema#', '$id': 'https://raw.githubusercontent.com/taverntesting/tavern/...': False, 'required': ['username'], 'properties': {'username': {'type': 'string'}, 'password': {'type': 'string'}}}}}}}

    def verify_jsonschema(to_verify, schema):
        """Verify a generic file against a given jsonschema

        Args:
            to_verify (dict): Filename of source tests to check
            schema (dict): Schema to verify against

        Raises:
            BadSchemaError: Schema did not match
        """

        # pylint: disable=too-many-locals

        validator = CustomValidator(schema)

        try:
            validator.validate(to_verify)
        except jsonschema.ValidationError as e:
            real_context = []

            # ignore these strings because they're red herrings
            for c in e.context:
                description = c.schema.get("description", "<no description>")
                if description == "Reference to another stage from an included config file":
                    continue

                instance = c.instance
                filename = get_stage_filename(instance)
                if filename is None:
                    # Depending on what block raised the error, it mightbe difficult to tell what it was, so check the parent too
                    instance = e.instance
                    filename = get_stage_filename(instance)

                if filename:
                    with open(filename, "r", encoding="utf-8") as infile:
                        n_lines = len(infile.readlines())

                    first_line, last_line, _ = get_stage_lines(instance)
                    first_line = max(first_line - 2, 0)
                    last_line = min(last_line + 2, n_lines)

                    reg = re.compile(r"^\s*$")

                    lines = read_relevant_lines(instance, first_line, last_line)
                    lines = [line for line in lines if not reg.match(line.strip())]
                    content = "\n".join(list(lines))
                    real_context.append(
                        f"""
    {c.message}
    {filename}: line {first_line}-{last_line}:

    {content}
    """
                    )
                else:
                    real_context.append(
                        f"""
    {c.message}

    <error: unable to find input file for context>
    """
                    )

            msg = "\n---\n" + "\n---\n".join([str(i) for i in real_context])
>           raise BadSchemaError(msg) from None
E           tavern._core.exceptions.BadSchemaError: 
E           ---
E           
E           {'topic': '$aws/certificates/create/json/accepted', 'json': {'certificateId': <tavern._core.loader.AnythingSentinel object at 0x10646d8b0>, 'certificateOwnershipToken': <tavern._core.loader.AnythingSentinel object at 0x10646d8b0>, 'certificatePem': <tavern._core.loader.AnythingSentinel object at 0x10646d8b0>, 'privateKey': <tavern._core.loader.AnythingSentinel object at 0x10646d8b0>}, 'verify_response_with': {'function': 'save_certificate:save_certificate'}, 'save': {'json': {'createCertResp_certificateId': 'certificateId', 'createCertResp_certificatePem': 'certificatePem', 'createCertResp_privateKey': 'privateKey', 'createCertResp_certificateOwnershipToken': 'certificateOwnershipToken'}}, 'timeout': 10} is not valid under any of the given schemas
E           /Users/alexberger/dev/aai/cloud-infrastructure/integration-tests/fleet-provisioning/test_accept_valid_fleet_provision.tavern.yml: line 23-44:
E           
E                     json: {}
E                 mqtt_response:
E                     topic: $aws/certificates/create/json/accepted
E                     json:
E                         certificateId: !anything
E                         certificateOwnershipToken: !anything
E                         certificatePem: !anything
E                         privateKey: !anything
E                     verify_response_with:
E                         function: save_certificate:save_certificate
E                     save:
E                         json:
E                             createCertResp_certificateId: certificateId
E                             createCertResp_certificatePem: certificatePem
E                             createCertResp_privateKey: privateKey
E                             createCertResp_certificateOwnershipToken: certificateOwnershipToken
E                     timeout: 10
E               - name: Provision Certificate
E                 mqtt_publish:

../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/tavern/_core/schema/jsonschema.py:171: BadSchemaError
michaelboulton commented 1 year ago

Can you give an example of the test you were running to get the error?

michaelboulton commented 1 year ago

I'm guessing this was due to the accidental breaking changes to 'save' in MQTT blocks, this should be fixed in 2.0.2