Looks like there was a breaking change with the 2.0 release. Could you please take a look what is breaking in the new release? I am getting following error when running tests based on tavern test framework.
=========================================================================================================== test session starts ============================================================================================================
platform darwin -- Python 3.9.16, pytest-7.2.1, pluggy-1.0.0
rootdir: /
plugins: tavern-2.0.1
collected 1 item
test_accept_valid_fleet_provision.tavern.yml F [100%]
================================================================================================================= FAILURES =================================================================================================================
_______________________________ /Users/alexberger/dev/aai/cloud-infrastructure/integration-tests/fleet-provisioning/test_accept_valid_fleet_provision.tavern.yml::Test accept valid fleet provision request ________________________________
cls = <class '_pytest.runner.CallInfo'>, func = <function call_runtest_hook.<locals>.<lambda> at 0x10bf38b80>, when = 'call', reraise = (<class '_pytest.outcomes.Exit'>, <class 'KeyboardInterrupt'>)
@classmethod
def from_call(
cls,
func: "Callable[[], TResult]",
when: "Literal['collect', 'setup', 'call', 'teardown']",
reraise: Optional[
Union[Type[BaseException], Tuple[Type[BaseException], ...]]
] = None,
) -> "CallInfo[TResult]":
"""Call func, wrapping the result in a CallInfo.
:param func:
The function to call. Called without arguments.
:param when:
The phase in which the function is called.
:param reraise:
Exception or exceptions that shall propagate if raised by the
function, instead of being wrapped in the CallInfo.
"""
excinfo = None
start = timing.time()
precise_start = timing.perf_counter()
try:
> result: Optional[TResult] = func()
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/_pytest/runner.py:339:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> lambda: ihook(item=item, **kwds), when=when, reraise=reraise
)
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/_pytest/runner.py:260:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <_HookCaller 'pytest_runtest_call'>, args = (), kwargs = {'item': <YamlItem Test accept valid fleet provision request>}, argname = 'item', firstresult = False
def __call__(self, *args, **kwargs):
if args:
raise TypeError("hook calling supports only keyword arguments")
assert not self.is_historic()
# This is written to avoid expensive operations when not needed.
if self.spec:
for argname in self.spec.argnames:
if argname not in kwargs:
notincall = tuple(set(self.spec.argnames) - kwargs.keys())
warnings.warn(
"Argument(s) {} which are declared in the hookspec "
"can not be found in this hook call".format(notincall),
stacklevel=2,
)
break
firstresult = self.spec.opts.get("firstresult")
else:
firstresult = False
> return self._hookexec(self.name, self.get_hookimpls(), kwargs, firstresult)
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/pluggy/_hooks.py:265:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <_pytest.config.PytestPluginManager object at 0x10bde3b80>, hook_name = 'pytest_runtest_call'
methods = [<HookImpl plugin_name='runner', plugin=<module '_pytest.runner' from '/Users/alexberger/.pyenv/versions/fp/lib/python....threadexception' from '/Users/alexberger/.pyenv/versions/fp/lib/python3.9/site-packages/_pytest/threadexception.py'>>]
kwargs = {'item': <YamlItem Test accept valid fleet provision request>}, firstresult = False
def _hookexec(self, hook_name, methods, kwargs, firstresult):
# called from all hookcaller instances.
# enable_tracing will set its own wrapping function at self._inner_hookexec
> return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/pluggy/_manager.py:80:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
hook_name = 'pytest_runtest_call'
hook_impls = [<HookImpl plugin_name='runner', plugin=<module '_pytest.runner' from '/Users/alexberger/.pyenv/versions/fp/lib/python....threadexception' from '/Users/alexberger/.pyenv/versions/fp/lib/python3.9/site-packages/_pytest/threadexception.py'>>]
caller_kwargs = {'item': <YamlItem Test accept valid fleet provision request>}, firstresult = False
def _multicall(hook_name, hook_impls, caller_kwargs, firstresult):
"""Execute a call into multiple python functions/methods and return the
result(s).
``caller_kwargs`` comes from _HookCaller.__call__().
"""
__tracebackhide__ = True
results = []
excinfo = None
try: # run impl and wrapper setup functions in a loop
teardowns = []
try:
for hook_impl in reversed(hook_impls):
try:
args = [caller_kwargs[argname] for argname in hook_impl.argnames]
except KeyError:
for argname in hook_impl.argnames:
if argname not in caller_kwargs:
raise HookCallError(
f"hook call must provide argument {argname!r}"
)
if hook_impl.hookwrapper:
try:
gen = hook_impl.function(*args)
next(gen) # first yield
teardowns.append(gen)
except StopIteration:
_raise_wrapfail(gen, "did not yield")
else:
res = hook_impl.function(*args)
if res is not None:
results.append(res)
if firstresult: # halt further impl calls
break
except BaseException:
excinfo = sys.exc_info()
finally:
if firstresult: # first result hooks return a single value
outcome = _Result(results[0] if results else None, excinfo)
else:
outcome = _Result(results, excinfo)
# run all wrapper post-yield blocks
for gen in reversed(teardowns):
try:
gen.send(outcome)
_raise_wrapfail(gen, "has second yield")
except StopIteration:
pass
> return outcome.get_result()
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/pluggy/_callers.py:60:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <pluggy._result._Result object at 0x10beea370>
def get_result(self):
"""Get the result(s) for this hook call.
If the hook was marked as a ``firstresult`` only a single value
will be returned otherwise a list of results.
"""
__tracebackhide__ = True
if self._excinfo is None:
return self._result
else:
ex = self._excinfo
> raise ex[1].with_traceback(ex[2])
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/pluggy/_result.py:60:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
hook_name = 'pytest_runtest_call'
hook_impls = [<HookImpl plugin_name='runner', plugin=<module '_pytest.runner' from '/Users/alexberger/.pyenv/versions/fp/lib/python....threadexception' from '/Users/alexberger/.pyenv/versions/fp/lib/python3.9/site-packages/_pytest/threadexception.py'>>]
caller_kwargs = {'item': <YamlItem Test accept valid fleet provision request>}, firstresult = False
def _multicall(hook_name, hook_impls, caller_kwargs, firstresult):
"""Execute a call into multiple python functions/methods and return the
result(s).
``caller_kwargs`` comes from _HookCaller.__call__().
"""
__tracebackhide__ = True
results = []
excinfo = None
try: # run impl and wrapper setup functions in a loop
teardowns = []
try:
for hook_impl in reversed(hook_impls):
try:
args = [caller_kwargs[argname] for argname in hook_impl.argnames]
except KeyError:
for argname in hook_impl.argnames:
if argname not in caller_kwargs:
raise HookCallError(
f"hook call must provide argument {argname!r}"
)
if hook_impl.hookwrapper:
try:
gen = hook_impl.function(*args)
next(gen) # first yield
teardowns.append(gen)
except StopIteration:
_raise_wrapfail(gen, "did not yield")
else:
> res = hook_impl.function(*args)
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/pluggy/_callers.py:39:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
item = <YamlItem Test accept valid fleet provision request>
def pytest_runtest_call(item: Item) -> None:
_update_current_test_var(item, "call")
try:
del sys.last_type
del sys.last_value
del sys.last_traceback
except AttributeError:
pass
try:
item.runtest()
except Exception as e:
# Store trace info to allow postmortem debugging
sys.last_type = type(e)
sys.last_value = e
assert e.__traceback__ is not None
# Skip *this* frame
sys.last_traceback = e.__traceback__.tb_next
> raise e
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/_pytest/runner.py:175:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
item = <YamlItem Test accept valid fleet provision request>
def pytest_runtest_call(item: Item) -> None:
_update_current_test_var(item, "call")
try:
del sys.last_type
del sys.last_value
del sys.last_traceback
except AttributeError:
pass
try:
> item.runtest()
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/_pytest/runner.py:167:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <YamlItem Test accept valid fleet provision request>
def runtest(self):
self.global_cfg = load_global_cfg(self.config)
load_plugins(self.global_cfg)
# INTERNAL
xfail = self.spec.get("_xfail", False)
try:
fixture_values = self._load_fixture_values()
self.global_cfg.variables.update(fixture_values)
call_hook(
self.global_cfg,
"pytest_tavern_beta_before_every_test_run",
test_dict=self.spec,
variables=self.global_cfg.variables,
)
> verify_tests(self.spec)
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/tavern/_core/pytest/item.py:194:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
test_spec = {'test_name': 'Test accept valid fleet provision request', 'includes': [{'name': 'device test data valid', 'descriptio...emplate/provision/json/accepted', 'json': {'deviceConfiguration': {}, 'thingName': '{deviceUuid:s}'}, 'timeout': 10}}]}
with_plugins = True
def verify_tests(test_spec, with_plugins=True):
"""Verify that a specific test block is correct
Todo:
Load schema file once. Requires some caching of the file
Args:
test_spec (dict): Test in dictionary form
Raises:
BadSchemaError: Schema did not match
"""
here = os.path.dirname(os.path.abspath(__file__))
schema_filename = os.path.join(here, "tests.jsonschema.yaml")
schema = load_schema_file(schema_filename, with_plugins)
> verify_jsonschema(test_spec, schema)
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/tavern/_core/schema/files.py:148:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
to_verify = {'test_name': 'Test accept valid fleet provision request', 'includes': [{'name': 'device test data valid', 'descriptio...emplate/provision/json/accepted', 'json': {'deviceConfiguration': {}, 'thingName': '{deviceUuid:s}'}, 'timeout': 10}}]}
schema = {'$schema': 'http://json-schema.org/draft-07/schema#', '$id': 'https://raw.githubusercontent.com/taverntesting/tavern/...': False, 'required': ['username'], 'properties': {'username': {'type': 'string'}, 'password': {'type': 'string'}}}}}}}
def verify_jsonschema(to_verify, schema):
"""Verify a generic file against a given jsonschema
Args:
to_verify (dict): Filename of source tests to check
schema (dict): Schema to verify against
Raises:
BadSchemaError: Schema did not match
"""
# pylint: disable=too-many-locals
validator = CustomValidator(schema)
try:
validator.validate(to_verify)
except jsonschema.ValidationError as e:
real_context = []
# ignore these strings because they're red herrings
for c in e.context:
description = c.schema.get("description", "<no description>")
if description == "Reference to another stage from an included config file":
continue
instance = c.instance
filename = get_stage_filename(instance)
if filename is None:
# Depending on what block raised the error, it mightbe difficult to tell what it was, so check the parent too
instance = e.instance
filename = get_stage_filename(instance)
if filename:
with open(filename, "r", encoding="utf-8") as infile:
n_lines = len(infile.readlines())
first_line, last_line, _ = get_stage_lines(instance)
first_line = max(first_line - 2, 0)
last_line = min(last_line + 2, n_lines)
reg = re.compile(r"^\s*$")
lines = read_relevant_lines(instance, first_line, last_line)
lines = [line for line in lines if not reg.match(line.strip())]
content = "\n".join(list(lines))
real_context.append(
f"""
{c.message}
{filename}: line {first_line}-{last_line}:
{content}
"""
)
else:
real_context.append(
f"""
{c.message}
<error: unable to find input file for context>
"""
)
msg = "\n---\n" + "\n---\n".join([str(i) for i in real_context])
> raise BadSchemaError(msg) from None
E tavern._core.exceptions.BadSchemaError:
E ---
E
E {'topic': '$aws/certificates/create/json/accepted', 'json': {'certificateId': <tavern._core.loader.AnythingSentinel object at 0x10646d8b0>, 'certificateOwnershipToken': <tavern._core.loader.AnythingSentinel object at 0x10646d8b0>, 'certificatePem': <tavern._core.loader.AnythingSentinel object at 0x10646d8b0>, 'privateKey': <tavern._core.loader.AnythingSentinel object at 0x10646d8b0>}, 'verify_response_with': {'function': 'save_certificate:save_certificate'}, 'save': {'json': {'createCertResp_certificateId': 'certificateId', 'createCertResp_certificatePem': 'certificatePem', 'createCertResp_privateKey': 'privateKey', 'createCertResp_certificateOwnershipToken': 'certificateOwnershipToken'}}, 'timeout': 10} is not valid under any of the given schemas
E /Users/alexberger/dev/aai/cloud-infrastructure/integration-tests/fleet-provisioning/test_accept_valid_fleet_provision.tavern.yml: line 23-44:
E
E json: {}
E mqtt_response:
E topic: $aws/certificates/create/json/accepted
E json:
E certificateId: !anything
E certificateOwnershipToken: !anything
E certificatePem: !anything
E privateKey: !anything
E verify_response_with:
E function: save_certificate:save_certificate
E save:
E json:
E createCertResp_certificateId: certificateId
E createCertResp_certificatePem: certificatePem
E createCertResp_privateKey: privateKey
E createCertResp_certificateOwnershipToken: certificateOwnershipToken
E timeout: 10
E - name: Provision Certificate
E mqtt_publish:
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/tavern/_core/schema/jsonschema.py:171: BadSchemaError
Looks like there was a breaking change with the 2.0 release. Could you please take a look what is breaking in the new release? I am getting following error when running tests based on tavern test framework.