fruch
commented
1 month ago is that consistent ? i.e. if you trigger again that CI run ? it happens every single time ?
Open cezarmoise opened 1 month ago
fruch
commented
1 month ago is that consistent ? i.e. if you trigger again that CI run ? it happens every single time ?
cezarmoise
commented
1 month ago The first commit where this happens is 27475153553311c9c9dab1aaf9bd6b5f4f0e4925, the commit that merges the yaml changes. However I see that this run has errors while this one you rebuit does not, even though they are on the same ccm commit (9370f298cec8f6ce06468aa12a5aa33d38b0a7bf). I also did a rebuild and it failed again
Does not happen locally, tested with up to date branch
Error 1
Stacktrace
self =
@pytest.mark.scylla_mode("!debug")
def test_stop_node_during_index_build(self):
"""
Stop one node during index building and read data by index
"""
> self._node_action_during_index_build(node_action="stop", nodes=4, rf=3, num_rows=500_000)
secondary_indexes_test.py:1362:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
secondary_indexes_test.py:1391: in _node_action_during_index_build
session = self.prepare(nodes=nodes, rf=rf, keyspace_name=keyspace_name, session_node=3)
secondary_indexes_test.py:138: in prepare
cluster.populate(populate, use_vnodes=True)
../scylla/.local/lib/python3.12/site-packages/ccmlib/cluster.py:329: in populate
self.new_node(i, debug=debug, initial_token=tk, data_center=dc, rack=rack)
../scylla/.local/lib/python3.12/site-packages/ccmlib/cluster.py:336: in new_node
node = self.create_node(name=f'node{i}',
../scylla/.local/lib/python3.12/site-packages/ccmlib/scylla_cluster.py:95: in create_node
return ScyllaNode(name, self, auto_bootstrap, None,
../scylla/.local/lib/python3.12/site-packages/ccmlib/scylla_node.py:54: in __init__
super().__init__(name, cluster, auto_bootstrap,
../scylla/.local/lib/python3.12/site-packages/ccmlib/node.py:125: in __init__
self.import_config_files()
../scylla/.local/lib/python3.12/site-packages/ccmlib/scylla_node.py:980: in import_config_files
self._update_config()
../scylla/.local/lib/python3.12/site-packages/ccmlib/node.py:1589: in _update_config
yaml.dump(values, f)
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/main.py:590: in dump
self._context_manager.dump(data)
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/main.py:953: in dump
self._yaml.representer.represent(data)
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/representer.py:84: in represent
self.serializer.serialize(node)
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/serializer.py:105: in serialize
self.emitter.emit(
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/emitter.py:267: in emit
self.state()
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/emitter.py:370: in expect_document_start
self.write_indicator('---', True)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
indicator = '---', need_whitespace = True, whitespace = False, indention = False
def write_indicator(
self,
indicator: Any,
need_whitespace: Any,
whitespace: bool = False,
indention: bool = False,
) -> None:
if self.whitespace or not need_whitespace:
data = indicator
else:
data = ' ' + indicator
self.whitespace = whitespace
self.indention = self.indention and indention
self.column += len(data)
self.open_ended = False
if bool(self.encoding):
data = data.encode(self.encoding)
> self.stream.write(data)
E ValueError: I/O operation on closed file.
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/emitter.py:1249: ValueError
Error 2
Stacktrace
self =
@pytest.mark.scylla_mode("!debug")
@pytest.mark.skip_if(with_feature("tablets") & issue_open("#16739"))
def test_interrupt_build_process_and_resharding_max_to_half_test(self):
"""Test that an interrupted MV build process is resumed after interrupted resharding,
with resharding cpu_count() -> cpu_count() // 2"""
# For some reason, Scylla's hwloc only sees cpu_count() - 1 cpus
> self._do_resharding_test(self._max_shards(), self._half_shards(), interrupt_resharding=True)
materialized_views_test.py:4311:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
materialized_views_test.py:4333: in _do_resharding_test
self.__do_resharding_test(smp_before, smp_after, compression, interrupt_resharding)
materialized_views_test.py:4405: in __do_resharding_test
[logger.debug(future.result()) for future in futures]
/usr/local/lib/python3.12/concurrent/futures/_base.py:456: in result
return self.__get_result()
/usr/local/lib/python3.12/concurrent/futures/_base.py:401: in __get_result
raise self._exception
/usr/local/lib/python3.12/concurrent/futures/thread.py:58: in run
result = self.fn(*self.args, **self.kwargs)
materialized_views_test.py:4340: in __do_start_node_with_interrupt_resharding
node.start(jvm_args=jvm_args, no_wait=True)
../scylla/.local/lib/python3.12/site-packages/ccmlib/scylla_node.py:711: in start
scylla_process = self._start_scylla(args=args, marks=marks, update_pid=update_pid,
../scylla/.local/lib/python3.12/site-packages/ccmlib/scylla_node.py:340: in _start_scylla
self._update_pid(self._process_scylla)
../scylla/.local/lib/python3.12/site-packages/ccmlib/node.py:1900: in _update_pid
self.__update_status()
../scylla/.local/lib/python3.12/site-packages/ccmlib/node.py:1777: in __update_status
self._update_config()
../scylla/.local/lib/python3.12/site-packages/ccmlib/node.py:1589: in _update_config
yaml.dump(values, f)
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/main.py:594: in dump
return self.dump_all([data], stream, transform=transform)
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/main.py:605: in dump_all
self._context_manager.teardown_output()
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/main.py:914: in teardown_output
self._yaml.serializer.close()
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/serializer.py:91: in close
self.emitter.emit(StreamEndEvent())
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/emitter.py:267: in emit
self.state()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
first = False
def expect_document_start(self, first: bool = False) -> None:
if isinstance(self.event, DocumentStartEvent):
if (self.event.version or self.event.tags) and self.open_ended:
self.write_indicator('...', True)
self.write_indent()
if self.event.version:
version_text = self.prepare_version(self.event.version)
self.write_version_directive(version_text)
self.tag_prefixes = self.DEFAULT_TAG_PREFIXES.copy()
if self.event.tags:
handles = sorted(self.event.tags.keys())
for handle in handles:
prefix = self.event.tags[handle]
self.tag_prefixes[prefix] = handle
handle_text = self.prepare_tag_handle(handle)
prefix_text = self.prepare_tag_prefix(prefix)
self.write_tag_directive(handle_text, prefix_text)
implicit = (
first
and not self.event.explicit
and not self.canonical
and not self.event.version
and not self.event.tags
and not self.check_empty_document()
)
if not implicit:
self.write_indent()
self.write_indicator('---', True)
if self.canonical:
self.write_indent()
self.state = self.expect_document_root
elif isinstance(self.event, StreamEndEvent):
if self.open_ended:
self.write_indicator('...', True)
self.write_indent()
self.write_stream_end()
self.state = self.expect_nothing
else:
> raise EmitterError(f'expected DocumentStartEvent, but got {self.event!s}')
E ruamel.yaml.emitter.EmitterError: expected DocumentStartEvent, but got MappingStartEvent(tag='tag:yaml.org,2002:map', implicit=True, flow_style=False)
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/emitter.py:381: EmitterError