tarantool / tt

Command-line utility to manage Tarantool applications
Other
101 stars 12 forks source link

tests: flaky failures in running/test_running_cluster.py tests #1000

Closed elhimov closed 2 weeks ago

elhimov commented 2 weeks ago

There is a flaky test in test/integration/running/test_running_cluster.py teststest_log.py (see here)

# Check status.
            for inst in instances:
                file = wait_file(os.path.join(run_dir, inst), 'tarantool.pid', [])
                assert file != ""
                file = wait_file(os.path.join(run_dir, inst), control_socket, [])
                assert file != ""

                with open(os.path.join(run_dir, inst, 'tarantool.pid')) as f:
                    assert pidByInstanceName[inst] != f.readline()

           status_cmd = [tt_cmd, "status", app_name]
           status_rc, status_out = run_command_and_get_output(status_cmd, cwd=tmpdir)
           assert status_rc == 0
           status_info = extract_status(status_out)
           for inst in instances:
                assert status_info[app_name+":"+inst]["STATUS"] == "RUNNING"

        finally:
>           stop_application(tt_cmd, app_name, tmpdir, instances)

test/integration/running/test_running_cluster.py:130: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

tt_cmd = PosixPath('/tmp/pytest-of-runner/pytest-0/tt_build0/tt')
app_name = 'small_cluster_app'
workdir = '/tmp/pytest-of-runner/pytest-0/test_running_base_functionalit1'
instances = ['storage-master', 'storage-replica']

    def stop_application(tt_cmd, app_name, workdir, instances):
        stop_cmd = [tt_cmd, "stop", "-y", app_name]
        stop_rc, stop_out = run_command_and_get_output(stop_cmd, cwd=workdir)
        assert stop_rc == 0

        for inst in instances:
>           assert re.search(rf"The Instance {app_name}:{inst} \(PID = \d+\) has been terminated.",
                             stop_out)
E           AssertionError: assert None
E            +  where None = <function search at 0x7f41507c17e0>('The Instance small_cluster_app:storage-master \\(PID = \\d+\\) has been terminated.', '   • The Instance small_cluster_app:storage-replica (PID = 50076) has been terminated.\n')
E            +    where <function search at 0x7f41507c17e0> = re.search

test/integration/running/test_running_cluster.py:35: AssertionError
----------------------------- Captured stdout call -----------------------------
 INSTANCE                           STATUS   PID    MODE  CONFIG               BOX      UPSTREAM 
 small_cluster_app:storage-master   RUNNING  50018  RO    startup_in_progress  loading  --       
 small_cluster_app:storage-replica  RUNNING  50017  RO    startup_in_progress  loading  --       

 INSTANCE                           STATUS   PID    MODE  CONFIG               BOX      UPSTREAM 
 small_cluster_app:storage-master   RUNNING  50075  RO    startup_in_progress  loading  --       
 small_cluster_app:storage-replica  RUNNING  50076  RO    startup_in_progress  loading  --       

   • The Instance small_cluster_app:storage-replica (PID = 50076) has been terminated.

=========================== short test summary info ============================
FAILED test/integration/running/test_running_cluster.py::test_running_base_functionality - AssertionError: assert None
 +  where None = <function search at 0x7f41507c17e0>('The Instance small_cluster_app:storage-master \\(PID = \\d+\\) has been terminated.', '   • The Instance small_cluster_app:storage-replica (PID = 50076) has been terminated.\n')
 +    where <function search at 0x7f41507c17e0> = re.search
==== 1 failed, 940 passed, 81 skipped, 45 deselected in 1007.06s (0:16:47) =====
Error: running "python3 -m pytest -m not slow and not slow_ee and not notarantool test/integration" failed with exit code 1