search

red-hat-storage / ocs-ci

https://ocs-ci.readthedocs.io/en/latest/
MIT License
109 stars 166 forks source link

test_noobaa_db_backup_recovery_locally failing on warps3.run_benchmark #10046

Open vkathole opened 3 months ago

vkathole commented 3 months ago

RP : https://reportportal-ocs4.apps.ocp-c1.prod.psi.redhat.com/ui/#ocs/launches/678/21659/1030022/1030025/log https://reportportal-ocs4.apps.ocp-c1.prod.psi.redhat.com/ui/#ocs/launches/678/21271/1016601/1016604/log

vkathole commented 3 months ago

`self = <test_noobaa_db_backup_and_recovery.TestNoobaaBackupAndRecovery object at 0x7f8862a222b0> bucket_factory = <function bucket_factory_fixture.._create_buckets at 0x7f88624dbee0> noobaa_db_backup_and_recovery_locally = <function noobaa_db_backup_and_recovery_locally..factory at 0x7f88624f0550> warps3 = <ocs_ci.ocs.warp.Warp object at 0x7f886872a190> mcg_obj_session = <ocs_ci.ocs.resources.mcg.MCG object at 0x7f8862a22c10>

@pytest.mark.polarion_id("OCS-4842") @pytest.mark.bugzilla("214035") @skipif_ocs_version("<4.8") def test_noobaa_db_backup_recovery_locally( self, bucket_factory, noobaa_db_backup_and_recovery_locally, warps3, mcg_obj_session, ): """ Test to verify Backup and Restore for Multicloud Object Gateway database locally Backup procedure:

tests/cross_functional/kcs/test_noobaa_db_backup_and_recovery.py:139:

ocs_ci/ocs/warp.py:196: in run_benchmark self.pod_obj.exec_cmd_on_pod(cmd, out_yaml_format=False, timeout=timeout) ocs_ci/ocs/resources/pod.py:193: in exec_cmd_on_pod return self.ocp.exec_oc_cmd( ocs_ci/ocs/ocp.py:189: in exec_oc_cmd out = run_cmd( ocs_ci/utility/utils.py:487: in run_cmd completed_process = exec_cmd(

cmd = ['oc', '--kubeconfig', '/home/jenkins/current-cluster-dir/openshift-cluster-dir/auth/kubeconfig', '-n', 'openshift-storage', 'rsh', ...] secrets = None, timeout = 4000, ignore_error = False, threading_lock = None silent = False, use_shell = False cluster_config = <ocs_ci.framework.MultiClusterConfig object at 0x7f8896b9cfd0> kwargs = {} masked_cmd = 'oc -n openshift-storage rsh warppod-1-fr2q8 /home/warp/warp mixed --duration=10m --host=s3.openshift-storage.svc --in...6730e96184c5cb331355b2a9e50 --analyze.out=output.csv --warp-client=10.128.2.55:7761,10.131.0.34:7761,10.128.2.54:7761 ' kubepath = '/home/jenkins/current-cluster-dir/openshift-cluster-dir/auth/kubeconfig' kube_index = 1, plugin_list = 'oc plugin list' cp = CompletedProcess(args=['oc', 'plugin', 'list'], returncode=1, stdout=b'', stderr=b'Unable to read directory "/home/jen.../jenkins/.local/bin: no such file or directory. Skipping...\nerror: unable to find any kubectl plugins in your PATH\n') subcmd = '_n'

def exec_cmd( cmd, secrets=None, timeout=600, ignore_error=False, threading_lock=None, silent=False, use_shell=False, cluster_config=None, **kwargs, ): """ Run an arbitrary command locally

If the command is grep and matching pattern is not found, then this function
returns "command terminated with exit code 1" in stderr.

Args:
    cmd (str): command to run
    secrets (list): A list of secrets to be masked with asterisks
        This kwarg is popped in order to not interfere with
        subprocess.run(``**kwargs``)
    timeout (int): Timeout for the command, defaults to 600 seconds.
    ignore_error (bool): True if ignore non zero return code and do not
        raise the exception.
    threading_lock (threading.RLock): threading.RLock object that is used
        for handling concurrent oc commands
    silent (bool): If True will silent errors from the server, default false
    use_shell (bool): If True will pass the cmd without splitting
    cluster_config (MultiClusterConfig): In case of multicluster environment this object
            will be non-null

Raises:
    CommandFailed: In case the command execution fails

Returns:
    (CompletedProcess) A CompletedProcess object of the command that was executed
    CompletedProcess attributes:
    args: The list or str args passed to run().
    returncode (str): The exit code of the process, negative for signals.
    stdout     (str): The standard output (None if not captured).
    stderr     (str): The standard error (None if not captured).

"""
masked_cmd = mask_secrets(cmd, secrets)
log.info(f"Executing command: {masked_cmd}")
if isinstance(cmd, str) and not kwargs.get("shell"):
    cmd = shlex.split(cmd)
if cluster_config and cmd[0] == "oc" and "--kubeconfig" not in cmd:
    kubepath = cluster_config.RUN["kubeconfig"]
    kube_index = 1
    # check if we have an oc plugin in the command
    plugin_list = "oc plugin list"
    cp = subprocess.run(
        shlex.split(plugin_list),
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )
    subcmd = cmd[1].split("-")
    if len(subcmd) > 1:
        subcmd = "_".join(subcmd)
    if not isinstance(subcmd, str) and isinstance(subcmd, list):
        subcmd = str(subcmd[0])

    for l in cp.stdout.decode().splitlines():
        if subcmd in l:
            # If oc cmdline has plugin name then we need to push the
            # --kubeconfig to next index
            kube_index = 2
            log.info(f"Found oc plugin {subcmd}")
    cmd = list_insert_at_position(cmd, kube_index, ["--kubeconfig"])
    cmd = list_insert_at_position(cmd, kube_index + 1, [kubepath])
if threading_lock and cmd[0] == "oc":
    threading_lock.acquire()
completed_process = subprocess.run(
    cmd,
    stdout=subprocess.PIPE,
    stderr=subprocess.PIPE,
    stdin=subprocess.PIPE,
    timeout=timeout,
    **kwargs,
)
if threading_lock and cmd[0] == "oc":
    threading_lock.release()
masked_stdout = mask_secrets(completed_process.stdout.decode(), secrets)
if len(completed_process.stdout) > 0:
    log.debug(f"Command stdout: {masked_stdout}")
else:
    log.debug("Command stdout is empty")

masked_stderr = mask_secrets(completed_process.stderr.decode(), secrets)
if len(completed_process.stderr) > 0:
    if not silent:
        log.warning(f"Command stderr: {masked_stderr}")
else:
    log.debug("Command stderr is empty")
log.debug(f"Command return code: {completed_process.returncode}")
if completed_process.returncode and not ignore_error:
    masked_stderr = bin_xml_escape(filter_out_emojis(masked_stderr))
    if (
        "grep" in masked_cmd
        and b"command terminated with exit code 1" in completed_process.stderr
    ):
        log.info(f"No results found for grep command: {masked_cmd}")
    else:

      raise CommandFailed(
            f"Error during execution of command: {masked_cmd}."
            f"\nError is {masked_stderr}"
        )

E ocs_ci.ocs.exceptions.CommandFailed: Error during execution of command: oc -n openshift-storage rsh warppod-1-fr2q8 /home/warp/warp mixed --duration=10m --host=s3.openshift-storage.svc --insecure=True --tls=True --debug=True --access-key=zAcE2nBTWFr3YYl1tbXS --secret-key=cAZpIdMSHfJa8yvgCyLUoIgUXaXHWLkUj17ObrV7 --noclear --noprefix --concurrent=10 --objects=100 --obj.size=1MiB --get-distrib=2 --put-distrib=2 --delete-distrib=1 --stat-distrib=0 --bucket=s3-bucket-5f76730e96184c5cb331355b2a9e50 --analyze.out=output.csv --warp-client=10.128.2.55:7761,10.131.0.34:7761,10.128.2.54:7761 . E Error is warp: Stage failed. Client 10.128.2.55:7761 returned error. We encountered an internal error. Please try again. E (0) benchserver.go:498 cli.(*connections).waitForStage.func1(..) E Release-Tag:(no tag) | Commit:(dev) | Host:warppod-1-fr2q8 | OS:linux | Arch:amd64 | Lang:go1.19.4 | Mem:3.8 MB/24 MB | Heap:3.8 MB/12 MB. E command terminated with exit code 1

ocs_ci/utility/utils.py:697: CommandFailed

`

vkathole commented 3 months ago

Looks like a timeout issue as test case has passed in same platform in recent run RP: https://reportportal-ocs4.apps.ocp-c1.prod.psi.redhat.com/ui/#ocs/launches/678/22636/1084612/1084615/log

github-actions[bot] commented 3 days ago

This issue has been automatically marked as stale because it has not had recent activity. It will be closed in 30 days if no further activity occurs.