system_test/test_nsfs_system.py fails with permission error cannot read directory '/nsfs': Permission denied

[vkathole]

Jenkins Link: https://ocs4-jenkins-csb-odf-qe.apps.ocp-c1.prod.psi.redhat.com/job/qe-deploy-ocs-cluster/34220/testReport/junit/tests.cross_functional.system_test.test_nsfs_system/TestNSFSSystem/test_nsfs/

Logs: http://magna002.ceph.redhat.com/ocsci-jenkins/openshift-clusters/vkathole-sys22/vkathole-sys22_20240222T185001/logs/ocs-ci-logs-1708633106/

`self = <system_test.test_nsfs_system.TestNSFSSystem object at 0x7f6d78288b20> mcg_obj = <ocs_ci.ocs.resources.mcg.MCG object at 0x7f6d799e03d0> nsfs_bucket_factory = <function nsfs_bucket_factory_fixture..nsfs_bucket_factory_implementation at 0x7f6d789d7e50> awscli_pod_session = <ocs_ci.ocs.resources.pod.Pod object at 0x7f6d77ffc100> test_directory_setup = SetupDirs(origin_dir='test_nsfs/origin', result_dir='test_nsfs/result') snapshot_factory = <function snapshot_factory_fixture..factory at 0x7f6d789fe1f0> noobaa_db_backup_and_recovery = <function noobaa_db_backup_and_recovery..factory at 0x7f6d789fe310>

@pytest.mark.polarion_id("OCS-3952")
def test_nsfs(
    self,
    mcg_obj,
    nsfs_bucket_factory,
    awscli_pod_session,
    test_directory_setup,
    snapshot_factory,
    noobaa_db_backup_and_recovery,
):
    """
    The objectives of this test case are:
    1) To verify S3 operations on NSFS buckets
    2) NSFS based data is accessible/intact when cluster related operations pod failures are performed
    3) Noobaa DB backup and recovery does not impact data on NSFS

    """
    s3_ops_obj = "obj-key"
    s3_ops_copy_obj = "copy-obj-key"
    s3_ops_obj_data = "object data-" + str(uuid.uuid4().hex)
    nsfs_obj_pattern = "nsfs-obj"

    nsfs_objs = [
        NSFS(
            method="OC",
            pvc_size=10,
        ),
        NSFS(
            method="OC",
            pvc_size=10,
            mount_existing_dir=True,
        ),
    ]
    for nsfs_obj in nsfs_objs:

      nsfs_bucket_factory(nsfs_obj)

tests/cross_functional/system_test/test_nsfs_system.py:81:

---

tests/conftest.py:5717: in nsfs_bucket_factory_implementation endpoint_pod.exec_cmd_on_pod("chmod -R 777 /nsfs") ocs_ci/ocs/resources/pod.py:192: in exec_cmd_on_pod return self.ocp.exec_oc_cmd( ocs_ci/ocs/ocp.py:188: in exec_oc_cmd out = run_cmd( ocs_ci/utility/utils.py:484: in run_cmd completed_process = exec_cmd(

---

cmd = ['oc', '--kubeconfig', '/home/jenkins/current-cluster-dir/openshift-cluster-dir/auth/kubeconfig', '-n', 'openshift-storage', 'rsh', ...] secrets = None, timeout = 600, ignore_error = False, threading_lock = None silent = False, use_shell = False cluster_config = <ocs_ci.framework.MultiClusterConfig object at 0x7f6da8912550> kwargs = {} masked_cmd = 'oc -n openshift-storage rsh noobaa-endpoint-58b4787bd8-42dqj chmod -R 777 /nsfs' kubepath = '/home/jenkins/current-cluster-dir/openshift-cluster-dir/auth/kubeconfig' kube_index = 1, plugin_list = 'oc plugin list' cp = CompletedProcess(args=['oc', 'plugin', 'list'], returncode=1, stdout=b'', stderr=b'Unable to read directory "/home/jen.../jenkins/.local/bin: no such file or directory. Skipping...\nerror: unable to find any kubectl plugins in your PATH\n') subcmd = '_n'

def exec_cmd(
    cmd,
    secrets=None,
    timeout=600,
    ignore_error=False,
    threading_lock=None,
    silent=False,
    use_shell=False,
    cluster_config=None,
    **kwargs,
):
    """
    Run an arbitrary command locally

    If the command is grep and matching pattern is not found, then this function
    returns "command terminated with exit code 1" in stderr.

    Args:
        cmd (str): command to run
        secrets (list): A list of secrets to be masked with asterisks
            This kwarg is popped in order to not interfere with
            subprocess.run(``**kwargs``)
        timeout (int): Timeout for the command, defaults to 600 seconds.
        ignore_error (bool): True if ignore non zero return code and do not
            raise the exception.
        threading_lock (threading.RLock): threading.RLock object that is used
            for handling concurrent oc commands
        silent (bool): If True will silent errors from the server, default false
        use_shell (bool): If True will pass the cmd without splitting
        cluster_config (MultiClusterConfig): In case of multicluster environment this object
                will be non-null

    Raises:
        CommandFailed: In case the command execution fails

    Returns:
        (CompletedProcess) A CompletedProcess object of the command that was executed
        CompletedProcess attributes:
        args: The list or str args passed to run().
        returncode (str): The exit code of the process, negative for signals.
        stdout     (str): The standard output (None if not captured).
        stderr     (str): The standard error (None if not captured).

    """
    masked_cmd = mask_secrets(cmd, secrets)
    log.info(f"Executing command: {masked_cmd}")
    if isinstance(cmd, str) and not kwargs.get("shell"):
        cmd = shlex.split(cmd)
    if cluster_config and cmd[0] == "oc" and "--kubeconfig" not in cmd:
        kubepath = cluster_config.RUN["kubeconfig"]
        kube_index = 1
        # check if we have an oc plugin in the command
        plugin_list = "oc plugin list"
        cp = subprocess.run(
            shlex.split(plugin_list),
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
        )
        subcmd = cmd[1].split("-")
        if len(subcmd) > 1:
            subcmd = "_".join(subcmd)
            log.info(f"searching for plugin: {subcmd}")
        if not isinstance(subcmd, str) and isinstance(subcmd, list):
            subcmd = str(subcmd[0])

        for l in cp.stdout.decode().splitlines():
            if subcmd in l:
                # If oc cmdline has plugin name then we need to push the
                # --kubeconfig to next index
                kube_index = 2
                log.info(f"Found oc plugin {subcmd}")
        cmd = list_insert_at_position(cmd, kube_index, ["--kubeconfig"])
        cmd = list_insert_at_position(cmd, kube_index + 1, [kubepath])
    if threading_lock and cmd[0] == "oc":
        threading_lock.acquire()
    completed_process = subprocess.run(
        cmd,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        stdin=subprocess.PIPE,
        timeout=timeout,
        **kwargs,
    )
    if threading_lock and cmd[0] == "oc":
        threading_lock.release()
    masked_stdout = mask_secrets(completed_process.stdout.decode(), secrets)
    if len(completed_process.stdout) > 0:
        log.debug(f"Command stdout: {masked_stdout}")
    else:
        log.debug("Command stdout is empty")

    masked_stderr = mask_secrets(completed_process.stderr.decode(), secrets)
    if len(completed_process.stderr) > 0:
        if not silent:
            log.warning(f"Command stderr: {masked_stderr}")
    else:
        log.debug("Command stderr is empty")
    log.debug(f"Command return code: {completed_process.returncode}")
    if completed_process.returncode and not ignore_error:
        masked_stderr = bin_xml_escape(filter_out_emojis(masked_stderr))
        if (
            "grep" in masked_cmd
            and b"command terminated with exit code 1" in completed_process.stderr
        ):
            log.info(f"No results found for grep command: {masked_cmd}")
        else:

          raise CommandFailed(

f"Error during execution of command: {masked_cmd}." f"\nError is {masked_stderr}" ) E ocs_ci.ocs.exceptions.CommandFailed: Error during execution of command: oc -n openshift-storage rsh noobaa-endpoint-58b4787bd8-42dqj chmod -R 777 /nsfs. E Error is chmod: changing permissions of '/nsfs/nsfs-ns-store-c2f57596fa974481a1fa38eec5': Permission denied E chmod: cannot read directory '/nsfs/nsfs-ns-store-c2f57596fa974481a1fa38eec5': Permission denied E command terminated with exit code 1

ocs_ci/utility/utils.py:685: CommandFailed`

[github-actions[bot]]

This issue has been automatically marked as stale because it has not had recent activity. It will be closed in 30 days if no further activity occurs.

[github-actions[bot]]

This issue has been automatically marked as stale because it has not had recent activity. It will be closed in 30 days if no further activity occurs.

[github-actions[bot]]

This issue has been automatically closed due to inactivity. Please re-open if this still requires investigation.