test_bucket_creation failed with ocs_ci.ocs.exceptions.CommandFailed: Error during execution of command: /home/jenkins/workspace/qe-deploy-ocs-cluster-prod/ocs-ci/data/mcg-cli backingstore create pv-pool pv-backingstore-216e7721521d4379baea7305 --num-volumes 1 --pv-size-gb 50 --storage-class ocs-storagecluster-ceph-rbd -n openshift-storage.

Run details:
URL: https://reportportal-ocs4.apps.ocp-c1.prod.psi.redhat.com/ui/#OCS/launches/362/6633/272793/272796/272802/log Run ID: 1669598191 Test Case: test_bucket_creation ODF Build: 4.12.0-120 OCP Version: 4.12 Job name: VSPHERE6 UPI 1AZ RHCOS MCG Only Deployment 3M 3W tier1 Jenkins job: https://ocs4-jenkins-csb-odf-qe.apps.ocp-c1.prod.psi.redhat.com/job/qe-deploy-ocs-cluster-prod/6146/ Logs URL: http://magna002.ceph.redhat.com/ocsci-jenkins/openshift-clusters/j-005vu1rmcg33-t1/j-005vu1rmcg33-t1_20221128T003211/logs/
Failure Details:
Message: ocs_ci.ocs.exceptions.CommandFailed: Error during execution of command: /home/jenkins/workspace/qe-deploy-ocs-cluster-prod/ocs-ci/data/mcg-cli backingstore create pv-pool pv-backingstore-216e7721521d4379baea7305 --num-volumes 1 --pv-size-gb 50 --storage-class ocs-storagecluster-ceph-rbd -n openshift-storage.
Error is time="2022-11-28T01:32:25Z" level=info msg="✅ Exists: NooBaa \"noobaa\"\n"
time="2022-11-28T01:32:25Z" level=info msg="❌ Not Found: StorageClass \"ocs-storagecluster-ceph-rbd\"\n"
time="2022-11-28T01:32:25Z" level=fatal msg="❌ Could not get StorageClass \"ocs-storagecluster-ceph-rbd\" for system in namespace \"openshift-storage\""
Type: None

Text:
self = <tests.manage.mcg.test_bucket_creation.TestBucketCreation object at 0x7f4d6a8398e0>
bucket_class_factory = <function bucket_class_factory.<locals>._create_bucket_class at 0x7f4d6baee8b0>
bucket_factory = <function bucket_factory_fixture.<locals>._create_buckets at 0x7f4d6baee9d0>
amount = 1, interface = 'CLI'
bucketclass_dict = {'backingstore_dict': {'pv': [(1, 50, 'ocs-storagecluster-ceph-rbd')]}, 'interface': 'CLI'}

    @pytest.mark.parametrize(
        argnames="amount,interface,bucketclass_dict",
        argvalues=[
            pytest.param(
                *[3, "S3", None],
                marks=[pytest.mark.polarion_id("OCS-1298"), tier1, acceptance],
            ),
            pytest.param(
                *[100, "S3", None],
                marks=[
                    pytest.mark.skip(ERRATIC_TIMEOUTS_SKIP_REASON),
                    performance,
                    pytest.mark.polarion_id("OCS-1823"),
                ],
            ),
            pytest.param(
                *[1000, "S3", None],
                marks=[
                    pytest.mark.skip(ERRATIC_TIMEOUTS_SKIP_REASON),
                    performance,
                    pytest.mark.polarion_id("OCS-1824"),
                ],
            ),
            pytest.param(
                *[3, "OC", None],
                marks=[tier1, acceptance, pytest.mark.polarion_id("OCS-1298")],
            ),
            pytest.param(
                *[100, "OC", None],
                marks=[
                    pytest.mark.skip(ERRATIC_TIMEOUTS_SKIP_REASON),
                    performance,
                    pytest.mark.polarion_id("OCS-1826"),
                ],
            ),
            pytest.param(
                *[1000, "OC", None],
                marks=[
                    pytest.mark.skip(ERRATIC_TIMEOUTS_SKIP_REASON),
                    performance,
                    pytest.mark.polarion_id("OCS-1827"),
                ],
            ),
            pytest.param(
                *[3, "CLI", None],
                marks=[tier1, acceptance, pytest.mark.polarion_id("OCS-1298")],
            ),
            pytest.param(
                *[100, "CLI", None],
                marks=[
                    pytest.mark.skip(ERRATIC_TIMEOUTS_SKIP_REASON),
                    performance,
                    pytest.mark.polarion_id("OCS-1825"),
                ],
            ),
            pytest.param(
                *[1000, "CLI", None],
                marks=[
                    pytest.mark.skip(ERRATIC_TIMEOUTS_SKIP_REASON),
                    performance,
                    pytest.mark.polarion_id("OCS-1828"),
                ],
            ),
            pytest.param(
                *[
                    1,
                    "OC",
                    {
                        "interface": "OC",
                        "backingstore_dict": {
                            "pv": [(1, 50, DEFAULT_STORAGECLASS_RBD)]
                        },
                    },
                ],
                marks=[tier1, pytest.mark.polarion_id("OCS-2331")],
            ),
            pytest.param(
                *[
                    1,
                    "CLI",
                    {
                        "interface": "CLI",
                        "backingstore_dict": {
                            "pv": [(1, 50, DEFAULT_STORAGECLASS_RBD)]
                        },
                    },
                ],
                marks=[tier1, pytest.mark.polarion_id("OCS-2331")],
            ),
        ],
        ids=[
            "3-S3-DEFAULT-BACKINGSTORE",
            "100-S3-DEFAULT-BACKINGSTORE",
            "1000-S3-DEFAULT-BACKINGSTORE",
            "3-OC-DEFAULT-BACKINGSTORE",
            "100-OC-DEFAULT-BACKINGSTORE",
            "1000-OC-DEFAULT-BACKINGSTORE",
            "3-CLI-DEFAULT-BACKINGSTORE",
            "100-CLI-DEFAULT-BACKINGSTORE",
            "1000-CLI-DEFAULT-BACKINGSTORE",
            "1-OC-PVPOOL",
            "1-CLI-PVPOOL",
        ],
    )
    def test_bucket_creation(
        self, bucket_class_factory, bucket_factory, amount, interface, bucketclass_dict
    ):
        """
        Test bucket creation using the S3 SDK, OC command or MCG CLI.
        The factory checks the bucket's health by default.
        """
>       bucket_factory(amount, interface, bucketclass=bucketclass_dict)

tests/manage/mcg/test_bucket_creation.py:141: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/conftest.py:2557: in _create_buckets
    bucketclass if bucketclass is None else bucket_class_factory(bucketclass)
ocs_ci/ocs/resources/bucketclass.py:169: in _create_bucket_class
    for backingstore in backingstore_factory(
ocs_ci/ocs/resources/backingstore.py:316: in _create_backingstore
    cmdMap[method.lower()][cloud.lower()](
ocs_ci/ocs/bucket_utils.py:738: in cli_create_pv_backingstore
    mcg_obj.exec_mcg_cmd(cmd)
ocs_ci/ocs/resources/mcg.py:875: in exec_mcg_cmd
    result = exec_cmd(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

cmd = ['/home/jenkins/workspace/qe-deploy-ocs-cluster-prod/ocs-ci/data/mcg-cli', 'backingstore', 'create', 'pv-pool', 'pv-backingstore-216e7721521d4379baea7305', '--num-volumes', ...]
secrets = None, timeout = 600, ignore_error = False, threading_lock = None
silent = False, kwargs = {}
masked_cmd = '/home/jenkins/workspace/qe-deploy-ocs-cluster-prod/ocs-ci/data/mcg-cli backingstore create pv-pool pv-backingstore-216e7721521d4379baea7305 --num-volumes 1 --pv-size-gb 50 --storage-class ocs-storagecluster-ceph-rbd -n openshift-storage'
completed_process = CompletedProcess(args=['/home/jenkins/workspace/qe-deploy-ocs-cluster-prod/ocs-ci/data/mcg-cli', 'backingstore', 'crea...d\x8c Could not get StorageClass \\"ocs-storagecluster-ceph-rbd\\" for system in namespace \\"openshift-storage\\""\n')
masked_stdout = ''
masked_stderr = 'time="2022-11-28T01:32:25Z" level=info msg="✅ Exists: NooBaa \\"noobaa\\"\\n"\ntime="2022-11-28T01:32:25Z" level=info...msg="❌ Could not get StorageClass \\"ocs-storagecluster-ceph-rbd\\" for system in namespace \\"openshift-storage\\""\n'

    def exec_cmd(
        cmd,
        secrets=None,
        timeout=600,
        ignore_error=False,
        threading_lock=None,
        silent=False,
        **kwargs,
    ):
        """
        Run an arbitrary command locally

        If the command is grep and matching pattern is not found, then this function
        returns "command terminated with exit code 1" in stderr.

        Args:
            cmd (str): command to run
            secrets (list): A list of secrets to be masked with asterisks
                This kwarg is popped in order to not interfere with
                subprocess.run(``**kwargs``)
            timeout (int): Timeout for the command, defaults to 600 seconds.
            ignore_error (bool): True if ignore non zero return code and do not
                raise the exception.
            threading_lock (threading.Lock): threading.Lock object that is used
                for handling concurrent oc commands
            silent (bool): If True will silent errors from the server, default false

        Raises:
            CommandFailed: In case the command execution fails

        Returns:
            (CompletedProcess) A CompletedProcess object of the command that was executed
            CompletedProcess attributes:
            args: The list or str args passed to run().
            returncode (str): The exit code of the process, negative for signals.
            stdout     (str): The standard output (None if not captured).
            stderr     (str): The standard error (None if not captured).

        """
        masked_cmd = mask_secrets(cmd, secrets)
        log.info(f"Executing command: {masked_cmd}")
        if isinstance(cmd, str):
            cmd = shlex.split(cmd)
        if threading_lock and cmd[0] == "oc":
            threading_lock.acquire()
        completed_process = subprocess.run(
            cmd,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            stdin=subprocess.PIPE,
            timeout=timeout,
            **kwargs,
        )
        if threading_lock and cmd[0] == "oc":
            threading_lock.release()
        masked_stdout = mask_secrets(completed_process.stdout.decode(), secrets)
        if len(completed_process.stdout) > 0:
            log.debug(f"Command stdout: {masked_stdout}")
        else:
            log.debug("Command stdout is empty")

        masked_stderr = mask_secrets(completed_process.stderr.decode(), secrets)
        if len(completed_process.stderr) > 0:
            if not silent:
                log.warning(f"Command stderr: {masked_stderr}")
        else:
            log.debug("Command stderr is empty")
        log.debug(f"Command return code: {completed_process.returncode}")
        if completed_process.returncode and not ignore_error:
            if (
                "grep" in masked_cmd
                and b"command terminated with exit code 1" in completed_process.stderr
            ):
                log.info(f"No results found for grep command: {masked_cmd}")
            else:
>               raise CommandFailed(
                    f"Error during execution of command: {masked_cmd}."
                    f"\nError is {masked_stderr}"
                )
E               ocs_ci.ocs.exceptions.CommandFailed: Error during execution of command: /home/jenkins/workspace/qe-deploy-ocs-cluster-prod/ocs-ci/data/mcg-cli backingstore create pv-pool pv-backingstore-216e7721521d4379baea7305 --num-volumes 1 --pv-size-gb 50 --storage-class ocs-storagecluster-ceph-rbd -n openshift-storage.
E               Error is time="2022-11-28T01:32:25Z" level=info msg="✅ Exists: NooBaa \"noobaa\"\n"
E               time="2022-11-28T01:32:25Z" level=info msg="❌ Not Found: StorageClass \"ocs-storagecluster-ceph-rbd\"\n"
E               time="2022-11-28T01:32:25Z" level=fatal msg="❌ Could not get StorageClass \"ocs-storagecluster-ceph-rbd\" for system in namespace \"openshift-storage\""

ocs_ci/utility/utils.py:636: CommandFailed
red-hat-storage / ocs-ci
