Unit test get_bid_ask_realtime_raw_data_reader

[sonaalKant]

Lets add unit test for this method https://github.com/kaizen-ai/kaizenflow/blob/423470c58122af11b12a1b9c3ea889abcad52342/im_v2/common/data/client/im_raw_data_client.py#L373

Test if bid_ask_db_signature is correctly built from the params.

Input params: stage = 'preprod', data_vendor = 'MOCK_VENDOR', universe_version = 'v7_5`

FYI @samarth9008

[smitpatel49]

I encountered an Assertion error in parsing the signature:

Traceback (most recent call last):
  File "/usr/lib/python3.9/unittest/case.py", line 59, in testPartExecutor
    yield
  File "/usr/lib/python3.9/unittest/case.py", line 593, in run
    self._callTestMethod(testMethod)
  File "/usr/lib/python3.9/unittest/case.py", line 550, in _callTestMethod
    method()
  File "/app/im_v2/common/data/client/test/test_im_raw_data_client.py", line 1158, in test1
    actual = imvcdcimrdc.get_bid_ask_realtime_raw_data_reader(
  File "/app/im_v2/common/data/client/im_raw_data_client.py", line 391, in get_bid_ask_realtime_raw_data_reader
    bid_ask_raw_data_reader = RawDataReader(bid_ask_db_signature, stage=stage)
  File "/app/im_v2/common/data/client/im_raw_data_client.py", line 45, in __init__
    self.args = dsdascut.parse_dataset_signature_to_args(
  File "/app/data_schema/dataset_schema_utils.py", line 258, in parse_dataset_signature_to_args
    hdbg.dassert_eq(validate_dataset_signature(signature, dataset_schema), True)
  File "/app/helpers/hdbg.py", line 172, in dassert_eq
    _dfatal(txt, msg, *args, only_warning=only_warning)
  File "/app/helpers/hdbg.py", line 142, in _dfatal
    dfatal(dfatal_txt)
  File "/app/helpers/hdbg.py", line 71, in dfatal
    raise assertion_type(ret)
AssertionError: 
################################################################################
* Failed assertion *
'False'
==
'True'
################################################################################

[sonaalKant]

@smitpatel49 did you try to debug why the assertion was raised?

[smitpatel49]

I tried and I think that validate_dataset_signature() does not produce the schema we need. I tried using 'CCXT' as input parameter but I don't think I have the aws credentials.

[mihir1906]

For the input parameters:
stage = 'preprod'
data_vendor = 'MOCK_VENDOR'
universe_version = 'v7_5'
exchange_id = 'binance'

I'm getting the following error: ____ ERROR collecting research_amp/soccer_prediction/paired_ttest.py ____ Traceback (most recent call last): File "/venv/lib/python3.9/site-packages/_pytest/runner.py", line 341, in from_call result: Optional[TResult] = func() File "/venv/lib/python3.9/site-packages/_pytest/runner.py", line 372, in call = CallInfo.from_call(lambda: list(collector.collect()), "collect") File "/venv/lib/python3.9/site-packages/_pytest/python.py", line 531, in collect self._inject_setup_module_fixture() File "/venv/lib/python3.9/site-packages/_pytest/python.py", line 545, in _inject_setup_module_fixture self.obj, ("setUpModule", "setup_module") File "/venv/lib/python3.9/site-packages/_pytest/python.py", line 310, in obj self._obj = obj = self._getobj() File "/venv/lib/python3.9/site-packages/_pytest/python.py", line 528, in _getobj return self._importtestmodule() File "/venv/lib/python3.9/site-packages/_pytest/python.py", line 617, in _importtestmodule mod = import_path(self.path, mode=importmode, root=self.config.rootpath) File "/venv/lib/python3.9/site-packages/_pytest/pathlib.py", line 567, in import_path importlib.import_module(module_name) File "/usr/lib/python3.9/importlib/init.py", line 127, in import_module return _bootstrap._gcd_import(name[level:], package, level) File "", line 1030, in _gcd_import File "", line 1007, in _find_and_load File "", line 986, in _find_and_load_unlocked File "", line 680, in _load_unlocked File "/venv/lib/python3.9/site-packages/_pytest/assertion/rewrite.py", line 178, in exec_module exec(co, module.dict) File "/app/research_amp/soccer_prediction/paired_t_test.py", line 104, in glm_predictions = get_predictions_data(bucket_name=bucket_name, dataset_path=s3_file_path) File "/app/research_amp/soccer_prediction/paired_t_test.py", line 62, in get_predictions_data s3 = haws.get_service_resource(aws_profile="ck", service_name="s3") File "/app/helpers/haws.py", line 75, in get_service_resource session = get_session(aws_profile) File "/app/helpers/haws.py", line 44, in get_session credentials = hs3.get_aws_credentials(aws_profile=aws_profile) File "/app/helpers/hs3.py", line 723, in get_aws_credentials config = _get_aws_config(file_name) File "/app/helpers/hs3.py", line 520, in _get_aws_config hdbg.dassert_file_exists(file_name) File "/app/helpers/hdbg.py", line 785, in dassert_file_exists _dfatal(txt, msg, *args, only_warning=only_warning) File "/app/helpers/hdbg.py", line 142, in _dfatal dfatal(dfatal_txt) File "/app/helpers/hdbg.py", line 71, in dfatal raise assertion_type(ret) AssertionError: ################################################################################

• Failed assertion * File '/home/.aws/credentials' doesn't exist ################################################################################

The test accesses the AWS credentials that don't exist in the specified path. @sonaalKant any suggestions on how this can be handled?

[mihir1906]

@sonaalKant as I don't have the file '/home/.aws/credentials', am I supposed to use a library like moto to mock AWS services for testing purposes?