megvii-research / hpman

A hyperparameter manager for deep learning experiments.
MIT License
95 stars 11 forks source link

fix: replace attrdict #25

Closed Kingfish404 closed 2 years ago

Kingfish404 commented 2 years ago

All tests pass for python3.7 and python3.6, also work fine in internal projects.

Warning: on python3.9 and upper, some tests fail:

test shell log
λ ~/Developer/git/hpman/ master python3.9 -m pytest
=================================================================================== test session starts ===================================================================================
platform darwin -- Python 3.9.9, pytest-6.2.5, py-1.10.0, pluggy-1.0.0
rootdir: /Users/jinyu/Developer/git/hpman
plugins: anyio-2.0.2, cov-3.0.0
collected 44 items                                                                                                                                                                        

tests/test_callable.py ......                                                                                                                                                       [ 13%]
tests/test_hpm_db.py .........                                                                                                                                                      [ 34%]
tests/test_hpm_zoo.py .                                                                                                                                                             [ 36%]
tests/test_parse_file.py .....                                                                                                                                                      [ 47%]
tests/test_parse_source.py ....F........F....                                                                                                                                       [ 88%]
tests/test_setget.py ...                                                                                                                                                            [ 95%]
tests/test_source_helper.py ..                                                                                                                                                      [100%]

======================================================================================== FAILURES =========================================================================================
____________________________________________________________________________ TestParseSource.test_parse_hints _____________________________________________________________________________

self = 
test_datas = {"_('hp0', 1, a=1, b=2, c={'d': 3, 'e': 4})": ['hp0', 1, , {'a': 1, 'b': 2, 'c': {'d': 3, 'e': 4}}], ...a=[1, 3, 4], b=2, c={'d': 3, 'e': 4})": ['hp1', 1, , {'a': [1, 3, 4], 'b': 2, 'c': {'d': 3, 'e': 4}}]}

    def _run(self, test_datas):
        """test_data spec:
            {
                '_(key, value)': [
                    key, value
                ],
                ...
            }
        """
        try:
            for expression, kv in test_datas.items():
                name = kv[0]
                value = kv[1]
                ast_node_type = kv[2]

                m = self.hpm.parse_source(expression)
                # check default value
                if isinstance(value, hpman.NotLiteralEvaluable):
                    self.assertEqual(type(m.get_value(name)), hpman.NotLiteralEvaluable)
                else:
                    self.assertEqual(m.get_value(name), value)

                # check ast node type
>               self.assertEqual(
                    type(m.get_occurrence_of_value(name)["ast_node"]), ast_node_type
                )

tests/test_parse_source.py:227: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = , first = , second = , msg = None

    def assertEqual(self, first, second, msg=None):
        """Fail if the two objects are unequal as determined by the '=='
           operator.
        """
        assertion_func = self._getAssertEqualityFunc(first, second)
>       assertion_func(first, second, msg=msg)

/usr/local/Cellar/python@3.9/3.9.9/Frameworks/Python.framework/Versions/3.9/lib/python3.9/unittest/case.py:837: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = , first = , second = , msg = " != "

    def _baseAssertEqual(self, first, second, msg=None):
        """The default assertEqual implementation, not type specific."""
        if not first == second:
            standardMsg = '%s != %s' % _common_shorten_repr(first, second)
            msg = self._formatMessage(msg, standardMsg)
>           raise self.failureException(msg)
E           AssertionError:  != 

/usr/local/Cellar/python@3.9/3.9.9/Frameworks/Python.framework/Versions/3.9/lib/python3.9/unittest/case.py:830: AssertionError

During handling of the above exception, another exception occurred:

self = 

    def test_parse_hints(self):
>       self._run(
            {
                "_('hp0', 1, a=1, b=2, c={'d': 3, 'e': 4})": [
                    "hp0",
                    1,
                    ast.Num,
                    {"a": 1, "b": 2, "c": {"d": 3, "e": 4}},
                ],
                "_('hp1', 1, a=[1, 3, 4], b=2, c={'d': 3, 'e': 4})": [
                    "hp1",
                    1,
                    ast.Num,
                    {"a": [1, 3, 4], "b": 2, "c": {"d": 3, "e": 4}},
                ],
            }
        )

tests/test_parse_source.py:34: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
tests/test_parse_source.py:237: in _run
    self.fail("parse failed with exception: {}".format(e))
E   AssertionError: parse failed with exception:  != 
________________________________________________________________________ TestParseSource.test_parse_type_with_pod _________________________________________________________________________

self = 
test_datas = {"_('hp_float', 3.14)": ['hp_float', 3.14, ], "_('hp_float_ieee', 1e-5)": ['hp_float_ieee', 1e-05, ], "_('hp_int_hex', 0x18)": ['hp_int_hex', 24, ], ...}

    def _run(self, test_datas):
        """test_data spec:
            {
                '_(key, value)': [
                    key, value
                ],
                ...
            }
        """
        try:
            for expression, kv in test_datas.items():
                name = kv[0]
                value = kv[1]
                ast_node_type = kv[2]

                m = self.hpm.parse_source(expression)
                # check default value
                if isinstance(value, hpman.NotLiteralEvaluable):
                    self.assertEqual(type(m.get_value(name)), hpman.NotLiteralEvaluable)
                else:
                    self.assertEqual(m.get_value(name), value)

                # check ast node type
>               self.assertEqual(
                    type(m.get_occurrence_of_value(name)["ast_node"]), ast_node_type
                )

tests/test_parse_source.py:227: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = , first = , second = , msg = None

    def assertEqual(self, first, second, msg=None):
        """Fail if the two objects are unequal as determined by the '=='
           operator.
        """
        assertion_func = self._getAssertEqualityFunc(first, second)
>       assertion_func(first, second, msg=msg)

/usr/local/Cellar/python@3.9/3.9.9/Frameworks/Python.framework/Versions/3.9/lib/python3.9/unittest/case.py:837: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = , first = , second = 
msg = " != "

    def _baseAssertEqual(self, first, second, msg=None):
        """The default assertEqual implementation, not type specific."""
        if not first == second:
            standardMsg = '%s != %s' % _common_shorten_repr(first, second)
            msg = self._formatMessage(msg, standardMsg)
>           raise self.failureException(msg)
E           AssertionError:  != 

/usr/local/Cellar/python@3.9/3.9.9/Frameworks/Python.framework/Versions/3.9/lib/python3.9/unittest/case.py:830: AssertionError

During handling of the above exception, another exception occurred:

self = 

    def test_parse_type_with_pod(self):
>       self._run(
            {
                "_('hp_int', 123)": ["hp_int", 123, ast.Num],
                "_('hp_int_hex', 0x18)": ["hp_int_hex", 0x18, ast.Num],
                "_('hp_float', 3.14)": ["hp_float", 3.14, ast.Num],
                "_('hp_float_ieee', 1e-5)": ["hp_float_ieee", 1e-5, ast.Num],
                "_('hp_str', 'string')": ["hp_str", "string", ast.Str],
            }
        )

tests/test_parse_source.py:23: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
tests/test_parse_source.py:237: in _run
    self.fail("parse failed with exception: {}".format(e))
E   AssertionError: parse failed with exception:  != 
================================================================================= short test summary info =================================================================================
FAILED tests/test_parse_source.py::TestParseSource::test_parse_hints - AssertionError: parse failed with exception:  != 
FAILED tests/test_parse_source.py::TestParseSource::test_parse_type_with_pod - AssertionError: parse failed with exception:  != 
============================================================================== 2 failed, 42 passed in 0.43s ===============================================================================
Kingfish404 commented 2 years ago

Split from #23 because attrdict's replace could case fault and need more test.