search

automl / CARP-S

A Framework for Comparing N Hyperparameter Optimizers on M Benchmarks.
https://automl.github.io/CARP-S/latest/
Other
8 stars 0 forks source link

[Tests] Clean install results in test 2 failing #58

Closed eddiebergman closed 6 months ago

eddiebergman commented 6 months ago 
pytest
@pytest.mark.parametrize(
        ("path", "key"),
        [
            ("carps/configs/optimizer", "optimizer_id"),
            ("carps/configs/problem", "problem_id"),
        ],
    )
    def test_unique_ids(path, key):
        path = Path(path)
        paths = list(path.glob("**/*.yaml"))
        values = []
        for p in paths:
            cfg = OmegaConf.load(p)
            value = cfg.get(key)
            values.append(value)

>       assert len(set(values)) == len(values), (
            "Duplicate " f"{key}, they need to have a unique name: {[k for k, v in Counter(values).items() if v > 1]}"
        )
E       AssertionError: Duplicate problem_id, they need to have a unique name: ['yahpo/so/lcbench/189873/None', 'yahpo/so/rbv2_super/1457/None', 'yahpo/so/rbv2_xgboost/12/None', 'yahpo/so/rbv2_rpart/40499/None', 'yahpo/so/rbv2_super/1053/None']
E       assert 4479 == 4484
E        +  where 4479 = len({'dummy', 'hpob/124/41', 'hpob/124/59', 'hpob/151/12', 'hpob/151/20', 'hpob/151/28', ...})
E        +    where {'dummy', 'hpob/124/41', 'hpob/124/59', 'hpob/151/12', 'hpob/151/20', 'hpob/151/28', ...} = set(['yahpo/rbv2_ranger/1590/None', 'yahpo/rbv2_svm/40670/None', 'yahpo/rbv2_aknn/37/None', 'yahpo/rbv2_rpart/40685/None', 'yahpo/rbv2_glmnet/1056/None', 'yahpo/rbv2_aknn/28/None', ...])
E        +  and   4484 = len(['yahpo/rbv2_ranger/1590/None', 'yahpo/rbv2_svm/40670/None', 'yahpo/rbv2_aknn/37/None', 'yahpo/rbv2_rpart/40685/None', 'yahpo/rbv2_glmnet/1056/None', 'yahpo/rbv2_aknn/28/None', ...])
eddiebergman commented 6 months ago

Closing and going to give a summary on CI in a seperate issue