brain-score / core

MIT License
1 stars 6 forks source link

improve error message when plugin not found #16

Open mschrimpf opened 1 year ago

mschrimpf commented 1 year ago

If a plugin currently is not found, the error message looks like this:

>       actual_score = score(model_identifier=model_identifier, benchmark_identifier=benchmark_identifier)

test_integration.py:17: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../../brain-score_language/brainscore_language/__init__.py:79: in score
    return wrap_score(__file__,
../../brain-score_core/brainscore_core/plugin_management/conda_score.py:87: in wrap_score
    result = score_function(model_identifier, benchmark_identifier)
../../brain-score_language/brainscore_language/__init__.py:54: in _run_score
    model: ArtificialSubject = load_model(model_identifier)
../../brain-score_language/brainscore_language/__init__.py:42: in load_model
    import_plugin('brainscore_language', 'models', identifier)
../../brain-score_core/brainscore_core/plugin_management/import_plugin.py:79: in import_plugin
    importer = ImportPlugin(library_root, plugin_type, identifier)
../../brain-score_core/brainscore_core/plugin_management/import_plugin.py:19: in __init__
    self.plugin_dirname = self.locate_plugin()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <brainscore_core.plugin_management.import_plugin.ImportPlugin object at 0x7fd639994ca0>

    def locate_plugin(self) -> str:
        """
        Searches all `plugin_type` __init.py__ files for the plugin denoted with `identifier`.
        If a match is found of format {plugin_type}_registry[{identifier}],
        returns name of directory where __init.py__ is located
        """
        plugins = [d.name for d in self.plugins_dir.iterdir() if d.is_dir()]

        specified_plugin_dirname = None
        plugin_registrations_count = 0
        for plugin_dirname in plugins:
            if plugin_dirname.startswith('.') or plugin_dirname.startswith('_'):  # ignore e.g. __pycache__
                continue
            plugin_dirpath = self.plugins_dir / plugin_dirname
            init_file = plugin_dirpath / "__init__.py"
            with open(init_file) as f:
                registry_name = self.plugin_type.strip(
                    's') + '_registry'  # remove plural and determine variable name, e.g. "models" -> "model_registry"
                plugin_registrations = [line for line in f if f"{registry_name}['{self.identifier}']"
                                        in line.replace('\"', '\'')]
                if len(plugin_registrations) > 0:
                    specified_plugin_dirname = plugin_dirname
                    plugin_registrations_count += 1

>       assert plugin_registrations_count > 0, f"No registrations found for {self.identifier}"
E       AssertionError: No registrations found for pixels

../../brain-score_core/brainscore_core/plugin_management/import_plugin.py:45: AssertionError

I can see this quickly being overwhelming to the average user, e.g. "registrations" probably does not make immediate sense.