search

SpikeInterface / spikeinterface

A Python-based module for creating flexible and robust spike sorting pipelines.
https://spikeinterface.readthedocs.io
MIT License
492 stars 188 forks source link

Error running read_sorter_folder after installing pre-release #3136

Open twheatcroft07 opened 2 months ago

twheatcroft07 commented 2 months ago

Hi there,

I want to use the SortingAnalyzer, so I created a new environment, and installed spikeinterface like so (from here)

pip install --pre spikeinterface

When I try to run the below code I get an error. This code worked fine in the normal version of spikeinterface.

sorting_KS = si.read_sorter_folder(kilosort_output_folder, register_recording=False, sorting_info=False)

The error is below

THANKS!

{
    "name": "UndefinedVariableError",
    "message": "name 'np' is not defined",
    "stack": "---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\scope.py:231, in Scope.resolve(self, key, is_local)
    230 if self.has_resolvers:
--> 231     return self.resolvers[key]
    233 # if we're here that means that we have no locals and we also have
    234 # no resolvers

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\collections\\__init__.py:1015, in ChainMap.__getitem__(self, key)
   1014         pass
-> 1015 return self.__missing__(key)

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\collections\\__init__.py:1007, in ChainMap.__missing__(self, key)
   1006 def __missing__(self, key):
-> 1007     raise KeyError(key)

KeyError: 'np'

During handling of the above exception, another exception occurred:

KeyError                                  Traceback (most recent call last)
File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\scope.py:242, in Scope.resolve(self, key, is_local)
    238 try:
    239     # last ditch effort we look in temporaries
    240     # these are created when parsing indexing expressions
    241     # e.g., df[df > 0]
--> 242     return self.temps[key]
    243 except KeyError as err:

KeyError: 'np'

The above exception was the direct cause of the following exception:

UndefinedVariableError                    Traceback (most recent call last)
Cell In[6], line 55
     52     print(f'KS2.5 sorting completed for {local_path}')
     53 else:
     54     # Read the existing sorter folder
---> 55     sorting_KS = si.read_sorter_folder(kilosort_output_folder, register_recording=False, sorting_info=False)
     56     print(f'KS2.5 folder already exists for {local_path}, skipping spike sorting.')
     58 if run_ironclust:
     59     # Define the IronClust output folder

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\spikeinterface\\sorters\\runsorter.py:674, in read_sorter_folder(folder, register_recording, sorting_info, raise_error)
    672 sorter_name = log[\"sorter_name\"]
    673 SorterClass = sorter_dict[sorter_name]
--> 674 sorting = SorterClass.get_result_from_folder(
    675     folder, register_recording=register_recording, sorting_info=sorting_info
    676 )
    677 return sorting

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\spikeinterface\\sorters\\basesorter.py:318, in BaseSorter.get_result_from_folder(cls, output_folder, register_recording, sorting_info)
    312     raise SpikeSortingError(
    313         f\"Spike sorting error trace:\
{log['error_trace']}\
\"
    314         f\"Spike sorting failed. You can inspect the runtime trace in {output_folder}/spikeinterface_log.json.\"
    315     )
    317 if sorter_output_folder.is_dir():
--> 318     sorting = cls._get_result_from_folder(sorter_output_folder)
    319 else:
    320     # back-compatibility
    321     sorting = cls._get_result_from_folder(output_folder)

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\spikeinterface\\sorters\\external\\kilosortbase.py:260, in KilosortBase._get_result_from_folder(cls, sorter_output_folder)
    258     sorter_params = json.load(f)[\"sorter_params\"]
    259 keep_good_only = sorter_params.get(\"keep_good_only\", False)
--> 260 sorting = KiloSortSortingExtractor(folder_path=sorter_output_folder, keep_good_only=keep_good_only)
    261 return sorting

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\spikeinterface\\extractors\\phykilosortextractors.py:278, in KiloSortSortingExtractor.__init__(self, folder_path, keep_good_only, remove_empty_units)
    277 def __init__(self, folder_path: Path | str, keep_good_only: bool = False, remove_empty_units: bool = True):
--> 278     BasePhyKilosortSortingExtractor.__init__(
    279         self,
    280         folder_path,
    281         exclude_cluster_groups=None,
    282         keep_good_only=keep_good_only,
    283         remove_empty_units=remove_empty_units,
    284     )
    286     self._kwargs = {\"folder_path\": str(Path(folder_path).absolute()), \"keep_good_only\": keep_good_only}

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\spikeinterface\\extractors\\phykilosortextractors.py:122, in BasePhyKilosortSortingExtractor.__init__(self, folder_path, exclude_cluster_groups, keep_good_only, remove_empty_units, load_all_cluster_properties)
    119     del cluster_info[\"id\"]
    121 if remove_empty_units:
--> 122     cluster_info = cluster_info.query(f\"cluster_id in {unique_unit_ids}\")
    124 # update spike clusters and times values
    125 bad_clusters = [clust for clust in clust_id if clust not in cluster_info[\"cluster_id\"].values]

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\frame.py:4823, in DataFrame.query(self, expr, inplace, **kwargs)
   4821 kwargs[\"level\"] = kwargs.pop(\"level\", 0) + 1
   4822 kwargs[\"target\"] = None
-> 4823 res = self.eval(expr, **kwargs)
   4825 try:
   4826     result = self.loc[res]

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\frame.py:4949, in DataFrame.eval(self, expr, inplace, **kwargs)
   4946     kwargs[\"target\"] = self
   4947 kwargs[\"resolvers\"] = tuple(kwargs.get(\"resolvers\", ())) + resolvers
-> 4949 return _eval(expr, inplace=inplace, **kwargs)

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\eval.py:336, in eval(expr, parser, engine, local_dict, global_dict, resolvers, level, target, inplace)
    327 # get our (possibly passed-in) scope
    328 env = ensure_scope(
    329     level + 1,
    330     global_dict=global_dict,
   (...)
    333     target=target,
    334 )
--> 336 parsed_expr = Expr(expr, engine=engine, parser=parser, env=env)
    338 if engine == \"numexpr\" and (
    339     is_extension_array_dtype(parsed_expr.terms.return_type)
    340     or getattr(parsed_expr.terms, \"operand_types\", None) is not None
   (...)
    344     )
    345 ):
    346     warnings.warn(
    347         \"Engine has switched to 'python' because numexpr does not support \"
    348         \"extension array dtypes. Please set your engine to python manually.\",
    349         RuntimeWarning,
    350         stacklevel=find_stack_level(),
    351     )

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\expr.py:809, in Expr.__init__(self, expr, engine, parser, env, level)
    807 self.parser = parser
    808 self._visitor = PARSERS[parser](self.env, self.engine, self.parser)
--> 809 self.terms = self.parse()

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\expr.py:828, in Expr.parse(self)
    824 def parse(self):
    825     \"\"\"
    826     Parse an expression.
    827     \"\"\"
--> 828     return self._visitor.visit(self.expr)

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\expr.py:412, in BaseExprVisitor.visit(self, node, **kwargs)
    410 method = f\"visit_{type(node).__name__}\"
    411 visitor = getattr(self, method)
--> 412 return visitor(node, **kwargs)

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\expr.py:418, in BaseExprVisitor.visit_Module(self, node, **kwargs)
    416     raise SyntaxError(\"only a single expression is allowed\")
    417 expr = node.body[0]
--> 418 return self.visit(expr, **kwargs)

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\expr.py:412, in BaseExprVisitor.visit(self, node, **kwargs)
    410 method = f\"visit_{type(node).__name__}\"
    411 visitor = getattr(self, method)
--> 412 return visitor(node, **kwargs)

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\expr.py:421, in BaseExprVisitor.visit_Expr(self, node, **kwargs)
    420 def visit_Expr(self, node, **kwargs):
--> 421     return self.visit(node.value, **kwargs)

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\expr.py:412, in BaseExprVisitor.visit(self, node, **kwargs)
    410 method = f\"visit_{type(node).__name__}\"
    411 visitor = getattr(self, method)
--> 412 return visitor(node, **kwargs)

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\expr.py:719, in BaseExprVisitor.visit_Compare(self, node, **kwargs)
    717     op = self.translate_In(ops[0])
    718     binop = ast.BinOp(op=op, left=node.left, right=comps[0])
--> 719     return self.visit(binop)
    721 # recursive case: we have a chained comparison, a CMP b CMP c, etc.
    722 left = node.left

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\expr.py:412, in BaseExprVisitor.visit(self, node, **kwargs)
    410 method = f\"visit_{type(node).__name__}\"
    411 visitor = getattr(self, method)
--> 412 return visitor(node, **kwargs)

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\expr.py:532, in BaseExprVisitor.visit_BinOp(self, node, **kwargs)
    531 def visit_BinOp(self, node, **kwargs):
--> 532     op, op_class, left, right = self._maybe_transform_eq_ne(node)
    533     left, right = self._maybe_downcast_constants(left, right)
    534     return self._maybe_evaluate_binop(op, op_class, left, right)

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\expr.py:454, in BaseExprVisitor._maybe_transform_eq_ne(self, node, left, right)
    452     left = self.visit(node.left, side=\"left\")
    453 if right is None:
--> 454     right = self.visit(node.right, side=\"right\")
    455 op, op_class, left, right = self._rewrite_membership_op(node, left, right)
    456 return op, op_class, left, right

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\expr.py:412, in BaseExprVisitor.visit(self, node, **kwargs)
    410 method = f\"visit_{type(node).__name__}\"
    411 visitor = getattr(self, method)
--> 412 return visitor(node, **kwargs)

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\expr.py:564, in BaseExprVisitor.visit_List(self, node, **kwargs)
    563 def visit_List(self, node, **kwargs) -> Term:
--> 564     name = self.env.add_tmp([self.visit(e)(self.env) for e in node.elts])
    565     return self.term_type(name, self.env)

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\expr.py:412, in BaseExprVisitor.visit(self, node, **kwargs)
    410 method = f\"visit_{type(node).__name__}\"
    411 visitor = getattr(self, method)
--> 412 return visitor(node, **kwargs)

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\expr.py:659, in BaseExprVisitor.visit_Call(self, node, side, **kwargs)
    657 def visit_Call(self, node, side=None, **kwargs):
    658     if isinstance(node.func, ast.Attribute) and node.func.attr != \"__call__\":
--> 659         res = self.visit_Attribute(node.func)
    660     elif not isinstance(node.func, ast.Name):
    661         raise TypeError(\"Only named functions are supported\")

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\expr.py:644, in BaseExprVisitor.visit_Attribute(self, node, **kwargs)
    641 ctx = node.ctx
    642 if isinstance(ctx, ast.Load):
    643     # resolve the value
--> 644     resolved = self.visit(value).value
    645     try:
    646         v = getattr(resolved, attr)

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\expr.py:412, in BaseExprVisitor.visit(self, node, **kwargs)
    410 method = f\"visit_{type(node).__name__}\"
    411 visitor = getattr(self, method)
--> 412 return visitor(node, **kwargs)

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\expr.py:545, in BaseExprVisitor.visit_Name(self, node, **kwargs)
    544 def visit_Name(self, node, **kwargs) -> Term:
--> 545     return self.term_type(node.id, self.env, **kwargs)

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\ops.py:91, in Term.__init__(self, name, env, side, encoding)
     89 tname = str(name)
     90 self.is_local = tname.startswith(LOCAL_TAG) or tname in DEFAULT_GLOBALS
---> 91 self._value = self._resolve_name()
     92 self.encoding = encoding

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\ops.py:115, in Term._resolve_name(self)
    110 if local_name in self.env.scope and isinstance(
    111     self.env.scope[local_name], type
    112 ):
    113     is_local = False
--> 115 res = self.env.resolve(local_name, is_local=is_local)
    116 self.update(res)
    118 if hasattr(res, \"ndim\") and res.ndim > 2:

File c:\\Users\\sabatini\\.conda\\envs\\twSI\\Lib\\site-packages\\pandas\\core\\computation\\scope.py:244, in Scope.resolve(self, key, is_local)
    242     return self.temps[key]
    243 except KeyError as err:
--> 244     raise UndefinedVariableError(key, is_local) from err

UndefinedVariableError: name 'np' is not defined"
}
alejoe91 commented 2 months ago

Hi @twheatcroft07

Seems like a pandas issue? What numpy/pandas versions you have installed?

We are forcing numpy < 2, and maybe your pandas version is too recent

Jianfengliu0413 commented 1 month ago

it seems "spikeinterface_log.json" is not supported at all, instead, si_folder.json and structures have changed.

alejoe91 commented 1 month ago

right, if this was saved from an old version this might be the reason!

@twheatcroft07 you can always load the KS folder directly (should be in kilosort_output_folder/sorter_output) with:

sorting=  se.read_kilosort(kilosort_output_folder/sorter_output)