`delete_downstream_merge` should capture all parts with fk-refs

[MichaelCoulter]

im trying to run this command:

(Nwbfile & {'nwb_file_name':'BS2820231107_.nwb'}).delete()

and i get this error:

Error stack

```python --------------------------------------------------------------------------- NetworkXError Traceback (most recent call last) File ~/spyglass/src/spyglass/utils/dj_mixin.py:237, in SpyglassMixin._merge_tables(self) 236 try: --> 237 _ = search_descendants(self) 238 except NetworkXError as e: File ~/spyglass/src/spyglass/utils/dj_mixin.py:234, in SpyglassMixin._merge_tables..search_descendants(parent) 233 visited.add(master_name) --> 234 search_descendants(master_ft) File ~/spyglass/src/spyglass/utils/dj_mixin.py:222, in SpyglassMixin._merge_tables..search_descendants(parent) 217 def search_descendants(parent): 218 # TODO: Add check that parents are in the graph. If not, raise error 219 # asking user to import the table. 220 # TODO: Make a `is_merge_table` helper, and check for false 221 # positives in the mixin init. --> 222 for desc in parent.descendants(as_objects=True): 223 if ( 224 MERGE_PK not in desc.heading.names 225 or not (master_name := get_master(desc.full_table_name)) 226 or master_name in merge_tables 227 ): File ~/anaconda3/envs/spyglass2/lib/python3.9/site-packages/datajoint/table.py:212, in Table.descendants(self, as_objects) 205 """ 206 207 :param as_objects: False - a list of table names; True - a list of table objects. 208 :return: list of tables descendants in topological order. 209 """ 210 return [ 211 FreeTable(self.connection, node) if as_objects else node --> 212 for node in self.connection.dependencies.descendants(self.full_table_name) 213 if not node.isdigit() 214 ] File ~/anaconda3/envs/spyglass2/lib/python3.9/site-packages/datajoint/dependencies.py:170, in Dependencies.descendants(self, full_table_name) 169 self.load(force=False) --> 170 nodes = self.subgraph(nx.algorithms.dag.descendants(self, full_table_name)) 171 return unite_master_parts( 172 [full_table_name] + list(nx.algorithms.dag.topological_sort(nodes)) 173 ) File ~/anaconda3/envs/spyglass2/lib/python3.9/site-packages/networkx/algorithms/dag.py:53, in descendants(G, source) 52 if not G.has_node(source): ---> 53 raise nx.NetworkXError(f"The node {source} is not in the graph.") 54 des = {n for n, d in nx.shortest_path_length(G, source=source).items()} NetworkXError: The node `position_linearization_merge`.`linearized_position_output` is not in the graph. During handling of the above exception, another exception occurred: ValueError Traceback (most recent call last) Cell In [4], line 1 ----> 1 (Nwbfile & {'nwb_file_name':'BS2820231107_.nwb'}).delete() File ~/spyglass/src/spyglass/utils/dj_mixin.py:587, in SpyglassMixin.delete(self, force_permission, *args, **kwargs) 585 def delete(self, force_permission=False, *args, **kwargs): 586 """Alias for cautious_delete, overwrites datajoint.table.Table.delete""" --> 587 self.cautious_delete(force_permission=force_permission, *args, **kwargs) File ~/spyglass/src/spyglass/utils/dj_mixin.py:550, in SpyglassMixin.cautious_delete(self, force_permission, *args, **kwargs) 547 if not force_permission: 548 self._check_delete_permission() --> 550 merge_deletes = self.delete_downstream_merge( 551 dry_run=True, 552 disable_warning=True, 553 return_parts=False, 554 ) 556 safemode = ( 557 dj.config.get("safemode", True) 558 if kwargs.get("safemode") is None 559 else kwargs["safemode"] 560 ) 562 if merge_deletes: File ~/spyglass/src/spyglass/utils/dj_mixin.py:339, in SpyglassMixin.delete_downstream_merge(self, restriction, dry_run, reload_cache, disable_warning, return_parts, **kwargs) 336 restriction = restriction or self.restriction or True 338 merge_join_dict = {} --> 339 for name, chain in self._merge_chains.items(): 340 join = chain.join(restriction) 341 if join: File ~/anaconda3/envs/spyglass2/lib/python3.9/functools.py:993, in cached_property.__get__(self, instance, owner) 991 val = cache.get(self.attrname, _NOT_FOUND) 992 if val is _NOT_FOUND: --> 993 val = self.func(instance) 994 try: 995 cache[self.attrname] = val File ~/spyglass/src/spyglass/utils/dj_mixin.py:262, in SpyglassMixin._merge_chains(self) 251 """Dict of chains to merges downstream of self 252 253 Format: {full_table_name: TableChains}. (...) 259 delete_downstream_merge call. 260 """ 261 merge_chains = {} --> 262 for name, merge_table in self._merge_tables.items(): 263 chains = TableChains(self, merge_table, connection=self.connection) 264 if len(chains): File ~/anaconda3/envs/spyglass2/lib/python3.9/functools.py:993, in cached_property.__get__(self, instance, owner) 991 val = cache.get(self.attrname, _NOT_FOUND) 992 if val is _NOT_FOUND: --> 993 val = self.func(instance) 994 try: 995 cache[self.attrname] = val File ~/spyglass/src/spyglass/utils/dj_mixin.py:240, in SpyglassMixin._merge_tables(self) 238 except NetworkXError as e: 239 table_name = "".join(e.args[0].split("`")[1:4]) --> 240 raise ValueError(f"Please import {table_name} and try again.") 242 logger.info( 243 f"Building merge cache for {self.table_name}.\n\t" 244 + f"Found {len(merge_tables)} downstream merge tables" 245 ) 247 return merge_tables ValueError: Please import position_linearization_merge.linearized_position_output and try again. ```

any help is appreciated. thanks.

[edeno]

ValueError: Please import position_linearization_merge.linearized_position_output and try again.

[MichaelCoulter]

I tried that and got a new error

Error stack

```python --------------------------------------------------------------------------- DataJointError Traceback (most recent call last) Cell In [11], line 3 1 #from spyglass.linearization.merge import linearized_position_output ----> 3 (Nwbfile & {'nwb_file_name':'BS2820231107_.nwb'}).delete() File ~/spyglass/src/spyglass/utils/dj_mixin.py:587, in SpyglassMixin.delete(self, force_permission, *args, **kwargs) 585 def delete(self, force_permission=False, *args, **kwargs): 586 """Alias for cautious_delete, overwrites datajoint.table.Table.delete""" --> 587 self.cautious_delete(force_permission=force_permission, *args, **kwargs) File ~/spyglass/src/spyglass/utils/dj_mixin.py:577, in SpyglassMixin.cautious_delete(self, force_permission, *args, **kwargs) 574 self._log_delete(start) 575 return --> 577 super().delete(*args, **kwargs) # Additional confirm here 579 self._log_delete(start=start, merge_deletes=merge_deletes) File ~/anaconda3/envs/spyglass2/lib/python3.9/site-packages/datajoint/table.py:574, in Table.delete(self, transaction, safemode, force_parts) 572 if transaction: 573 self.connection.cancel_transaction() --> 574 raise DataJointError( 575 "Attempt to delete part table {part} before deleting from " 576 "its master {master} first.".format(part=part, master=master) 577 ) 579 # Confirm and commit 580 if delete_count == 0: DataJointError: Attempt to delete part table `common_usage`.`export_selection__file` before deleting from its master `common_usage`.`export_selection` first. ```

[edeno]

This just means you have to delete the master entry before deleting the part. We have some things in the works to make it easier. I am surprised that this table is relevant for you. Any thoughts on why that is @CBroz1 ?

[CBroz1]

• DJ 1158 is designed to address Attempt to delete part...
• This file was used to test export functionality, putting fk refs in export_selection__file - I've deleted these for you
• The delete_downstream_merge functionality was designed to only ID merge tables for which this would be an issue. It's worth considering the feature request of changing that to include all part tables with foreign key inheritance