noDatabaseForPredictions=True fails with KeyError: 'Cluster Labels'

[aretaon]

Error description

ComplexFinder does not run in no-database mode.

How to reproduce

Starting from the example files provided the following code works:

from ComplexFinder.src.main import ComplexFinder
X = "ComplexFinder/example-data/D0"
ComplexFinder(analysisName = "ExampleRun_01",
              runName = "ExampleRun_01_noDB",
              idColumn = "Uniprot ID",
              noDatabaseForPredictions=True,
              grouping={'WT': 'D0_aebersold.txt'}).run(X)

returns

-> 1672 df = df.sort_values(by="Cluster Labels") KeyError: 'Cluster Labels'

However, running the same code with noDatabaseForPredictions=False leads to no errors.

System properties

OS: Fedora 40 Python: 3.8 Dependencies: jupyter = ">=1.1.1,<2" asteval = "<=0.9.19" certifi = "<=2022.12.7" cycler = "<=0.10.0" cython = "<=0.29.21" future = "<=0.18.2" hdbscan = "<=0.8.29" joblib = "<=1.2.0" kiwisolver = "<=1.3.1" llvmlite = "<=0.34.0" lmfit = "<=1.0.1" matplotlib = "<=3.3.2" numba = "<=0.51.2" numpy = "<=1.22.0" pandas = "<=1.1.4" pillow = "<=9.3.0" pyparsing = "<=2.4.7" python-dateutil = "<=2.8.1" pytz = "<=2020.4" scikit-learn = "<=0.23.2" scipy = "<=1.5.4" seaborn = "<=0.11.0" six = "<=1.15.0" threadpoolctl = "<=2.1.0" umap-learn = ">=0.5.0" uncertainties = "<=3.1.4" imbalanced-learn = ">=0.7.0,<0.8"

Full traceback:

KeyError Traceback (most recent call last) /tmp/ipykernel_1245616/3156604555.py in ?() ----> 1 ComplexFinder(analysisName = "ExampleRun_01", 2 runName = "ExampleRun_01_noDB", 3 idColumn = "Uniprot ID", 4 noDatabaseForPredictions=True,

~/nvme_data/Projects/ComplexFinder_debug/ComplexFinder/src/main.py in ?(self, X, maxValueToOne) 2207 groupFileNames = [groupFileNames] 2208 self._clusterInteractions(combinedInteractions,groupFiles = groupFileNames,groupName = groupName) 2209 else: 2210 print("Info :: Cluster Interactions") -> 2211 self._clusterInteractions(None) 2212 2213 2214 self.params["runTimes"]["Interaction Clustering and Embedding"] = time.time() - endTrainingTime

~/nvme_data/Projects/ComplexFinder_debug/ComplexFinder/src/main.py in ?(self, predInts, clusterMethod, plotEmbedding, groupFiles, combineProbs, groupName) 1668 umapKwargs = self.params["umapDefaultKwargs"], 1669 generateSquareMatrix = True, 1670 ) 1671 df = pd.DataFrame().from_dict({"Entry":intLabels,"Cluster Labels({})".format(analysisName):clusterLabels,"reachability":reachability,"core_distances":core_distances}) -> 1672 df = df.sort_values(by="Cluster Labels") 1673 df = df.set_index("Entry") 1674 1675 if pooledDistances is not None:

~/nvme_data/Projects/ComplexFinder_debug/.pixi/envs/default/lib/python3.8/site-packages/pandas/core/frame.py in ?(self, by, axis, ascending, inplace, kind, na_position, ignore_index, key) 5294 else: 5295 from pandas.core.sorting import nargsort 5296 5297 by = by[0] -> 5298 k = self._get_label_or_level_values(by, axis=axis) 5299 5300 # need to rewrap column in Series to apply key function 5301 if key is not None:

~/nvme_data/Projects/ComplexFinder_debug/.pixi/envs/default/lib/python3.8/site-packages/pandas/core/generic.py in ?(self, key, axis) 1559 values = self.xs(key, axis=other_axes[0])._values 1560 elif self._is_level_reference(key, axis=axis): 1561 values = self.axes[axis].get_level_values(key)._values 1562 else: -> 1563 raise KeyError(key) 1564 1565 # Check for duplicates 1566 if values.ndim > 1:

KeyError: 'Cluster Labels'

[aretaon]

Apperently the issue is differences in column naming between the part where a dataframe is written and a few lines below where the same column is used for sorting, e.g.

-                    noNoiseIndex = df.index[df["Cluster Labels"] > 0]
+                    noNoiseIndex = df.index[df["Cluster Labels({})".format(analysisName)] > 0]

fixing this a few times (see complete diff below) also fixes the error. If this is the indended behaviour please feel free to patch with the diff.

---

diff --git a/src/main.py b/src/main.py
index e7be847..33cbf41 100644
--- a/src/main.py
+++ b/src/main.py
@@ -1670,7 +1670,7 @@ class ComplexFinder(object):
                                                                                                     generateSquareMatrix = True,
                                                                                                     )
                     df = pd.DataFrame().from_dict({"Entry":intLabels,"Cluster Labels({})".format(analysisName):clusterLabels,"reachability":reachability,"core_distances":core_distances})
-                    df = df.sort_values(by="Cluster Labels")
+                    df = df.sort_values(by="Cluster Labels({})".format(analysisName))
                     df = df.set_index("Entry")

                     if pooledDistances is not None:
@@ -1679,7 +1679,7 @@ class ComplexFinder(object):
                     squaredDf = pd.DataFrame(matrix,columns=intLabels,index=intLabels).loc[df.index,df.index]
                     squaredDf.to_csv(os.path.join(pathToFolder,"SquaredSorted_{}.txt".format(self.currentAnalysisName)),sep="\t")

-                    noNoiseIndex = df.index[df["Cluster Labels"] > 0]
+                    noNoiseIndex = df.index[df["Cluster Labels({})".format(analysisName)] > 0]

                     squaredDf.loc[noNoiseIndex,noNoiseIndex].to_csv(os.path.join(pathToFolder,"NoNoiseSquaredSorted_{}.txt".format(self.currentAnalysisName)),sep="\t")
                     splitLabels = True
@@ -1691,7 +1691,7 @@ class ComplexFinder(object):
                     dfEmbed["clusterLabels({})".format(analysisName)] = clusterLabels
                     dfEmbed["labels({})".format(analysisName)] = intLabels
                     if splitLabels:
-                        dfEmbed["sLabels"] = dfEmbed["labels"].str.split("_",expand=True).values[:,0]
+                        dfEmbed["sLabels"] = dfEmbed["labels({})".format(analysisName)].str.split("_",expand=True).values[:,0]
                         dfEmbed = dfEmbed.set_index("sLabels")
                     else:
                         dfEmbed = dfEmbed.set_index("labels({})".format(analysisName))

[hnolCol]

Thank you very much and please excuse the late reply. I will double-check tomorrow afternoon and then accept/edit. Thanks again! Cheers Hendrik