Error in spatial_clustering with python 3.10

[birgits]

When I run the spatial_clustering with python 3.10 and load_shedding set to True, I get the following error:

KeyError Traceback (most recent call last)
File ~/virtualenvs/hyBit_p_310/lib/python3.10/site-packages/pandas/core/indexes/[base.py:3802](http://base.py:3802/), in Index.get_loc(self, key, method, tolerance)
3801 try:
-> 3802 return self._engine.get_loc(casted_key)
3803 except KeyError as err:

File ~/virtualenvs/hyBit_p_310/lib/python3.10/site-packages/pandas/_libs/index.pyx:138, in pandas._libs.index.IndexEngine.get_loc()

File ~/virtualenvs/hyBit_p_310/lib/python3.10/site-packages/pandas/_libs/index.pyx:165, in pandas._libs.index.IndexEngine.get_loc()

File pandas/_libs/hashtable_class_helper.pxi:5745, in pandas._libs.hashtable.PyObjectHashTable.get_item()

File pandas/_libs/hashtable_class_helper.pxi:5753, in pandas._libs.hashtable.PyObjectHashTable.get_item()

KeyError: '14381'

The above exception was the direct cause of the following exception:

KeyError Traceback (most recent call last)
Cell In[11], line 1
----> 1 etrago.spatial_clustering()

File ~/virtualenvs/hyBit_p_310/git_repos/eTraGo/etrago/cluster/[electrical.py:1058](http://electrical.py:1058/), in run_spatial_clustering(self)
1055 if self.args["network_clustering"]["active"]:
1056 [self.network](http://self.network/).generators.control = "PV"
-> 1058 elec_network, weight, n_clusters, busmap_foreign = preprocessing(self)
1060 if self.args["network_clustering"]["method"] == "kmeans":
1061 if self.args["network_clustering"]["k_elec_busmap"] == False:

File ~/virtualenvs/hyBit_p_310/git_repos/eTraGo/etrago/cluster/[electrical.py:796](http://electrical.py:796/), in preprocessing(etrago)
794 weight.index = weight.index.astype(str)
795 else:
--> 796 weight = weighting_for_scenario(network=network_elec, save=False)
798 return network_elec, weight, n_clusters, busmap_foreign

File ~/virtualenvs/hyBit_p_310/git_repos/eTraGo/etrago/cluster/[electrical.py:1005](http://electrical.py:1005/), in weighting_for_scenario(network, save)
982 fixed_capacity_fac = {
983 # A value of 1 is given to power plants where its availability
984 # does not depend on the weather
(...)
999 "nuclear": 1,
1000 }
1002 gen = network.generators[network.generators.carrier != "load shedding"][
1003 ["bus", "carrier", "p_nom"]
1004 ].copy()
-> 1005 gen["cf"] = gen.apply(calc_availability_factor, axis=1)
1006 gen["weight"] = gen["p_nom"] * gen["cf"]
1008 gen = (
1009 gen.groupby("bus")
1010 .weight.sum()
1011 .reindex(network.buses.index, fill_value=0.0)
1012 )

File ~/virtualenvs/hyBit_p_310/lib/python3.10/site-packages/pandas/core/[frame.py:9568](http://frame.py:9568/), in DataFrame.apply(self, func, axis, raw, result_type, args, **kwargs)
9557 from pandas.core.apply import frame_apply
9559 op = frame_apply(
9560 self,
9561 func=func,
(...)
9566 kwargs=kwargs,
9567 )
-> 9568 return op.apply().finalize(self, method="apply")

File ~/virtualenvs/hyBit_p_310/lib/python3.10/site-packages/pandas/core/[apply.py:764](http://apply.py:764/), in FrameApply.apply(self)
761 elif self.raw:
762 return self.apply_raw()
--> 764 return self.apply_standard()

File ~/virtualenvs/hyBit_p_310/lib/python3.10/site-packages/pandas/core/[apply.py:891](http://apply.py:891/), in FrameApply.apply_standard(self)
890 def apply_standard(self):
--> 891 results, res_index = self.apply_series_generator()
893 # wrap results
894 return self.wrap_results(results, res_index)

File ~/virtualenvs/hyBit_p_310/lib/python3.10/site-packages/pandas/core/[apply.py:907](http://apply.py:907/), in FrameApply.apply_series_generator(self)
904 with option_context("mode.chained_assignment", None):
905 for i, v in enumerate(series_gen):
906 # ignore SettingWithCopy here in case the user mutates
--> 907 results[i] = self.f(v)
908 if isinstance(results[i], ABCSeries):
909 # If we have a view on v, we need to make a copy because
910 # series_generator will swap out the underlying data
911 results[i] = results[i].copy(deep=False)

File ~/virtualenvs/hyBit_p_310/git_repos/eTraGo/etrago/cluster/[electrical.py:967](http://electrical.py:967/), in weighting_for_scenario.<locals>.calc_availability_factor(gen)
940 """
941 Calculate the availability factor for a given generator.
942
(...)
963
964 """
966 if gen["carrier"] in time_dependent:
--> 967 cf = network.generators_t["p_max_pu"].loc[:, [gen.name](http://gen.name/)].mean()
968 else:
969 try:

File ~/virtualenvs/hyBit_p_310/lib/python3.10/site-packages/pandas/core/[indexing.py:1067](http://indexing.py:1067/), in _LocationIndexer.getitem(self, key)
1065 if self._is_scalar_access(key):
1066 return self.obj._get_value(*key, takeable=self._takeable)
-> 1067 return self._getitem_tuple(key)
1068 else:
1069 # we by definition only have the 0th axis
1070 axis = self.axis or 0

File ~/virtualenvs/hyBit_p_310/lib/python3.10/site-packages/pandas/core/[indexing.py:1247](http://indexing.py:1247/), in _LocIndexer._getitem_tuple(self, tup)
1245 with suppress(IndexingError):
1246 tup = self._expand_ellipsis(tup)
-> 1247 return self._getitem_lowerdim(tup)
1249 # no multi-index, so validate all of the indexers
1250 tup = self._validate_tuple_indexer(tup)

File ~/virtualenvs/hyBit_p_310/lib/python3.10/site-packages/pandas/core/[indexing.py:967](http://indexing.py:967/), in _LocationIndexer._getitem_lowerdim(self, tup)
963 for i, key in enumerate(tup):
964 if is_label_like(key):
965 # We don't need to check for tuples here because those are
966 # caught by the _is_nested_tuple_indexer check above.
--> 967 section = self._getitem_axis(key, axis=i)
969 # We should never have a scalar section here, because
970 # _getitem_lowerdim is only called after a check for
971 # is_scalar_access, which that would be.
972 if section.ndim == self.ndim:
973 # we're in the middle of slicing through a MultiIndex
974 # revise the key wrt to section by inserting an _NS

File ~/virtualenvs/hyBit_p_310/lib/python3.10/site-packages/pandas/core/[indexing.py:1312](http://indexing.py:1312/), in _LocIndexer._getitem_axis(self, key, axis)
1310 # fall thru to straight lookup
1311 self._validate_key(key, axis)
-> 1312 return self._get_label(key, axis=axis)

File ~/virtualenvs/hyBit_p_310/lib/python3.10/site-packages/pandas/core/[indexing.py:1260](http://indexing.py:1260/), in _LocIndexer._get_label(self, label, axis)
1258 def _get_label(self, label, axis: int):
1259 # GH#5567 this will fail if the label is not present in the axis.
-> 1260 return self.obj.xs(label, axis=axis)

File ~/virtualenvs/hyBit_p_310/lib/python3.10/site-packages/pandas/core/[generic.py:4041](http://generic.py:4041/), in NDFrame.xs(self, key, axis, level, drop_level)
4039 if axis == 1:
4040 if drop_level:
-> 4041 return self[key]
4042 index = self.columns
4043 else:

File ~/virtualenvs/hyBit_p_310/lib/python3.10/site-packages/pandas/core/[frame.py:3807](http://frame.py:3807/), in DataFrame.getitem(self, key)
3805 if self.columns.nlevels > 1:
3806 return self._getitem_multilevel(key)
-> 3807 indexer = self.columns.get_loc(key)
3808 if is_integer(indexer):
3809 indexer = [indexer]

File ~/virtualenvs/hyBit_p_310/lib/python3.10/site-packages/pandas/core/indexes/[base.py:3804](http://base.py:3804/), in Index.get_loc(self, key, method, tolerance)
3802 return self._engine.get_loc(casted_key)
3803 except KeyError as err:
-> 3804 raise KeyError(key) from err
3805 except TypeError:
3806 # If we have a listlike key, _check_indexing_error will raise
3807 # InvalidIndexError. Otherwise we fall through and re-raise
3808 # the TypeError.
3809 self._check_indexing_error(key)

KeyError: '14381'

With load_shedding set to False, the spatial_clustering runs through.

[birgits]

I get another error in Etrago.plot_grid:

IndexError                                Traceback (most recent call last)
Cell In[22], line 1
----> 1 etrago.plot_grid(line_colors= "expansion_abs", bus_colors= "storage_expansion", bus_sizes= 0.000001)

File ~/virtualenvs/hyBit_p_310/git_repos/eTraGo/etrago/tools/[plot.py:2496](http://plot.py:2496/), in plot_grid(self, line_colors, bus_sizes, bus_colors, timesteps, osm, boundaries, filename, disaggregated, ext_min, ext_width)
   2492 v = np.linspace(boundaries[0], boundaries[1], 101)
   2494 # colorbar for line heatmap
   2495 cb = plt.colorbar(
-> 2496     ll[1], boundaries=v, ticks=v[0:101:10], fraction=0.046, pad=0.04
   2497 )
   2498 # Set legend label
   2499 cb.set_label(label)

IndexError: tuple index out of range

[ClaraBuettner]

I couldn't reproduce both of your issues, I can run and plot calculations with load shedding. Which pandas version do you use? I use 1.5.3.

[birgits]

I also use 1.5.3.. Weird.

[CarlosEpia]

I just ran eTraGo without any error message using the default settings using Python 3.8 and 3.10. Could you install the dev branch of eTraGo in a fresh virtual environment and try again?