Closed irivo closed 5 years ago
Hey, I had the same problem. I solved it by feeding all the arguments for the Cross-validator object.
Example:
automl = autosklearn.classification.AutoSklearnClassifier( time_left_for_this_task=120, per_run_time_limit=30, tmp_folder='/tmp/autosklearn_cv_example_tmp', output_folder='/tmp/autosklearn_cv_example_out', delete_tmp_folder_after_terminate=False, resampling_strategy=KFold, resampling_strategy_arguments={'n_splits': 5, 'shuffle': False, 'random_state': None}, )
As stated in the API Docs.
BaseCrossValidator or _RepeatedSplits or BaseShuffleSplit object: all arguments required by chosen class as specified in scikit-learn documentation. If arguments are not provided, scikit-learn defaults are used. If no defaults are available, an exception is raised. Refer to the ‘n_splits’ argument as ‘folds’.
Thank! Finally I did it, I read the documentation, but for some reason I misunderstood this moment.
Hi, I'm doing some tests using the BaseCrossValidation for LeaveOneOut, but I'm still having some errors. I already tried the suggested responses.
def main():
X, y = sklearn.datasets.load_boston(return_X_y=True)
feature_types = (['numerical'] * 3) + ['categorical'] + (['numerical'] * 9)
X_train, X_test, y_train, y_test = \
sklearn.model_selection.train_test_split(X, y, random_state=1)
automl = autosklearn.regression.AutoSklearnRegressor(
time_left_for_this_task=120,
per_run_time_limit=30,
tmp_folder='/tmp/autosklearn_kf_tmp',
output_folder='/tmp/autosklearn_kf_out',
delete_tmp_folder_after_terminate=False,
resampling_strategy=LeaveOneOut,
resampling_strategy_arguments={},
)
automl.fit(X_train.copy(), y_train.copy(), dataset_name='boston',
feat_type=feature_types)
automl.refit(X_train.copy(), y_train.copy())
print(automl.show_models())
predictions = automl.predict(X_test)
print("R2 score:", sklearn.metrics.r2_score(y_test, predictions))
if __name__ == '__main__':
main()
The error:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-2-c4b87957e7d1> in <module>
36
37 if __name__ == '__main__':
---> 38 main()
<ipython-input-2-c4b87957e7d1> in main()
23 # fit() changes the data in place, but refit needs the original data. We
24 # therefore copy the data. In practice, one should reload the data
---> 25 automl.fit(X_train.copy(), y_train.copy(), dataset_name='breast_cancer')
26 # During fit(), models are fit on individual cross-validation folds. To use
27 # all available data, we call refit() which trains all models in the
~/anaconda3/envs/autosk/lib/python3.7/site-packages/autosklearn/estimators.py in fit(self, X, y, X_test, y_test, metric, feat_type, dataset_name)
662 metric=metric,
663 feat_type=feat_type,
--> 664 dataset_name=dataset_name,
665 )
666
~/anaconda3/envs/autosk/lib/python3.7/site-packages/autosklearn/estimators.py in fit(self, **kwargs)
335 )
336 self._automl.append(automl)
--> 337 self._automl[0].fit(**kwargs)
338 else:
339 tmp_folder, output_folder = get_randomized_directory_names(
~/anaconda3/envs/autosk/lib/python3.7/site-packages/autosklearn/automl.py in fit(self, X, y, X_test, y_test, metric, feat_type, dataset_name, only_return_configuration_space, load_models)
994 dataset_name=dataset_name,
995 only_return_configuration_space=only_return_configuration_space,
--> 996 load_models=load_models,
997 )
998
~/anaconda3/envs/autosk/lib/python3.7/site-packages/autosklearn/automl.py in fit(self, X, y, task, metric, X_test, y_test, feat_type, dataset_name, only_return_configuration_space, load_models)
206 metric=metric,
207 load_models=load_models,
--> 208 only_return_configuration_space=only_return_configuration_space,
209 )
210
~/anaconda3/envs/autosk/lib/python3.7/site-packages/autosklearn/automl.py in _fit(self, datamanager, metric, load_models, only_return_configuration_space)
341 'cv', 'partial-cv',
342 'partial-cv-iterative-fit'] \
--> 343 and not issubclass(self._resampling_strategy, BaseCrossValidator)\
344 and not issubclass(self._resampling_strategy, _RepeatedSplits)\
345 and not issubclass(self._resampling_strategy, BaseShuffleSplit):
~/anaconda3/envs/autosk/lib/python3.7/abc.py in __subclasscheck__(cls, subclass)
141 def __subclasscheck__(cls, subclass):
142 """Override for issubclass(subclass, cls)."""
--> 143 return _abc_subclasscheck(cls, subclass)
144
145 def _dump_registry(cls, file=None):
TypeError: issubclass() arg 1 must be a class
Maybe someone can look into it a bit further and find the actual cause. You could try this and work from there. Gave me a terrible score, but compiled 🤷♂️ Best of luck!
import sklearn.model_selection
import sklearn.datasets
import sklearn.metrics
from sklearn.model_selection import LeaveOneOut as LOO
import autosklearn.regression
class LeaveOneOut(LOO):
def __init__(self, X):
self.X = X
def get_n_splits(self, X=None, y=None, groups=None):
return super().get_n_splits(self.X)
def main():
X, y = sklearn.datasets.load_boston(return_X_y=True)
feature_types = (['numerical'] * 3) + ['categorical'] + (['numerical'] * 9)
X_train, X_test, y_train, y_test = \
sklearn.model_selection.train_test_split(X, y, random_state=1)
automl = autosklearn.regression.AutoSklearnRegressor(
time_left_for_this_task=120,
per_run_time_limit=30,
tmp_folder='/tmp/autosklearn_regression_example_tmp',
output_folder='/tmp/autosklearn_regression_example_out',
resampling_strategy=LeaveOneOut,
resampling_strategy_arguments={'X': X_train}
)
automl.fit(X_train.copy(), y_train.copy(), dataset_name='boston',
feat_type=feature_types)
automl.refit(X_train.copy(), y_train.copy())
print(automl.show_models())
predictions = automl.predict(X_test)
print("R2 score:", sklearn.metrics.r2_score(y_test, predictions))
if __name__ == '__main__':
main()
@alchav06 your issue appears to be different from the one this thread is about. Could you please open a new issue so wen can properly keep track of this problem?
Hello. I'm trying to modify the cross validation example https://automl.github.io/auto-sklearn/master/examples/example_crossvalidation.html#sphx-glr-examples-example-crossvalidation-py , to use BaseCrossValidator object as a resampling_strategy argument, for example, LeaveOneOut, but I just can't figure out how to do it.
every time I get an error:
on the fourth try I get an error:
How to do it right?