1e0b608d-abc3-462f-82fe-b9f5cae107e2
commented
4 years ago Note: problem occurs only after performing the RandomizedSearchCV...
When applying a function in a multiprocess using concurrent.futures if the function includes anything else other than print(), it is not executed and the process freezes.
Here is the code to reproduce.
from xgboost import XGBRegressor
from sklearn.model_selection import KFold
import concurrent.futures
from sklearn.datasets import make_regression
import pandas as pd
import numpy as np
from sklearn.model_selection import RandomizedSearchCV# STEP 1 # ---------------------------------------------------------------------------- # simulate RandomizedSearchCV
data = make_regression(n_samples=500, n_features=100, n_informative=10, n_targets=1, random_state=5)
X = pd.DataFrame(data[0])
y = pd.Series(data[1])
kf = KFold(n_splits = 3, shuffle = True, random_state = 5)
model = XGBRegressor(n_jobs = -1)
params = {
'min_child_weight': [0.1, 1, 5],
'subsample': [0.5, 0.7, 1.0],
'colsample_bytree': [0.5, 0.7, 1.0],
'eta': [0.005, 0.01, 0.1],
'n_jobs': [-1]
}
random_search = RandomizedSearchCV(
model,
param_distributions = params,
n_iter = 50,
n_jobs = -1,
refit = True, # necessary for random_search.best_estimator_
cv = kf.split(X,y),
verbose = 1,
random_state = 5
)
random_search.fit(X, np.array(y))# STEP 2.0 # ---------------------------------------------------------------------------- # test if multiprocessing is working in the first place
def just_print():
print('Just printing')
with concurrent.futures.ProcessPoolExecutor() as executor:
results_temp = [executor.submit(just_print) for i in range(0,12)]
# # STEP 2.1 # ---------------------------------------------------------------------------- # test on a slightly more complex function
def fit_model():
# JUST CREATING A DATASET, NOT EVEN FITTING ANY MODEL!!! AND IT FREEZES
data = make_regression(n_samples=500, n_features=100, n_informative=10, n_targets=1, random_state=5)
# model = XGBRegressor(n_jobs = -1)
# model.fit(data[0],data[1])
print('Fit complete')
with concurrent.futures.ProcessPoolExecutor() as executor:
results_temp = [executor.submit(fit_model) for i in range(0,12)]
# Attached this code in a .py file.
Fidget-Spinner
ronaldoussoren
Note: these values reflect the state of the issue at the time it was migrated and might not reflect the current state.
Show more details
GitHub fields: ```python assignee = None closed_at = None created_at =
labels = ['OS-mac', '3.7']
title = 'concurrent.futures.ProcessPoolExecutor freezes depending on complexity'
updated_at =
user = 'https://github.com/DanilZherebtsov'
```
bugs.python.org fields:
```python
activity =
actor = 'ronaldoussoren'
assignee = 'none'
closed = False
closed_date = None
closer = None
components = ['macOS']
creation =
creator = 'DanilZ'
dependencies = []
files = ['49562']
hgrepos = []
issue_num = 42245
keywords = []
message_count = 13.0
messages = ['380220', '380225', '380228', '380229', '380233', '380236', '380240', '380764', '380766', '380768', '380793', '381414', '381438']
nosy_count = 6.0
nosy_names = ['bquinlan', 'ronaldoussoren', 'pitrou', 'ned.deily', 'kj', 'DanilZ']
pr_nums = []
priority = 'normal'
resolution = None
stage = None
status = 'open'
superseder = None
type = None
url = 'https://bugs.python.org/issue42245'
versions = ['Python 3.7']
```