search

autonomio / talos

Hyperparameter Experiments with TensorFlow and Keras
https://autonom.io
MIT License
1.62k stars 270 forks source link

KeyError: metrics in talos.scan #503

Closed william-yuan closed 3 years ago

william-yuan commented 3 years ago

talos.version = 1.0.0 tf.version = 2.2.0

The concise demo fails with 'KeyError: 'metrics.' The following code:

from keras.models import Sequential
from keras.layers import Dropout, Dense

import sys
import talos
x, y = talos.templates.datasets.breast_cancer()

x = talos.utils.rescale_meanzero(x)
def breast_cancer_model(x_train, y_train, x_val, y_val, params):

    model = Sequential()
    model.add(Dense(params['first_neuron'], input_dim=x_train.shape[1],
                    activation=params['activation'],
                    kernel_initializer=params['kernel_initializer']))

    model.add(Dropout(params['dropout']))

    model.add(Dense(1, activation=params['last_activation'],
                    kernel_initializer=params['kernel_initializer']))

    model.compile(loss=params['losses'],
                  optimizer=params['optimizer'],
                  metrics=['acc', talos.utils.metrics.f1score])

    history = model.fit(x_train, y_train, 
                        validation_data=[x_val, y_val],
                        batch_size=params['batch_size'],
                        callbacks=[talos.utils.live()],
                        epochs=params['epochs'],
                        verbose=0)

    return history, model

p = {'first_neuron':[9,10,11],
     'hidden_layers':[0, 1, 2],
     'batch_size': [30],
     'epochs': [100],
     'dropout': [0],
     'kernel_initializer': ['uniform','normal'],
     'optimizer': ['Nadam', 'Adam'],
     'losses': ['binary_crossentropy'],
     'activation':['relu', 'elu'],
     'last_activation': ['sigmoid']}
t = talos.Scan(x=x,
               y=y,
               model=breast_cancer_model,
               params=p,
               experiment_name='breast_cancer',
               round_limit=10)

...results in the following: 

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-1-7a8ddf94f1cb> in <module>
     47      'last_activation': ['sigmoid']}
     48 # and run the experiment
---> 49 t = talos.Scan(x=x,
     50                y=y,
     51                model=breast_cancer_model,

/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/talos/scan/Scan.py in __init__(self, x, y, params, model, experiment_name, x_val, y_val, val_split, random_method, seed, performance_target, fraction_limit, round_limit, time_limit, boolean_limit, reduction_method, reduction_interval, reduction_window, reduction_threshold, reduction_metric, minimize_loss, disable_progress_bar, print_params, clear_session, save_weights)
    194         # start runtime
    195         from .scan_run import scan_run
--> 196         scan_run(self)

/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/talos/scan/scan_run.py in scan_run(self)
     24         # otherwise proceed with next permutation
     25         from .scan_round import scan_round
---> 26         self = scan_round(self)
     27         self.pbar.update(1)
     28 

/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/talos/scan/scan_round.py in scan_round(self)
     17     # fit the model
     18     from ..model.ingest_model import ingest_model
---> 19     self.model_history, self.round_model = ingest_model(self)
     20     self.round_history.append(self.model_history.history)
     21 

/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/talos/model/ingest_model.py in ingest_model(self)
      4     through Scan() model paramater.'''
      5 
----> 6     return self.model(self.x_train,
      7                       self.y_train,
      8                       self.x_val,

<ipython-input-1-7a8ddf94f1cb> in breast_cancer_model(x_train, y_train, x_val, y_val, params)
     27                   metrics=['acc', talos.utils.metrics.f1score])
     28 
---> 29     history = model.fit(x_train, y_train, 
     30                         validation_data=[x_val, y_val],
     31                         batch_size=params['batch_size'],

/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py in _method_wrapper(self, *args, **kwargs)
     64   def _method_wrapper(self, *args, **kwargs):
     65     if not self._in_multi_worker_mode():  # pylint: disable=protected-access
---> 66       return method(self, *args, **kwargs)
     67 
     68     # Running inside `run_distribute_coordinator` already.

/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)
    828       self.stop_training = False
    829       train_function = self.make_train_function()
--> 830       callbacks.on_train_begin()
    831       # Handle fault-tolerance for multi-worker.
    832       # TODO(omalleyt): Fix the ordering issues that mean this has to

/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/tensorflow/python/keras/callbacks.py in on_train_begin(self, logs)
    445     logs = self._process_logs(logs)
    446     for callback in self.callbacks:
--> 447       callback.on_train_begin(logs)
    448 
    449   def on_train_end(self, logs=None):

/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/kerasplotlib/traininglog.py in on_train_begin(self, logs)
     31 
     32     def on_train_begin(self, logs={}):
---> 33         self.base_metrics = [metric for metric in self.params['metrics'] if not metric.startswith('val_')]
     34         if self.figsize is None:
     35             self.figsize = (

KeyError: 'metrics'
github-actions[bot] commented 3 years ago

Welcome to Talos community! Thanks so much for creating your first issue :)

qboy21 commented 3 years ago

same issue exactly.

mikkokotila commented 3 years ago

Use this instead: 

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dropout, Dense

%matplotlib inline

import sys
sys.path.insert(0, '/Users/mikko/Documents/GitHub/talos')
import talos

# then we load the dataset
x, y = talos.templates.datasets.breast_cancer()

# and normalize every feature to mean 0, std 1
x = talos.utils.rescale_meanzero(x)

# first we have to make sure to input data and params into the function
def breast_cancer_model(x_train, y_train, x_val, y_val, params):

    model = Sequential()
    model.add(Dense(params['first_neuron'], input_dim=x_train.shape[1],
                    activation=params['activation'],
                    kernel_initializer=params['kernel_initializer']))

    model.add(Dropout(params['dropout']))

    model.add(Dense(1, activation=params['last_activation'],
                    kernel_initializer=params['kernel_initializer']))

    model.compile(loss=params['losses'],
                  optimizer=params['optimizer'],
                  metrics=['acc', talos.utils.metrics.f1score])

    history = model.fit(x_train, y_train, 
                        validation_data=(x_val, y_val),
                        batch_size=params['batch_size'],
                        epochs=params['epochs'],
                        verbose=0)

    return history, model

  # then we can go ahead and set the parameter space
p = {'first_neuron':[9,10,11],
     'hidden_layers':[0, 1, 2],
     'batch_size': [30],
     'epochs': [100],
     'dropout': [0],
     'kernel_initializer': ['uniform','normal'],
     'optimizer': ['Nadam', 'Adam'],
     'losses': ['binary_crossentropy'],
     'activation':['relu', 'elu'],
     'last_activation': ['sigmoid']}

# and run the experiment
t = talos.Scan(x=x,
               y=y,
               model=breast_cancer_model,
               params=p,
               experiment_name='breast_cancer',
               round_limit=10)

Closing here. Feel free to open new issue if anything.

qboy21 commented 3 years ago

Sorry. The same issue persists. Here is my function, parameter space and experiment error. have you tested talos on the latest version of jupyter notebook?

first we have to make sure to input data and params into the function

def talos_model(x_train, y_train, x_val, y_val, params):

model = Sequential()
model.add(LSTM(params['first_neuron'], input_shape=(X_train_seq.shape[1], X_train_seq.shape[2]),
               kernel_initializer=params['kernel_initializer']))

model.add(Dropout(params['dropout']))

model.add(Dense(params['second_neuron'], activation=params['last_activation'],kernel_initializer=params['kernel_initializer']))

model.add(Dropout(params['dropout']))

model.add(Dense(params['third_neuron'], activation=params['last_activation'],kernel_initializer=params['kernel_initializer']))

model.add(Dense(1, activation=params['last_activation'],kernel_initializer=params['kernel_initializer']))

model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc', talos.utils.metrics.f1score])

history = model.fit(X_train_seq, y_train_seq, validation_data=[X_test_seq, y_test_seq], batch_size=params['batch_size'],
                    callbacks=[talos.utils.live()], epochs=params['epochs'], verbose=0, shuffle=False)

return history, model

then we can go ahead and set the parameter space

p = {'first_neuron':[20,30,50], 'second_neuron':[6,8,10], 'third_neuron':[2,4], 'hidden_layers':[2,3], 'batch_size': [31,45,60], 'epochs': [110,130,170], 'dropout': [0.1,0.3], 'kernel_initializer': ['normal'], 'activation':['relu', 'elu'], 'last_activation': ['sigmoid']}

and run the experiment

t = talos.Scan(x=X_train_seq,y=y_train_seq,model=talos_model, params=p, experiment_name='talos_test', round_limit=10)

ERROR:

KeyError Traceback (most recent call last)

in 1 # and run the experiment ----> 2 t = talos.Scan(x=X_train_seq,y=y_train_seq,model=talos_model, params=p, experiment_name='talos_test', round_limit=10) /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/talos/scan/Scan.py in __init__(self, x, y, params, model, experiment_name, x_val, y_val, val_split, random_method, seed, performance_target, fraction_limit, round_limit, time_limit, boolean_limit, reduction_method, reduction_interval, reduction_window, reduction_threshold, reduction_metric, minimize_loss, disable_progress_bar, print_params, clear_session, save_weights) 194 # start runtime 195 from .scan_run import scan_run --> 196 scan_run(self) /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/talos/scan/scan_run.py in scan_run(self) 24 # otherwise proceed with next permutation 25 from .scan_round import scan_round ---> 26 self = scan_round(self) 27 self.pbar.update(1) 28 /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/talos/scan/scan_round.py in scan_round(self) 17 # fit the model 18 from ..model.ingest_model import ingest_model ---> 19 self.model_history, self.round_model = ingest_model(self) 20 self.round_history.append(self.model_history.history) 21 /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/talos/model/ingest_model.py in ingest_model(self) 4 through Scan() model paramater.''' 5 ----> 6 return self.model(self.x_train, 7 self.y_train, 8 self.x_val, in talos_model(x_train, y_train, x_val, y_val, params) 18 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc', talos.utils.metrics.f1score]) 19 ---> 20 history = model.fit(X_train_seq, y_train_seq, validation_data=[X_test_seq, y_test_seq], batch_size=params['batch_size'], 21 callbacks=[talos.utils.live()], epochs=params['epochs'], verbose=0,shuffle=False) 22 ~/Library/Python/3.8/lib/python/site-packages/tensorflow/python/keras/engine/training.py in _method_wrapper(self, *args, **kwargs) 106 def _method_wrapper(self, *args, **kwargs): 107 if not self._in_multi_worker_mode(): # pylint: disable=protected-access --> 108 return method(self, *args, **kwargs) 109 110 # Running inside `run_distribute_coordinator` already. ~/Library/Python/3.8/lib/python/site-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing) 1077 train_function = self.make_train_function() 1078 self._train_counter.assign(0) -> 1079 callbacks.on_train_begin() 1080 training_logs = None 1081 # Handle fault-tolerance for multi-worker. ~/Library/Python/3.8/lib/python/site-packages/tensorflow/python/keras/callbacks.py in on_train_begin(self, logs) 499 if numpy_logs is None: # Only convert once. 500 numpy_logs = tf_utils.to_numpy_or_python_type(logs) --> 501 callback.on_train_begin(numpy_logs) 502 503 def on_train_end(self, logs=None): /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/kerasplotlib/traininglog.py in on_train_begin(self, logs) 31 32 def on_train_begin(self, logs={}): ---> 33 self.base_metrics = [metric for metric in self.params['metrics'] if not metric.startswith('val_')] 34 if self.figsize is None: 35 self.figsize = ( **KeyError: 'metrics'**
qboy21 commented 3 years ago

@mikkokotila Sorry. The same issue persists. Here is my function, parameter space and experiment error. have you tested talos on the latest version of jupyter notebook?

first we have to make sure to input data and params into the function

def talos_model(x_train, y_train, x_val, y_val, params):

model = Sequential()
model.add(LSTM(params['first_neuron'], input_shape=(X_train_seq.shape[1], X_train_seq.shape[2]),
               kernel_initializer=params['kernel_initializer']))

model.add(Dropout(params['dropout']))

model.add(Dense(params['second_neuron'], activation=params['last_activation'],kernel_initializer=params['kernel_initializer']))

model.add(Dropout(params['dropout']))

model.add(Dense(params['third_neuron'], activation=params['last_activation'],kernel_initializer=params['kernel_initializer']))

model.add(Dense(1, activation=params['last_activation'],kernel_initializer=params['kernel_initializer']))

model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc', talos.utils.metrics.f1score])

history = model.fit(X_train_seq, y_train_seq, validation_data=[X_test_seq, y_test_seq], batch_size=params['batch_size'],
                    callbacks=[talos.utils.live()], epochs=params['epochs'], verbose=0, shuffle=False)

return history, model

then we can go ahead and set the parameter space

p = {'first_neuron':[20,30,50], 'second_neuron':[6,8,10], 'third_neuron':[2,4], 'hidden_layers':[2,3], 'batch_size': [31,45,60], 'epochs': [110,130,170], 'dropout': [0.1,0.3], 'kernel_initializer': ['normal'],

'optimizer': ['Adam'],

'losses': ['binary_crossentropy'],

 'activation':['relu', 'elu'],
 'last_activation': ['sigmoid']}

and run the experiment

t = talos.Scan(x=X_train_seq,y=y_train_seq,model=talos_model, params=p, experiment_name='talos_test', round_limit=10)

ERROR:

KeyError Traceback (most recent call last)

in 1 # and run the experiment ----> 2 t = talos.Scan(x=X_train_seq,y=y_train_seq,model=talos_model, params=p, experiment_name='talos_test', round_limit=10) /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/talos/scan/Scan.py in __init__(self, x, y, params, model, experiment_name, x_val, y_val, val_split, random_method, seed, performance_target, fraction_limit, round_limit, time_limit, boolean_limit, reduction_method, reduction_interval, reduction_window, reduction_threshold, reduction_metric, minimize_loss, disable_progress_bar, print_params, clear_session, save_weights) 194 # start runtime 195 from .scan_run import scan_run --> 196 scan_run(self) /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/talos/scan/scan_run.py in scan_run(self) 24 # otherwise proceed with next permutation 25 from .scan_round import scan_round ---> 26 self = scan_round(self) 27 self.pbar.update(1) 28 /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/talos/scan/scan_round.py in scan_round(self) 17 # fit the model 18 from ..model.ingest_model import ingest_model ---> 19 self.model_history, self.round_model = ingest_model(self) 20 self.round_history.append(self.model_history.history) 21 /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/talos/model/ingest_model.py in ingest_model(self) 4 through Scan() model paramater.''' 5 ----> 6 return self.model(self.x_train, 7 self.y_train, 8 self.x_val, in talos_model(x_train, y_train, x_val, y_val, params) 18 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc', talos.utils.metrics.f1score]) 19 ---> 20 history = model.fit(X_train_seq, y_train_seq, validation_data=[X_test_seq, y_test_seq], batch_size=params['batch_size'], 21 callbacks=[talos.utils.live()], epochs=params['epochs'], verbose=0, shuffle=False) 22 ~/Library/Python/3.8/lib/python/site-packages/tensorflow/python/keras/engine/training.py in _method_wrapper(self, *args, **kwargs) 106 def _method_wrapper(self, *args, **kwargs): 107 if not self._in_multi_worker_mode(): # pylint: disable=protected-access --> 108 return method(self, *args, **kwargs) 109 110 # Running inside `run_distribute_coordinator` already. ~/Library/Python/3.8/lib/python/site-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing) 1077 train_function = self.make_train_function() 1078 self._train_counter.assign(0) -> 1079 callbacks.on_train_begin() 1080 training_logs = None 1081 # Handle fault-tolerance for multi-worker. ~/Library/Python/3.8/lib/python/site-packages/tensorflow/python/keras/callbacks.py in on_train_begin(self, logs) 499 if numpy_logs is None: # Only convert once. 500 numpy_logs = tf_utils.to_numpy_or_python_type(logs) --> 501 callback.on_train_begin(numpy_logs) 502 503 def on_train_end(self, logs=None): /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/kerasplotlib/traininglog.py in on_train_begin(self, logs) 31 32 def on_train_begin(self, logs={}): ---> 33 self.base_metrics = [metric for metric in self.params['metrics'] if not metric.startswith('val_')] 34 if self.figsize is None: 35 self.figsize = ( KeyError: 'metrics'
Bhisham-Sharma commented 3 years ago

I'm having this same issue with KeyError: metrics. Does anyone have solved this problem yet?

rpa1tera commented 2 years ago

I'm having this same issue with KeyError: metrics. Does anyone have solved this problem yet?

did you solve it?