Getting totally different MSE from actual run and talos hyperparameter optimization

I'm getting totally different MSE from actual run and talos 1.0 hyperparameter optimization. I'm using tensorflow 2.1.0

Initially I used talos to get the best hyperparameters. From the csv, I got MSE ~ 1e-4 which is good, as shown:

loss mean_squared_error val_loss val_mean_squared_error neurons

0.000106636 0.000106636 0.001984425 0.001984425 400

activation batch_size dropout epochs hidden_layers kernel_initializer lr

LeakyReLU 50 0 400 16 he_uniform 0.01

optimizer

I then use this parameters in my orginal code w/o talos. But the ans I got was NAN. Why is this so? How do I debug?

Part of my code is as follows:

...
def talos_model(x_train, y_train, x_val, y_val, params):
    #Start defining the input tensor:
    input_data = layers.Input(shape=(airfoil_parameters_no))

    #create the layers and pass them the input tensor to get the output tensor:

    hiddenOut = Dense(params['neurons'],
                      kernel_initializer=params['kernel_initializer'])(input_data)

    if params['activation'] == 'LeakyReLU':
         hiddenOut = LeakyReLU(alpha = 0.3)(input_data)

    else:
         hiddenOut = Activation(params['activation'])(input_data)

    for i in range(params['hidden_layers']):
        print ("adding layer", i+1)

        hiddenOut = Dense(params['neurons'],
                          kernel_initializer=params['kernel_initializer'])(hiddenOut)

        if params['activation'] == 'LeakyReLU':
             hiddenOut = LeakyReLU(alpha = 0.3)(hiddenOut)
        else:
             hiddenOut = Activation(params['activation'])(hiddenOut)

    c_end_out = Dense(1, activation='linear', name='c_end')(hiddenOut)

    #define the model's start and end points
    model = Model(input_data,outputs = [c_end_out])

    # Compile model
    model.compile(loss='mean_squared_error',
                  #optimizer=params['optimizer'],
                  optimizer=params['optimizer'](lr=lr_normalizer(params['lr'],params['optimizer'])),
                  metrics=['mean_squared_error'])

    # Fit the model
    history = model.fit(input_var_train, output_var_train, \
                               validation_data=(input_var_test,output_var_test), \
                               batch_size=params['batch_size'],
                               epochs=params['epochs'])

    return history, model
...
# run the experiment
t = talos.Scan(x = input_var_train,
               y = output_var_train,
               x_val = input_var_test,
               y_val = output_var_test,
               model = talos_model,
               params=p,
               #fraction_limit=0.1,
               #round_limit=round_limit,
               time_limit=time_to_stop,
               experiment_name='c_end',
               #random_method=quantum,
               seed=seed_value,
               minimize_loss=True)

talos_details = t.details

analyze_object = talos.Analyze(t)

#talos_best_model = t.best_model(metric='mean_squared_error', asc=True)

talos_rounds = analyze_object.rounds()

# get the highest result for any metric
talos_high = analyze_object.high('mean_squared_error')

# get the round with the best result
talos_rounds2high = analyze_object.rounds2high('mean_squared_error')

# get the best paramaters
talos_best_params = analyze_object.best_params('mean_squared_error', ['acc', 'loss', 'val_loss'])
#talos_best_params = analyze_object.best_params('loss', ['loss'])

# get correlation for hyperparameters against a metric
talos_correlate = analyze_object.correlate('mean_squared_error', ['acc', 'loss', 'val_loss'])

Thanks.

autonomio / talos

Getting totally different MSE from actual run and talos hyperparameter optimization #497