RuntimeError: Model-building function did not return a valid Keras Model instance

[dalalkrish]

I'm getting following error and I'm not able to figure out why:

RuntimeError: Model-building function did not return a valid Keras Model instance, found (<tensorflow.python.keras.engine.functional.Functional object at 0x7f74d8b849d0>, <tensorflow.python.keras.engine.functional.Functional object at 0x7f74d8b80810>)

I have read the answers here and here which seem to telling to import keras from tensorflow instead of stand alone keras which I'm doing but still getting the error. I would very much appreciate your help in figuring this out. Below is my entire code:

    from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation
    from tensorflow.keras.models import Model, Sequential
    from tensorflow.keras.losses import BinaryCrossentropy
    from tensorflow.keras.optimizers import Adam
    from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau
    from numba import njit
    import tensorflow as tf
    import numpy as np
    from sklearn.model_selection import KFold
    from sklearn.model_selection._split import _BaseKFold, indexable, _num_samples
    from sklearn.utils.validation import _deprecate_positional_args
    import pandas as pd
    import kerastuner as kt
    import gc
    from tqdm import tqdm
    from random import choices
    import warnings
    warnings.filterwarnings('ignore')

    class MyTuner(kt.Tuner):
        def run_trial(self, trial, x, y):
            cv = PurgedGroupTimeSeriesSplit(n_splits=5, group_gap = 20)
            val_losses = []

            for train_indices, test_indices in cv.split(x, groups=x[0]):
                x_train, y_train = x[train_indices, 1:], y[train_indices]
                x_test, y_test = x[test_indices, 1:], y[test_indices]

                x_train = apply_transformation(x_train)
                x_test = apply_transformation(x_test)

                model = self.hypermodel.build(trial.hyperparameters)
                model.fit(x_train, y_train, batch_size = hp.Int('batch_size', 500, 5000, step=500, default=4000),
                          epochs = hp.Int('epochs', 100, 1000, step=200, default=500))

                val_losses.append(model.evaluate(x_test, y_test))

            self.oracle.update_trial(trial.trial_id, {'val_loss': np.mean(val_losses)})
            self.save_model(trial.trial_id, model)

    def create_autoencoder(hp, input_dim, output_dim):
        i = Input(input_dim)
        encoded = BatchNormalization()(i)
        encoded = GaussianNoise(hp.Float('gaussian_noise', 1e-2, 1, sampling='log', default=5e-2))(encoded)
        encoded = Dense(hp.Int('encoder_dense', 100, 300, step=50, default=64), activation='relu')(encoded)
        decoded = Dropout(hp.Float('decoder_dropout_1', 1e-1, 1, sampling='log', default=0.2))(encoded)
        decoded = Dense(input_dim,name='decoded')(decoded)
        x = Dense(hp.Int('output_x', 32, 100, step=10, default=32),activation='relu')(decoded)
        x = BatchNormalization()(x)
        x = Dropout(hp.Float('x_dropout_1', 1e-1, 1, sampling='log', default=0.2))(x)
        x = Dense(hp.Int('output_x', 32, 100, step=10, default=32),activation='relu')(x)
        x = BatchNormalization()(x)
        x = Dropout(hp.Float('x_dropout_2', 1e-1, 1, sampling='log', default=0.2))(x)    
        x = Dense(output_dim,activation='sigmoid',name='label_output')(x)

        encoder = Model(inputs=i,outputs=encoded)
        autoencoder = Model(inputs=i,outputs=[decoded, x])

    #     optimizer = hp.Choice('optimizer', ['adam', 'sgd'])

        autoencoder.compile(optimizer=Adam(hp.Float('lr', 0.00001, 0.1, default=0.001)), 
                            loss='sparse_binary_crossentropy',
                            metrics=['accuracy'])

        return autoencoder, encoder

    build_model = lambda hp: create_autoencoder(hp, X[:, 1:].shape[1], y.shape[1])

    tuner = MyTuner(
                oracle=kt.oracles.BayesianOptimization(
                        objective=kt.Objective('val_loss', 'min'),
                        max_trials=20),
                hypermodel=build_model,
                directory='./',
                project_name='autoencoders')

    tuner.search(X, (X,y), callbacks=[EarlyStopping('val_loss',patience=5),
                                      ReduceLROnPlateau('val_loss',patience=3)])

    encoder_hp  = tuner.get_best_hyperparameters(1)[0]
    print("Best Encoder Hyper-parameter:", encoder_hp)

    best_autoencoder = tuner.get_best_models(1)[0]

[haifeng-jin]

@dalalkrish I know this use case. You can have a submodel implemented of the autoencoder, with the encode and decode function. In this way, you don't have to return two models, but only one autoencoder model. You can call the model's member function to do the encoding and decoding.

class Autoencoder(tf.keras.Model):
  ...

  def call(self, inputs):
    return self.decode(self.encode(inputs))

  def encode(self, input):
    ...
    return output

  def decode(self, input):
    ...
    return output

[loukasilias]

class MyTuner(kt.Tuner):

  def run_trial(self, trial,*args,**kwargs):
    # You can add additional HyperParameters for preprocessing and custom training loops
    # via overriding `run_trial`
    hp = trial.hyperparameters
    model = self.hypermodel.build(hp)
    batch_size = hp.Int('batch_size', 4, 5, step=1)

    super(MyTuner, self).run_trial(trial, *args, **kwargs)

def build_model(hp):

    model = MyModel(flag = False, 
                    units_1 = hp.Int('units_1', 256, 512, default=128), 
                    units_2 = hp.Int('units_2', 1, 3, default=3),
                    rate_1 = hp.Float('rate_1', 0, 1, sampling = 'linear', default=0.4), 
                    rate_2 = hp.Float('rate_2', 0, 1, sampling = 'linear', default=0.3),
                    k = hp.Int('dimension', 60, 100, default=80)
                   )

    model.compile(optimizer=tf.keras.optimizers.SGD(),loss="binary_crossentropy",metrics=["acc"])

    return model

where MyModel has been implemented by subclassing keras models (tf.keras.Model)

accuracy = []
precision = []
recall = []
auroc = []
for fold, (train_index, test_index) in enumerate(skf.split(X, y)):
    X_train, X_test = X[train_index], X[test_index]
    y_train, y_test = y[train_index], y[test_index]
    X_train, X_eval, y_train, y_eval = train_test_split(X_train, y_train, test_size=0.2, random_state=42)
    print("Currently on fold: {}".format(fold))

    train_data = BertGenerator(X_train,y_train,batch_size= 6)
    eval_data = BertGenerator(X_eval,y_eval,batch_size = 6)
    tuner = MyTuner(oracle=kt.oracles.BayesianOptimization(objective=kt.Objective('val_loss', 'min'),max_trials=20),
                    hypermodel=build_model,
                    directory='./',
                    project_name='autoencoders')

    print(tuner.search_space_summary())

    tuner.search(train_data, epochs = 1000, validation_data = eval_data, class_weight = class_weight_function(y_train),
                 callbacks=[tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=4)])

    print(tuner.results_summary())

This code runs for 1 epoch and then the following error raises: AttributeError: 'NoneType' object has no attribute 'replace'. I am not able to understand the reason for this error. Thank you in advance