Custom Parameters

[dangalea]

I am including a parameter named "first_layer_winfdow" which controls the window size of the first layer of my CNN, but I am getting "IndexError: only integers, slices (:), ellipsis (...), numpy.newaxis (None) and integer or boolean arrays are valid indices". This is because the params dictionary is being turned into a numpy array by ta.Scan(). Are custom parameters allowed, and if not, what is the list of the allowed parameters?

[mikkokotila]

Please share your parameter dictionary for reference. And do make sure to check out the corresponding section regarding accepted input formats in the docs.

[dangalea]

My parameter dictionary is as follows:

p = {
    "data_aug" : [True, False],
    "conv_layers" : [1, 2, 3],
    "batch_norm_pos" : ["none", "before", "after"],
    "first_layer_filters" : (1, 100, 1),
    "second_layer_filters" : (1, 100, 1),
    "third_layer_filters" : (1, 100, 1),
    "first_layer_window" : (1, 5, 1),
    "second_layer_window" : (1, 5, 1),
    "third_layer_window" : (1, 5, 1),
    "first_conv_layer_reg_norm" : [True, False],
    "second_conv_layer_reg_norm" : [True, False],
    "third_conv_layer_reg_norm" : [True, False],
    "first_conv_layer_reg_value" : (0, 0.1, 0.0001),
    "second_conv_layer_reg_value" : (0, 0.1, 0.0001),
    "third_conv_layer_reg_value" : (0, 0.1, 0.0001),
    "first_dense_layer_reg_norm" : [True, False],
    "second_dense_layer_reg_norm" : [True, False],
    "third_dense_layer_reg_norm" : [True, False],
    "first_dense_layer_reg_value" : (0, 1, 0.001),
    "second_dense_layer_reg_value" : (0, 1, 0.001),
    "third_dense_layer_reg_value" : (0, 1, 0.001),
    "dense_layers" : [1, 2, 3],
    "first_layer_neurons" : (1, 100, 1),
    "second_layer_neurons" : (1, 100, 1),
    "third_layer_neurons" : (1, 100, 1),
    "dropout" : [True, False],
    "dropout_rate" : (0, 1, 0.01),
    "activation" : ["relu", "elu"],
    "optimizer" : ["rmsprop", "adam", "nadam", "sgd"], 
    "learning_rate" : (0, 5, 0.01), 
    "decay" : (0, 5, 0.1) , 
    "batch_size" : [32, 64, 100, 200, 500]
}

while my model function is:

def` dl_model(x, y, params=p, data_val=data_val, labels_val=labels_val, aug_data=aug_data, aug_labels=aug_labels):

    if params['data_aug'] == True:
        training_data = aug_data
        training_labels = aug_labels
    else:
        training_data = data_train
        training_labels = labels_train

    model = models.Sequential()

    if params["second_conv_layer_reg_norm"] == True:
        model.add(layers.Conv2D(params["first_layer_filters"], (params["first_layer_window"], params["first_layer_window"]), input_shape=training_data.shape[1:], kernel_regularizer=regularizers.l2(params["first_conv_layer_reg_value"])))
    else:
        model.add(layers.Conv2D(params["first_layer_filters"], (params["first_layer_window"], params["first_layer_window"]), input_shape=training_data.shape[1:]))
    model.add(layers.Activation(params["activation"]))
    model.add(layers.MaxPooling2D(2, 2))

    if params["conv_layers"] > 2:
        if params["batch_norm_pos" == "before"]:
            model.add(layers.BatchNormalization())
        if params["second_conv_layer_reg_norm"] == True:
            model.add(layers.Conv2D(params["second_layer_filters"], (params["second_layer_window"], params["second_layer_window"]), kernel_regularizer=regularizers.l2(params["second_conv_layer_reg_value"])))
        else:
            model.add(layers.Conv2D(params["second_layer_filters"], (params["second_layer_window"], params["second_layer_window"])))

        if params["batch_norm_pos" == "after"]:
            model.add(layers.BatchNormalization())
        model.add(layers.Activation(params["activation"]))
        model.add(layers.MaxPooling2D(2, 2))

    if params["conv_layers"] == 3:
        if params["batch_norm_pos" == "before"]:
            model.add(layers.BatchNormalization())
        if params["third_conv_layer_reg_norm"] == True:
            model.add(layers.Conv2D(params["third_layer_filters"], (params["third_layer_window"], params["third_layer_window"]), kernel_regularizer=regularizers.l2(params["third_conv_layer_reg_value"])))
        else:
            model.add(layers.Conv2D(params["third_layer_filters"], (params["third_layer_window"], params["third_layer_window"])))
        if params["batch_norm_pos" == "after"]:
            model.add(layers.BatchNormalization())
        model.add(layers.Activation(params["activation"]))
        model.add(layers.MaxPooling2D(2, 2))

    model.add(layers.Flatten())

    if params["batch_norm_pos" == "before"]:
        model.add(layers.BatchNormalization())
    if params["first_dense_layer_reg_norm"] == True:
        model.add(layers.Dense(params["first_layer_neurons"], kernel_regularizer=regularizers.l2(params["first_dense_layer_reg_value"])))
    else:
        model.add(layers.Dense(params["first_layer_neurons"]))
    if params["batch_norm_pos" == "after"]:
        model.add(layers.BatchNormalization())
    model.add(layers.Activation(params["activation"]))

    if params["dropout"] == True:
        model.add(layers.Dropout(params["dropout_rate"]))

    if params["dense_layers"] > 2:
        if params["batch_norm_pos" == "before"]:
            model.add(layers.BatchNormalization())
        if params["second_dense_layer_reg_norm"] == True:
            model.add(layers.Dense(params["second_layer_neurons"], kernel_regularizer=regularizers.l2(params["second_dense_layer_reg_value"])))
        else:
            model.add(layers.Dense(params["second_layer_neurons"]))
        if params["batch_norm_pos" == "after"]:
                model.add(layers.BatchNormalization())
        model.add(layers.Activation(params["activation"]))

    if params["dense_layers"] == 3:
        if params["batch_norm_pos" == "before"]:
            model.add(layers.BatchNormalization())
        if params["third_dense_layer_reg_norm"] == True:
            model.add(layers.Dense(params["third_layer_neurons"], kernel_regularizer=regularizers.l2(params["first_dense_layer_reg_value"])))
        else:
            model.add(layers.Dense(params["third_layer_neurons"]))
        if params["batch_norm_pos" == "after"]:
                model.add(layers.BatchNormalization())
        model.add(layers.Activation(params["activation"]))

    model.add(layers.Dense(1, activation='sigmoid'))

    if params["activation"] == "rmsprop":
        opt = keras.optimizers.RMSprop(lr=params["learning_rate"], decay=params["decay"])
    elif params["activation"] == "adam":
        opt = keras.optimizers.Adam(lr=params["learning_rate"], decay=params["decay"])
    elif params["activation"] == "nadam":
        opt = keras.optimizers.Nadam(lr=params["learning_rate"], decay=params["decay"])
    elif param["activation"] == "sgd":
        opt = keras.optimizers.SGD(lr=params["learning_rate"], decay=params["decay"])

    model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc'])

    #callback to save the best 
    early_stop = callbacks.EarlyStopping(monitor="val_loss", patience=30, restore_best_weights=True)
    history = model.fit(x=data_train, y=labels_train, validation_data=(data_val, labels_val), shuffle=True, epochs=150, verbose=0, batch_size=params["batch_size"], callbacks=[early_stop])

    return history, model

and I am using ta.Scan as follows:

t = ta.Scan(data_train, labels_train,
            params=p,
            model=dl_model,
            grid_downsample=0.9)

[dangalea]

I have managed to fix my issue. It seems that the model function's first four variables need to be <training data, training_labels, val_data, val_labels> in that order. It would be good if this was explicitly described in the documentation.