broadinstitute / keras-rcnn

Keras package for region-based convolutional neural networks (RCNNs)
Other
555 stars 222 forks source link

Loss is constantly NaN #195

Open dberma15 opened 6 years ago

dberma15 commented 6 years ago

Hi,

I'm trying to train my dataset for the Data Science Bowl 2018 competition and I'm having trouble. The loss is always NaN no matter what I try. I know that my data set is structured properly, as it looks like this:

{'image': {'checksum': '2ce2a6891c485df5f5e1175b48cb305f', 'pathname': 'stage1_train/7f38885521586fc6011bef1314a9fb2aa1e4935bd581b2991e1d963395eab770/images/7f38885521586fc6011bef1314a9fb2aa1e4935bd581b2991e1d963395eab770.png', 'shape': {'r': 1024, 'c': 1024, 'channels': 3}}, 'objects': [{'bounding_box': {'minimum': {'r': 12, 'c': 14}, 'maximum': {'r': 14, 'c': 17}}, 'category': 'cell'}, {'bounding_box': {'minimum': {'r': 169, 'c': 109}, 'maximum': {'r': 170, 'c': 110}},  'category': 'cell'},....

So the problem isn't the data. Once I load it and try to run it, the model compiles, but the loss is consistently NaN and I can't figure out why. Can someone help?


import keras

import keras_rcnn.datasets.shape
import keras_rcnn.models
import keras_rcnn.preprocessing
import pickle
import numpy as np

def main():
    trainingdata = pickle.load(open('xtrdata.pkl','rb'))

    msk=np.random.random(len(trainingdata))<.9
    training_dictionary=[ trn for trn, m in zip(trainingdata,msk) if m]
    test_dictionary=[ trn for trn, m in zip(trainingdata,msk) if not m]
    # training_dictionary, test_dictionary = keras_rcnn.datasets.shape.load_data()

    categories = {"cell":1}

    generator = keras_rcnn.preprocessing.ObjectDetectionGenerator()

    generator = generator.flow_from_dictionary(
        dictionary=training_dictionary,
        categories=categories,
        target_size=(256, 256)
    )

    validation_data = keras_rcnn.preprocessing.ObjectDetectionGenerator()

    validation_data = validation_data.flow_from_dictionary(
        dictionary=test_dictionary,
        categories=categories,
        target_size=(256, 256)
    )

    keras.backend.set_learning_phase(1)

    model = keras_rcnn.models.RCNN(
        categories=["cell"],
        dense_units=512,
        input_shape=(256, 256, 3)
    )

    optimizer = keras.optimizers.Adam()
    model.compile(optimizer)
    model.save("test_rcnn.h5")
    model.fit_generator(
        epochs=100, steps_per_epoch=4,
        generator=generator,
        validation_data=validation_data)

if __name__ == '__main__':
    main()