CLI example

[mgvel]

#gnanavel.mutharasu@gmail.com
import os
import random
import argparse
import datetime
import numpy as np
import pandas as pd

from skimage import io
from PIL import ImageFile
from logzero import logger

import tensorflow as tf
from keras import Model
from keras import utils
from keras import backend as K
from keras.utils import multi_gpu_model
from keras.utils import Sequence
from keras.layers import Dense

from keras.optimizers import Adam
from keras.applications.inception_v3 import InceptionV3
from keras.callbacks import ModelCheckpoint
from sklearn.model_selection import KFold

# fix for loading truncated images
ImageFile.LOAD_TRUNCATED_IMAGES = True

# Fix for saving trained model into disk
os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE'
logger.info("Python module imports completed.")

def flip_axis(x, axis):
    """Flip function is no more in Keras"""
    x = np.asarray(x).swapaxes(axis, 0)
    x = x[::-1, ...]
    x = x.swapaxes(0, axis)
    return x

class TrainSequence(Sequence):
    """A sequence "generator" for training. This multiprocessing-based
    method is faster than the multithreading-based generators."""
    def __init__(self,
                 paths,
                 labels,
                 batch_size,
                 num_classes):
        self.paths = paths
        self.labels = labels
        self.batch_size = batch_size
        self.num_classes = num_classes

    # Keras expects this method to return # of batches your dataset contains
    def __len__(self):
        return int(np.floor(len(self.paths) / float(self.batch_size)))

    # Keras expects this method to return the batch according to index 'idx'.
    def __getitem__(self, idx):
        # Get the full paths and class labels of the examples in batch 'idx'.
        batch_paths = self.paths[idx * self.batch_size:(idx + 1) *
                                 self.batch_size]
        batch_labels = self.labels[idx * self.batch_size:(idx + 1) *
                                   self.batch_size]
        # Collect the image data and labels for the batch.
        x_batch = []
        y_batch = []
        for i in range(len(batch_paths)):
            img = io.imread(batch_paths[i])
            if np.random.random() < 0.5:
                img = flip_axis(img, 0)
            rand_4 = np.random.choice(4, 1)
            np.rot90(img, k=rand_4)
            x_batch.append(img)
            y_batch.append(batch_labels[i])

        # Convert pixel values to float32 and scale to interval 0-1.
        x_batch = np.array(x_batch, np.float32) / 255
        y_batch = utils.to_categorical(y_batch, self.num_classes)
        return x_batch, y_batch

def poly_decay(epoch):
    # initialize the maximum number of epochs, base learning rate,
    # and power of the polynomial
    maxEpochs = epochs,
    power = 1.0

    # compute the new learning rate based on polynomial decay
    alpha = baseLR * (1 - (epoch / float(maxEpochs))) ** power

    # return the new learning rate
    return alpha

# Build network
def Inception_V3(gpus, weights, input_shape, num_classes):
    if gpus > 1:
        hostdevice = '/cpu:0'
    else:
        hostdevice = '/device:gpu:0'
    with tf.device(hostdevice):
        model = InceptionV3(weights=weights,
                            include_top=False,
                            input_shape=input_shape,
                            pooling='avg'
                            )
        x = Dense(num_classes, activation='sigmoid')(model.output)
        model = Model(inputs=model.input, outputs=x)
    return model

def replace_path_with_slurm_id(tile, job_id):
    job_id_str = '/tmp/mutharas_kfold'
    tile = tile.replace('/home/mutharas/work/myoma', job_id_str)
    return tile

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Training leiomyoma tiles \
                                     with InceptionV3 model")
    parser.add_argument('-t', '--tiles', dest='tiles_file', required=True,
                        type=argparse.FileType('r'),
                        help='file containing list of tiles path')
    parser.add_argument('-iw', '--img_width', type=int,
                        help='Input image width')
    parser.add_argument('-ih', '--img_height', type=int,
                        help='Input image height')
    parser.add_argument('-e', '--epochs', type=int,
                        help='Number of Epochs to run the training')
    parser.add_argument('-p', '--percent', type=int,
                        help='Percentage of Epochs to run in training')
    parser.add_argument('-b', '--batch', type=int,
                        help='Batch size for generator')
    parser.add_argument('-g', '--gpus', type=int,
                        help='Number of GPUs to use')
    parser.add_argument('-w', '--weights',
                        help='Weights to use with InceptionV3', default=None)
    parser.add_argument('-r', '--learning_rate', type=float,
                        help='learning rate', default=0.001)
    parser.add_argument('-x', '--foldx', type=int,
                        help='partition of samples to train', default=10)
    parser.add_argument('-j', '--jobid', help='JOBID from slurm')

    args = parser.parse_args()

    # SLURM JOB ID for file naming
    JOBID = args.jobid

    tiles_df = pd.read_csv(args.tiles_file, sep='\t')
    logger.info(tiles_df.head())
    tiles = tiles_df['tile']
    tiles = [tile.strip() for tile in tiles]
    tiles = [replace_path_with_slurm_id(tile, args.jobid) for tile in  tiles]

    labels = tiles_df['label']
    labels = [label.strip() for label in labels]
    labels = [np.uint8(1) if label == "mutant"
              else np.uint8(0) for label in labels]

    # select 90% of tiles for training
    nb_train_samples = int(np.ceil(len(tiles) * 0.9))
    train_tiles = tiles[0:nb_train_samples+1]
    train_labels = labels[0:nb_train_samples+1]

    # select 10% of tiles for validation
    nb_validation_samples = int(np.ceil(len(tiles) * 0.1))
    valid_tiles = tiles[nb_train_samples+1:]
    valid_labels = labels[nb_train_samples+1:]

    # Shuffle tiles and labels
    train_data = list(zip(train_tiles, train_labels))
    random.shuffle(train_data)
    train_tiles, train_labels = zip(*train_data)
    del train_data

    valid_data = list(zip(valid_tiles, valid_labels))
    random.shuffle(valid_data)
    valid_tiles, valid_labels = zip(*valid_data)
    del valid_data

    img_width = args.img_width
    img_height = args.img_height

    # training parameters
    epochs = args.epochs
    batch_size = args.batch

    fold_num = args.foldx

    # number of GPUs to use
    GPUS = args.gpus
    logger.info("Training with %d GPUs...", GPUS)

    total_batch_size = batch_size * GPUS

    dtype = 'float32'
    K.set_floatx(dtype)
    # K.set_epsilon(1e-07)
    logger.debug("Float32 setup completed")

    # Image Format configuration
    if K.image_data_format() == 'channels_first':
        input_shape = (3, img_width, img_height)
        logger.info("using alpha channel")
    else:
        input_shape = (img_width, img_height, 3)
        logger.warning("NOT using alpha channel")

    # Generators
    train_generator = TrainSequence(train_tiles, train_labels,
                                    total_batch_size, 2)
    valid_generator = TrainSequence(valid_tiles, valid_labels,
                                    total_batch_size, 2)

    model = Inception_V3(gpus=GPUS, weights=args.weights,
                         input_shape=input_shape, num_classes=2)

    logger.debug("compiling model in %d GPUS...", GPUS)
    if GPUS > 1:
        # make the model parallel
        multimodel = multi_gpu_model(model, gpus=GPUS)
        # Compile model
        multimodel.compile(loss='binary_crossentropy',
                           optimizer=Adam(lr=args.learning_rate),
                           metrics=['accuracy'])
    else:
        model.compile(loss='binary_crossentropy',
                      optimizer=Adam(lr=args.learning_rate),
                      metrics=['accuracy'])
        multimodel = model

    # checkpoint
    filepath = "results/K-fold-InceptionV3-gpus_{GPUS:02d} \
                                   -epochs_{epochs:02d} \
                                   -k-fold_'{fold_num:02d} \
                                   -{val_acc:.2f}.h5 \
                                   -job_{JOBID:04d}"

    checkpoint = ModelCheckpoint(filepath, monitor='val_acc',
                                 verbose=1,
                                 save_best_only=False,
                                 mode='max')

    # Generators
    logger.debug("training the network...")
    multimodel.fit_generator(
        generator=train_generator,
        validation_data=valid_generator,
        epochs=epochs,
        # using only 5% of an epoch
        steps_per_epoch=(args.percent / 100) * nb_train_samples //
        total_batch_size,
        # TODO on epoch end ??
        validation_steps=nb_validation_samples // total_batch_size,
        # adding class weight
        class_weight={0: 0.75, 1: 0.25},
        verbose=1,
        use_multiprocessing=False,
        workers=1,
        max_queue_size=total_batch_size * 3,
        )

    # save trained model to disk
    logger.info("saving trained model to disk...")
    stamp = str(datetime.datetime.now()).split('.')[0].replace(' ', '_')
    output = 'results/K-fold-InceptionV3_' + str(GPUS) + \
             'gpus_' + str(epochs) + \
             '_epochs_' + str(JOBID) + \
             '-k-fold_' + str(fold_num) + \
             '_at_' + stamp + \
             '_598px.h5'
    logger.info("saving\t %s", output)
    model.save(output)

[amruthaa08]

What is the purpose of if __name__ == "__main__": before creating the parser?

[mgvel]

main function is the place where a python scripts starts it's execution.

There is no need to specify main as default like in c/c++. But by manually invoking this main function, we can control the order of things in some cases. Here we are forcing the programs to take command lines arguments before proceeding with any other action in the script.

[amruthaa08]

Understood sir, thank you