3 dimensional input for ConvNN (ND4J.vstack)

[AgremE]

/ConvNN Class File/

    public class CovNNTesting {
    public CovNNTesting(){}
    private static final Logger log = LoggerFactory.getLogger(CovNNTesting.class);

    public static void main(String[] args){
        // Open a file.

        int nChannels = 1; // Number of input channels
        int outputNum = 20; // The number of possible outcomes
        int batchSize = 50; // Test batch size
        int nEpochs = 1; // Number of training epochs
        int iterations = 1; // Number of training iterations
        int seed = 123; //

        DataAccess dataaccess = new DataAccess();
        log.info("Build model....");
        MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder()
                .seed(seed)
                .iterations(iterations) 
                .regularization(true).l2(0.0005)
                .learningRate(.01)
                .weightInit(WeightInit.XAVIER)
                .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                .updater(Updater.NESTEROVS).momentum(0.9)
                .list()
                .layer(0, new ConvolutionLayer.Builder(5, 5)
                        //nIn and nOut specify depth. nIn here is the nChannels and nOut is the number of filters to be applied
                        .nIn(nChannels)
                        .stride(1, 1)
                        .nOut(20)
                        .activation("identity")
                        .build())
                .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX)
                        .kernelSize(2,2)
                        .stride(2,2)
                        .build())
                .layer(2, new ConvolutionLayer.Builder(5, 5)
                        //Note that nIn need not be specified in later layers
                        .stride(1, 1)
                        .nOut(50)
                        .activation("identity")
                        .build())
                .layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX)
                        .kernelSize(2,2)
                        .stride(2,2)
                        .build())
                .layer(4, new DenseLayer.Builder().activation("relu")
                        .nOut(500).build())
                .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                        .nOut(outputNum)
                        .activation("softmax")
                        .build())
                .setInputType(InputType.convolutional(23,23,35)) //See note below
                .backprop(true).pretrain(false);

        MultiLayerConfiguration conf = builder.build();
        MultiLayerNetwork model = new MultiLayerNetwork(conf);
        model.init();

        log.info("Load data....");

        DataSetIterator train = dataaccess.readData(100,true);
        DataSetIterator test = dataaccess.readData(100, false);

        log.info("Train model....");

        model.setListeners(new ScoreIterationListener(1));
        for( int i=0; i<nEpochs; i++ ) {
            // we combine both into data
            model.fit(train);
            log.info("*** Completed epoch {} ***", i);

            log.info("Evaluate model....");
            Evaluation eval = new Evaluation(outputNum);
            while(test.hasNext()){
                DataSet ds = test.next();
                INDArray output = model.output(ds.getFeatureMatrix(), false);
                eval.eval(ds.getLabels(), output);

            }
            log.info(eval.stats());
            test.reset();
        }
        log.info("****************Example finished********************");
}

/Data Access File/

    package hdf5javacpp;

    import java.util.ArrayList;

    import org.deeplearning4j.datasets.iterator.impl.ListDataSetIterator;
    import org.nd4j.linalg.api.ndarray.INDArray;
    import org.nd4j.linalg.cpu.nativecpu.NDArray;
    import org.nd4j.linalg.dataset.DataSet;
    import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
    import org.nd4j.linalg.factory.Nd4j;
    import org.nd4j.linalg.util.ArrayUtil;

    import hdf.hdf5lib.H5;
    import hdf.hdf5lib.HDF5Constants;
    import hdf.hdf5lib.exceptions.HDF5LibraryException;

    public class DataAccess {

    public static INDArray converBoardDataToNDArray(int[][][] boardData ){
        double[][][] data = coverArrayToDoubleArray(boardData);
        double[]flate = ArrayUtil.flattenDoubleArray(data);
        int[] shape = new int[]{23,23,35};
        INDArray myArr = Nd4j.create(flate, shape,'c');
        return myArr;
    }
    public static INDArray coverActionDataToINDArray(int[] action){
        double[] data = coverArraytoDouble1D(action);
        INDArray myArr = Nd4j.create(data);
        return myArr;
    }
    public static double[][][] coverArrayToDoubleArray(int[][][] boardData){

        double[][][] output = new double[boardData.length][boardData[0].length][boardData[0][0].length];

        for(int i = 0; i < boardData.length; i++){
            for(int j = 0; j < boardData[0].length; j++){
                for(int k = 0; k < boardData[0][0].length; k++){
                    output[i][j][k] = (double)boardData[i][j][k];
                }
            }
        }
        return output;
    }
    public static double[] coverArraytoDouble1D(int[] actionData){
        double[] output = new double[actionData.length];
        for(int i = 0; i < actionData.length; i++){
            output[i] = (double)actionData[i];
        }
        return output;
    }

    public DataSetIterator readData(int bacthSize,boolean train){
        int dataLenght = 10;
        int[] dataGameStep = new int[dataLenght];
        int[][][] boardGameSet = new int[23][23][35];
        int[] actionGameSet = new int[20];
        double[][][][] data_storage = new double[10][23][23][35];
        int[][][][] temp_data = new int[10][23][23][25];
        int[][] temp_action = new int[10][20];
        int dataset_game_lenght_id = -1;
        int dataset_board_id = -1;
        int dataset_action_id = -1;
        int file_id = -1;
        ArrayList<INDArray> collectedINDArrayBoard = new ArrayList<>();
        ArrayList<INDArray> collectedINDArrayLabel = new ArrayList<>();
        INDArray totalInput;
        INDArray totalLabel;
        DataSet dataSet = null;
        try {
            if(file_id < 0){
                if(train){
                    file_id = H5.H5Fopen("train.h5", HDF5Constants.H5F_ACC_RDWR, HDF5Constants.H5P_DEFAULT);
                }else{
                    file_id = H5.H5Fopen("test.h5", HDF5Constants.H5F_ACC_RDWR, HDF5Constants.H5P_DEFAULT);
                }
            }
        }
        catch (Exception e) {
            e.printStackTrace();
        }
        // Open a Group of game step from specific game number
        try{
            if(file_id >= 0){
                dataset_game_lenght_id = H5.H5Dopen(file_id,"/GAME_ROUND_INFO", 
                        HDF5Constants.H5P_DEFAULT);
                H5.H5Dread(dataset_game_lenght_id, HDF5Constants.H5T_NATIVE_INT,
                          HDF5Constants.H5S_ALL,HDF5Constants.H5S_ALL, 
                          HDF5Constants.H5P_DEFAULT, dataGameStep);
                H5.H5Dclose(dataset_game_lenght_id);
                for(int i = 0; i < dataLenght; i++){
                    for(int j=1; j < dataGameStep[i]; j++){
                        String path_boad = "GameNum_"+i+"_GameStep_"+j+"_"+"DATASET_BOARD_INFO";
                        dataset_board_id = H5.H5Dopen(file_id,path_boad,HDF5Constants.H5P_DEFAULT);
                        H5.H5Dread(dataset_board_id, HDF5Constants.H5T_NATIVE_INT,
                                HDF5Constants.H5S_ALL,HDF5Constants.H5S_ALL, HDF5Constants.H5P_DEFAULT,
                                boardGameSet);
                        String path_action = "GameNum_"+i+"_GameStep_"+j+"_"+"DATASET_ACTION_INFO";
                        dataset_action_id = H5.H5Dopen(file_id,path_action,HDF5Constants.H5P_DEFAULT);
                        H5.H5Dread(dataset_action_id, HDF5Constants.H5T_NATIVE_INT,HDF5Constants.H5S_ALL,
                                HDF5Constants.H5S_ALL,HDF5Constants.H5P_DEFAULT, actionGameSet);

                        INDArray input = Nd4j.toFlattened(converBoardDataToNDArray(boardGameSet));
                        INDArray label = Nd4j.toFlattened(coverActionDataToINDArray(actionGameSet));
                        collectedINDArrayBoard.add(input);
                        collectedINDArrayLabel.add(label);
                        H5.H5Dclose(dataset_board_id);
                        H5.H5Dclose(dataset_action_id);

                    }
                }
                H5.H5Fclose(file_id);
                //System.out.println(collectedINDArrayBoard.get(0).rank());
                totalInput = Nd4j.vstack(collectedINDArrayBoard);
                totalLabel = Nd4j.vstack(collectedINDArrayLabel);
                //System.out.println(totalLabel.rank());
                dataSet = new DataSet(totalInput,totalLabel);
            }
        }catch (Exception e) {
            e.printStackTrace();
        }
        return new ListDataSetIterator(dataSet.asList(),bacthSize);
}

/Error Log File/

Exception in thread "main" org.deeplearning4j.exception.DL4JInvalidInputException: Got rank 3 array as input to ConvolutionLayer (layer name = (not named), layer index = 0) with shape [100, 23, 35]. Expected rank 4 array with shape [minibatchSize, layerInputDepth, inputHeight, inputWidth].
    at org.deeplearning4j.nn.layers.convolution.ConvolutionLayer.preOutput(ConvolutionLayer.java:215)
    at org.deeplearning4j.nn.layers.convolution.ConvolutionLayer.activate(ConvolutionLayer.java:291)
    at org.deeplearning4j.nn.layers.BaseLayer.activate(BaseLayer.java:397)
    at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.activationFromPrevLayer(MultiLayerNetwork.java:554)
    at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.feedForwardToLayer(MultiLayerNetwork.java:677)
    at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.computeGradientAndScore(MultiLayerNetwork.java:1871)
    at org.deeplearning4j.optimize.solvers.BaseOptimizer.gradientAndScore(BaseOptimizer.java:149)
    at org.deeplearning4j.optimize.solvers.StochasticGradientDescent.optimize(StochasticGradientDescent.java:54)
    at org.deeplearning4j.optimize.Solver.optimize(Solver.java:51)
    at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.fit(MultiLayerNetwork.java:1084)
    at hdf5javacpp.CovNNTesting.main(CovNNTesting.java:112)

[AgremE]

However, if I try to flat my data first by using

INDArray input = Nd4j.toFlattened(converBoardDataToNDArray(boardGameSet));
INDArray label = Nd4j.toFlattened(coverActionDataToINDArray(actionGameSet));

Then stack all of them together. After that, I construct Covolutional Neural Net with input type of "InputType.convolutionalFlat(23,23,35)", I got the following error:

"Exception in thread "main" org.deeplearning4j.exception.DL4JInvalidInputException: Cannot do forward pass in Convolution layer (layer name = (not named), layer index = 0): input array depth does not match CNN layer configuration (data input depth = 35, [minibatch,inputDepth,height,width]=[100, 35, 23, 23]; expected input depth = 1)
    at org.deeplearning4j.nn.layers.convolution.ConvolutionLayer.preOutput(ConvolutionLayer.java:228)
    at org.deeplearning4j.nn.layers.convolution.ConvolutionLayer.activate(ConvolutionLayer.java:291)
    at org.deeplearning4j.nn.layers.BaseLayer.activate(BaseLayer.java:397)
    at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.activationFromPrevLayer(MultiLayerNetwork.java:554)
    at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.feedForwardToLayer(MultiLayerNetwork.java:677)
    at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.computeGradientAndScore(MultiLayerNetwork.java:1871)
    at org.deeplearning4j.optimize.solvers.BaseOptimizer.gradientAndScore(BaseOptimizer.java:149)
    at org.deeplearning4j.optimize.solvers.StochasticGradientDescent.optimize(StochasticGradientDescent.java:54)
    at org.deeplearning4j.optimize.Solver.optimize(Solver.java:51)
    at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.fit(MultiLayerNetwork.java:1084)
    at hdf5javacpp.CovNNTesting.main(CovNNTesting.java:112)"

[turambar]

@AlexDBlack Relevant to support for 3D convolutions (eventually needed for Keras model import)?

[jadedfire]

+1 on needing 3D convolution support (true volume data)

[maxpumperla]

Closing in favour of https://github.com/deeplearning4j/deeplearning4j/issues/3174, will likely support this in the next few months.

[lock[bot]]

This thread has been automatically locked since there has not been any recent activity after it was closed. Please open a new issue for related bugs.