The issues of Dataset format

[Lxy85]

The dataset.h file has overloaded functions for reading datasets. There are file formats for reading and datasets for reading Tensor<type, 2> variable types. I tried to read the .csv file into the Tensor<type, 2> variable as it is, and the result of training is wrong. When reading a data set of Tensor type, what format should the data set be stored in? Thank you.

[davidge807]

Hi @Lxy85 Can you please provide us with some code to better understand your question?

[Lxy85]

Ok, here is the code, thank you @davidge807

include

include

include

include

include

include

include

include

include

include

include "/home/allen/opennn/opennn.h"

using namespace std; using namespace OpenNN;

Tensor<float, 2> data1(200,5); int tensor_size1=0;

void read_csvTotensor() { ifstream inFile1("/home/allen/Data/2.csv",ios::in);//Kinds are indicated by numbers if(!inFile1) { cout<<"fail to open the file！"<<endl; exit(1); } int j=0; string line1; while(getline(inFile1,line1)) { if(j>0&&j<101) { string field1; string s; istringstream sin(line1); char temp[3]; for(int p=0;p<5;p++) { getline(sin,field1,';'); s+=field1; for(int i=0;i<3;i++) { temp[i]=field1[i]; }

    data1(tensor_size1,p)=atof(temp);
  }
  tensor_size1++;
 }
   j++;

}

inFile1.close(); } int main() { read_csvTotensor(); cout << "OpenNN. Iris Plant Example." << endl;

srand(static_cast<unsigned>(time(nullptr)));

for(int i=0;i<10;i++) { for(int j=0;j<5;j++) { cout<<data1(i,j)<<endl; }

} // Data set

//DataSet data_set("/home/allen/Data/iris_plant_original.csv", ';', true); //DataSet data_set("/home/allen/Data/iris.data", ',', false); DataSet data_set(data1);

/*const Index input_variables_number = data_set.get_input_variables_number();
const Index target_variables_number = data_set.get_target_variables_number();
cout << "input_variables_number" << input_variables_number <<endl;
cout << "target_variables_number" << target_variables_number <<endl;*/
const Index input_variables_number = 4;
const Index target_variables_number = 3;//It feels like the problem is here,butI set it up using the results of the original dataset output

// Neural network

const Index hidden_neurons_number = 3;
Tensor<Index, 1> architecture(3);
architecture[0] = input_variables_number;
architecture[1] = 10;
architecture[2] = target_variables_number;

NeuralNetwork neural_network(NeuralNetwork::ProjectType::Classification, architecture);

//NeuralNetwork neural_network(NeuralNetwork::ProjectType::Classification, {input_variables_number, hidden_neurons_number, target_variables_number});

// Training strategy

TrainingStrategy training_strategy(&neural_network, &data_set);

training_strategy.set_loss_method(TrainingStrategy::LossMethod::CROSS_ENTROPY_ERROR);
training_strategy.set_optimization_method(TrainingStrategy::OptimizationMethod::STOCHASTIC_GRADIENT_DESCENT);
training_strategy.set_display_period(10);
training_strategy.set_maximum_epochs_number(1);
training_strategy.perform_training();

// Testing analysis

const TestingAnalysis testing_analysis(&neural_network, &data_set);

Tensor<type, 2> inputs(3, 4);
Tensor<type, 2> outputs;

inputs.setValues({{type(5.1),type(3.5),type(1.4),type(0.2)},
                {type(6.4),type(3.2),type(4.5),type(1.5)},
                {type(6.3),type(2.7),type(4.9),type(1.8)}});

const Tensor<Index, 2> confusion = testing_analysis.calculate_confusion();
outputs = neural_network.calculate_outputs(inputs);

cout << "\nInputs:\n" << inputs << endl;

cout << "\nOutputs:\n" << outputs << endl;

cout << "\nConfusion matrix:\n" << confusion << endl;

return 0;

}

[davidge807]

Hi again @Lxy85 I've been trying to run your code on my device, and it seems that the error is in the function "read_csvTotensor()". Using it to import the file irisflowers.csv and printing the data obtained, it returns the following:

        5.1     5.1     5.1     5.1     5.1
        4.9     4.9     4.9     4.9     4.9
        4.7     4.7     4.7     4.7     4.7
        4.6     4.6     4.6     4.6     4.6
        ...

You can use the DataSet constructor DataSet::DataSet(const string& data_file_name, const char& separator, const bool& has_columns_names) and then DataSet::get_data() for obtaining Tensor<type,2> of data.

[Lxy85]

Hi again @Lxy85 I've been trying to run your code on my device, and it seems that the error is in the function "read_csvTotensor()". Using it to import the file irisflowers.csv and printing the data obtained, it returns the following:

        5.1     5.1     5.1     5.1     5.1
        4.9     4.9     4.9     4.9     4.9
        4.7     4.7     4.7     4.7     4.7
        4.6     4.6     4.6     4.6     4.6
        ...

You can use the DataSet constructor DataSet::DataSet(const string& data_file_name, const char& separator, const bool& has_columns_names) and then DataSet::get_data() for obtaining Tensor<type,2> of data.

Thank you!

[Lxy85]

Hi @davidge807 I tried the method you provided, but there are still problems during training. I found that the input_variables_number and target_variables_number obtained are different when the same dataset is trained in the form of files and in the form of Tensor variables. The following is the code of the main function and result. int main() { cout << "OpenNN. Iris Plant Example." << endl;

srand(static_cast<unsigned>(time(nullptr)));

//DataSet data_set("/home/allen/Data/3.csv", ';', false);
DataSet data_set("/home/allen/Data/iris.data", ',', false);

// cout<<"iris.data:"<<endl; //cout<<data_set.get_data()<<endl;

data1=data_set.get_data();
DataSet data(data1);
//cout<<"Tensor:"<<endl;
//cout<<data.get_data()<<endl;

const Index input_variables_number1 = data_set.get_input_variables_number();
const Index target_variables_number1 = data_set.get_target_variables_number();
cout << "input_variables_number:(iris.data)" << input_variables_number1 <<endl;
cout << "target_variables_number:(iris.data)" << target_variables_number1 <<endl;

const Index input_variables_number = data.get_input_variables_number();
const Index target_variables_number = data.get_target_variables_number();
cout << "input_variables_number(Tensor):" << input_variables_number <<endl;
cout << "target_variables_number(Tensor):" << target_variables_number <<endl;

// Neural network

const Index hidden_neurons_number = 3;
Tensor<Index, 1> architecture(3);
architecture[0] = input_variables_number;
architecture[1] = 10;
architecture[2] = target_variables_number;

NeuralNetwork neural_network(NeuralNetwork::ProjectType::Classification, architecture);

// Training strategy

TrainingStrategy training_strategy(&neural_network, &data);

// TrainingStrategy training_strategy(&neural_network, &data_set);

training_strategy.set_loss_method(TrainingStrategy::LossMethod::CROSS_ENTROPY_ERROR);
training_strategy.set_optimization_method(TrainingStrategy::OptimizationMethod::STOCHASTIC_GRADIENT_DESCENT);
training_strategy.set_display_period(50);
training_strategy.set_maximum_epochs_number(50);
training_strategy.perform_training();

// Testing analysis

//const TestingAnalysis testing_analysis(&neural_network, &data_set);
const TestingAnalysis testing_analysis(&neural_network, &data);

Tensor<type, 2> inputs(3, 4);
Tensor<type, 2> outputs;

inputs.setValues({{type(5.1),type(3.5),type(1.4),type(0.2)},
                {type(6.4),type(3.2),type(4.5),type(1.5)},
                {type(6.3),type(2.7),type(4.9),type(1.8)}});

const Tensor<Index, 2> confusion = testing_analysis.calculate_confusion();
outputs = neural_network.calculate_outputs(inputs);

cout << "\nInputs:\n" << inputs << endl;

cout << "\nOutputs:\n" << outputs << endl;

cout << "\nConfusion matrix:\n" << confusion << endl;

return 0;

} input_variables_number:(iris.data)4 target_variables_number:(iris.data)3 input_variables_number(Tensor):6 target_variables_number(Tensor):1

Confusion matrix: 10 0 4 16

[davidge807]

Hi again @Lxy85 If you initialize the data set using a Tensor, you must provide the input and target indexs. You may use data_set.set_input_target_columns(Tensor<Index,1> &input_indices, Tensor<Index,1> &target_indices) for that. Your code should be properly working when you add this line.

[Lxy85]

The problem has been resolved. Thank you! @davidge807