search

Tencent / NeuralNLP-NeuralClassifier

An Open-source Neural Hierarchical Multi-label Text Classification Toolkit
Other
1.83k stars 402 forks source link

f-score is equal to zero #54

Closed siyanew closed 4 years ago

siyanew commented 4 years ago

I have run train.py with this config and after 13 epoch I still get 0 fscore.

Config:

{
  "task_info":{
    "label_type": "multi_label",
    "hierarchical": true,
    "hierar_taxonomy": "data/rcv1.taxonomy",
    "hierar_penalty": 0.000001
  },
  "device": "cuda",
  "model_name": "TextCNN",
  "checkpoint_dir": "checkpoint_dir_rcv1",
  "model_dir": "trained_model_rcv1",
  "data": {
    "train_json_files": [
      "data/rcv1_train.json"
    ],
    "validate_json_files": [
      "data/rcv1_dev.json"
    ],
    "test_json_files": [
      "data/rcv1_test.json"
    ],
    "generate_dict_using_json_files": false,
    "generate_dict_using_all_json_files": false,
    "generate_dict_using_pretrained_embedding": false,
    "dict_dir": "dict_rcv1",
    "num_worker": 4
  },
  "feature": {
    "feature_names": [
      "token"
    ],
    "min_token_count": 2,
    "min_char_count": 2,
    "token_ngram": 0,
    "min_token_ngram_count": 0,
    "min_keyword_count": 0,
    "min_topic_count": 2,
    "max_token_dict_size": 1000000,
    "max_char_dict_size": 150000,
    "max_token_ngram_dict_size": 10000000,
    "max_keyword_dict_size": 100,
    "max_topic_dict_size": 100,
    "max_token_len": 256,
    "max_char_len": 1024,
    "max_char_len_per_token": 4,
    "token_pretrained_file": "",
    "keyword_pretrained_file": ""
  },
  "train": {
    "batch_size": 64,
    "start_epoch": 1,
    "num_epochs": 20,
    "num_epochs_static_embedding": 0,
    "decay_steps": 1000,
    "decay_rate": 1.0,
    "clip_gradients": 100.0,
    "l2_lambda": 0.0,
    "loss_type": "BCEWithLogitsLoss",
    "sampler": "fixed",
    "num_sampled": 5,
    "visible_device_list": "0",
    "hidden_layer_dropout": 0.5
  },
  "embedding": {
    "type": "embedding",
    "dimension": 64,
    "region_embedding_type": "context_word",
    "region_size": 5,
    "initializer": "uniform",
    "fan_mode": "FAN_IN",
    "uniform_bound": 0.25,
    "random_stddev": 0.01,
    "dropout": 0.0
  },
  "optimizer": {
    "optimizer_type": "Adam",
    "learning_rate": 0.008,
    "adadelta_decay_rate": 0.95,
    "adadelta_epsilon": 1e-08
  },
  "TextCNN": {
    "kernel_sizes": [
      2,
      3,
      4
    ],
    "num_kernels": 100,
    "top_k_max_pooling": 1
  },
  "TextRNN": {
    "hidden_dimension": 64,
    "rnn_type": "GRU",
    "num_layers": 1,
    "doc_embedding_type": "Attention",
    "attention_dimension": 16,
    "bidirectional": true
  },
  "DRNN": {
    "hidden_dimension": 5,
    "window_size": 3,
    "rnn_type": "GRU",
    "bidirectional": true,
    "cell_hidden_dropout": 0.1
  },
  "eval": {
    "text_file": "data/rcv1_test.json",
    "threshold": 0.5,
    "dir": "eval_dir",
    "batch_size": 1024,
    "is_flat": true,
    "top_k": 30,
    "model_dir": "checkpoint_dir_rcv1/TextCNN_best"
  },
  "TextVDCNN": {
    "vdcnn_depth": 9,
    "top_k_max_pooling": 8
  },
  "DPCNN": {
    "kernel_size": 3,
    "pooling_stride": 2,
    "num_kernels": 16,
    "blocks": 2
  },
  "TextRCNN": {
    "kernel_sizes": [
        2,
        3,
        4
    ],
    "num_kernels": 100,
    "top_k_max_pooling": 1,
    "hidden_dimension":64,
    "rnn_type": "GRU",
    "num_layers": 1,
    "bidirectional": true
  },
  "Transformer": {
    "d_inner": 128,
    "d_k": 32,
    "d_v": 32,
    "n_head": 4,
    "n_layers": 1,
    "dropout": 0.1,
    "use_star": true
  },
  "AttentiveConvNet": {
    "attention_type": "bilinear",
    "margin_size": 3,
    "type": "advanced",
    "hidden_size": 64
  },
  "log": {
    "logger_file": "log_test_rcv1_hierar",
    "log_level": "warn"
  }
}

Result:


Size of doc_token dict is 0
Size of doc_char dict is 0
Size of doc_token_ngram dict is 0
Size of doc_keyword dict is 0
Size of doc_topic dict is 0
Shrink dict over.
Size of doc_label dict is 100
Size of doc_token dict is 0
Size of doc_char dict is 0
Size of doc_token_ngram dict is 0
Size of doc_keyword dict is 0
Size of doc_topic dict is 0
Train performance at epoch 1 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 259686.
Loss is: 0.176090.
Validate performance at epoch 1 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 28687.
Loss is: 0.175878.
test performance at epoch 1 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 72472.
Loss is: 0.176301.
Epoch 1 cost time: 273 second
Train performance at epoch 2 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 259686.
Loss is: 0.185657.
Validate performance at epoch 2 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 28687.
Loss is: 0.185401.
test performance at epoch 2 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 72472.
Loss is: 0.185863.
Epoch 2 cost time: 275 second
Train performance at epoch 3 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 259686.
Loss is: 0.179540.
Validate performance at epoch 3 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 28687.
Loss is: 0.179308.
test performance at epoch 3 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 72472.
Loss is: 0.179765.
Epoch 3 cost time: 274 second
Train performance at epoch 4 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 259686.
Loss is: 0.181358.
Validate performance at epoch 4 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 28687.
Loss is: 0.181102.
test performance at epoch 4 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 72472.
Loss is: 0.181564.
Epoch 4 cost time: 274 second
Train performance at epoch 5 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 259686.
Loss is: 0.177109.
Validate performance at epoch 5 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 28687.
Loss is: 0.176923.
test performance at epoch 5 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 72472.
Loss is: 0.177328.
Epoch 5 cost time: 274 second
Train performance at epoch 6 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 259686.
Loss is: 0.180688.
Validate performance at epoch 6 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 28687.
Loss is: 0.180476.
test performance at epoch 6 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 72472.
Loss is: 0.180903.
Epoch 6 cost time: 275 second
Train performance at epoch 7 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 259686.
Loss is: 0.177683.
Validate performance at epoch 7 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 28687.
Loss is: 0.177484.
test performance at epoch 7 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 72472.
Loss is: 0.177902.
Epoch 7 cost time: 278 second
Train performance at epoch 8 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 259686.
Loss is: 0.180226.
Validate performance at epoch 8 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 28687.
Loss is: 0.179987.
test performance at epoch 8 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 72472.
Loss is: 0.180454.
Epoch 8 cost time: 277 second
Train performance at epoch 9 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 259686.
Loss is: 0.176957.
Validate performance at epoch 9 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 28687.
Loss is: 0.176734.
test performance at epoch 9 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 72472.
Loss is: 0.177183.
Epoch 9 cost time: 277 second
Train performance at epoch 10 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 259686.
Loss is: 0.192425.
Validate performance at epoch 10 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 28687.
Loss is: 0.192209.
test performance at epoch 10 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 72472.
Loss is: 0.192622.
Epoch 10 cost time: 275 second
Train performance at epoch 11 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 259686.
Loss is: 0.181247.
Validate performance at epoch 11 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 28687.
Loss is: 0.181020.
test performance at epoch 11 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 72472.
Loss is: 0.181468.
Epoch 11 cost time: 277 second
Train performance at epoch 12 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 259686.
Loss is: 0.179442.
Validate performance at epoch 12 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 28687.
Loss is: 0.179230.
test performance at epoch 12 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 72472.
Loss is: 0.179660.
Epoch 12 cost time: 276 second
Train performance at epoch 13 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 259686.
Loss is: 0.182252.
Validate performance at epoch 13 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 28687.
Loss is: 0.182036.
test performance at epoch 13 is precision: 0.000000, recall: 0.000000, fscore: 0.000000, macro-fscore: 0.000000, right: 0, predict: 0, standard: 72472.
Loss is: 0.182464.```
coderbyr commented 4 years ago

generate_dict_using_json_files

hi, you should set generate_dict_using_json_files or generate_dict_using_all_json_files be True, or the dataset won't generate right dict, all token will be convert to UNK.