search

PaddlePaddle / Paddle

PArallel Distributed Deep LEarning: Machine Learning Framework from Industrial Practice （『飞桨』核心框架，深度学习&机器学习高性能单机、分布式训练和跨平台部署）
http://www.paddlepaddle.org/
Apache License 2.0
22.26k stars 5.6k forks source link

模型训练时不收敛 #17051

Closed sunflowerlyb closed 4 years ago

sunflowerlyb commented 5 years ago

你好，当前模型训练一直不收敛，cost值先变小后变大，但是变小并没有到最小 修改了learning_rate为0.001、0.0001后依然存在，请问下是什么原因造成的呢？ 训练数据大小：222条，测试数据集大小30条 数据输入长度最长2000 batch_size为40 其他各参数设置如下： image

辛苦帮忙看下会是什么原因造成的吧? 我这个是按站点训练的模型，有些站点可以正常训练，有些站点会出现这种情况。 模型训练版本1.2.0

shippingwang commented 5 years ago

什么模型？replied: RNN 代码帮忙发下 @sunflowerlyb

shippingwang commented 5 years ago

减小bs试下？这个调参问题 您可以试着按照一些策略调一下 或者如果可以，把代码发出来看下

sunflowerlyb commented 5 years ago

代码： """ This file is used to train the model. """ import os import sys import math import time import random import argparse

import numpy as np import paddle import paddle.fluid as fluid

import reader

from network import lex_net

def file_reader(file_dir, word2id_dict, label2id_dict, word_replace_dict, filename_feature=""): """ define the reader to read files in file_dir """ word_dict_len = max(map(int, word2id_dict.values())) + 1 label_dict_len = max(map(int, label2id_dict.values())) + 1 def reader(): """ the data generator """ index = 0 for root, dirs, files in os.walk(file_dir): for filename in files: if not filename.startswith(filename_feature): continue cnt_org = 0 cnt_res = 0 for line in open(os.path.join(root, filename), 'r'): cnt_org = cnt_org + 1 index += 1 bad_line = False line = line.strip("\n") if len(line) == 0: continue seg_tag = line.rfind("\t") word_part = line[0:seg_tag] label_part = line[seg_tag + 1:] word_idx = [] seperator = chr(1) words = word_part.split(seperator) for word_tmp in words: word = word_tmp.decode('utf-8')

if word in word_replace_dict:

                    #    word = word_replace_dict[word]
                    if word in word2id_dict:
                        word_idx.append(int(word2id_dict[word]))
                    else:
                        word_idx.append(int(word2id_dict["OOV"]))
                target_idx = []
                labels = label_part.strip().split(" ")
                for label in labels:
                    if label in label2id_dict:
                        target_idx.append(int(label2id_dict[label]))
                    else:
                        target_idx.append(int(label2id_dict["O"]))
        word_idx = word_idx[0:len(word_idx) - 1]
                if len(word_idx) != len(target_idx):
                    continue             
    #    label_final = 0
    #    for i in range(len(word_idx) - 1, 0, -1):
    #        if (target_idx[i] != 0):
    #       label_final = i
    #       break
                yield word_idx, target_idx
                cnt_res = cnt_res + 1
            print file_dir, cnt_res, cnt_org
return reader

def load_dict(dict_path): """ Load a dict. The first column is the key and the second column is the value. """ result_dict = {} for line in open(dict_path, "r"): terms = line.strip("\n").split("\t") if len(terms) != 2: continue result_dict[terms[0].decode("utf-8")] = terms[1].decode("utf-8") return result_dict

def load_reverse_dict(dict_path): """ Load a dict. The first column is the value and the second column is the key. """ result_dict = {} for line in open(dict_path, "r"): terms = line.strip("\n").split("\t") if len(terms) != 2: continue result_dict[terms[1].decode("utf-8")] = terms[0].decode("utf-8") return result_dict

def parse_args(): """ Parsing the input parameters. """ parser = argparse.ArgumentParser("Training for lexical analyzer.") parser.add_argument( "--traindata_dir", type=str, default="./conf/train_data",

default="./data/train_data",

    help="The folder where the training data is located.")
parser.add_argument(
    "--testdata_dir",
    type=str,
    default="./conf/test_data",
    help="The folder where the training data is located.")
parser.add_argument(
    "--model_save_dir",
    type=str,
    default="./models",
    help="The model will be saved in this path.")
parser.add_argument(
    "--save_model_per_batchs",
    type=int,
    default=5,
    help="Save the model once per xxxx batch of training")
parser.add_argument(
    "--eval_window",
    type=int,
    default=20,
    help="Training will be suspended when the evaluation indicators on the validation set" \
         " no longer increase. The eval_window specifies the scope of the evaluation.")
parser.add_argument(
    "--batch_size",
    type=int,
    default=40,
    help="The number of sequences contained in a mini-batch, or the maximum" \
         "number of tokens (include paddings) contained in a mini-batch.")
parser.add_argument(
    "--corpus_type_list",
    type=str,
    #default=["human", "feed", "query", "title", "news"],
    default=["train_data"],
    nargs='+',
    help="The pattern list of different types of corpus used in training.")
parser.add_argument(
    "--corpus_proportion_list",
    type=float,
    #default=[0.2, 0.2, 0.2, 0.2, 0.2],
    default=[1],
    nargs='+',
    help="The proportion list of different types of corpus used in training.")
parser.add_argument(
    "--use_gpu",
    type=int,
    default=False,
    help="Whether or not to use GPU. 0-->CPU 1-->GPU")
parser.add_argument(
    "--traindata_shuffle_buffer",
    type=int,
    default=200000,
    help="The buffer size used in shuffle the training data.")
parser.add_argument(
    "--word_emb_dim",
    type=int,
    default=128,
    help="The dimension in which a word is embedded.")
parser.add_argument(
    "--grnn_hidden_dim",
    type=int,
    default=256,
    help="The number of hidden nodes in the GRNN layer.")
parser.add_argument(
    "--bigru_num",
    type=int,
    default=2,
    help="The number of bi_gru layers in the network.")
parser.add_argument(
    "--base_learning_rate",
    type=float,
    default=1e-4,
    help="The basic learning rate that affects the entire network.")
parser.add_argument(
    "--emb_learning_rate",
    type=float,
    default=5,
    help="The real learning rate of the embedding layer will be" \
    " (emb_learning_rate * base_learning_rate)."
)
parser.add_argument(
    "--crf_learning_rate",
    type=float,
    default=0.2,
    help="The real learning rate of the embedding layer will be" \
         " (crf_learning_rate * base_learning_rate)."
)
parser.add_argument(
    "--word_dict_path",
    type=str,
    default="./conf/word.txt",
    #default="./conf/word.dic",
    help="The path of the word dictionary."
)
parser.add_argument(
    "--label_dict_path",
    type=str,
    #default="./conf/tag.dic",
    default="./conf/label.txt",
    help="The path of the label dictionary."
)
parser.add_argument(
    "--word_rep_dict_path",
    type=str,
    default="./conf/q2b.dic",
    help="The path of the word replacement Dictionary."
)
parser.add_argument(
    "--num_iterations",
    type=int,
    default=0,
    help="The maximum number of iterations. If set to 0 (default), do not limit the number."
)
args = parser.parse_args()
if len(args.corpus_proportion_list) != len(args.corpus_type_list):
    sys.stderr.write(
        "The length of corpus_proportion_list should be equal to the length of corpus_type_list.\n"
    )
    exit(-1)
return args

def print_arguments(args): print('----------- Configuration Arguments -----------') for arg, value in sorted(vars(args).iteritems()): print('%s: %s' % (arg, value)) print('------------------------------------------------')

def to_lodtensor(data, place): """ Convert data in list into lodtensor. """ seq_lens = [len(seq) for seq in data] cur_len = 0 lod = [cur_len] for l in seq_lens: cur_len += l lod.append(cur_len) flattened_data = np.concatenate(data, axis=0).astype("int64") flattened_data = flattened_data.reshape([len(flattened_data), 1]) res = fluid.LoDTensor() res.set(flattened_data, place) res.set_lod([lod]) return res

def test(exe, chunk_evaluator, inference_program, test_data, place, cur_fetch_list): """ Test the network in training. """ chunk_evaluator.reset() for data in test_data(): word = to_lodtensor(map(lambda x: x[0], data), place) target = to_lodtensor(map(lambda x: x[1], data), place) result_list = exe.run( inference_program, feed={ "word": word, "target": target }, fetch_list=cur_fetch_list) number_infer = np.array(result_list[0]) number_label = np.array(result_list[1]) number_correct = np.array(result_list[2]) chunk_evaluator.update(number_infer[0], number_label[0], number_correct[0]) return chunk_evaluator.eval()

def train(args): """ Train the network. """ if not os.path.exists(args.model_save_dir): os.mkdir(args.model_save_dir)

word2id_dict = load_reverse_dict(args.word_dict_path)
label2id_dict = load_reverse_dict(args.label_dict_path)
word_rep_dict = load_dict(args.word_rep_dict_path)
word_dict_len = max(map(int, word2id_dict.values())) + 1                
label_dict_len = max(map(int, label2id_dict.values())) + 1 

avg_cost, crf_decode, word, target= lex_net(args, word_dict_len, label_dict_len)
sgd_optimizer = fluid.optimizer.SGD(learning_rate=args.base_learning_rate)
sgd_optimizer.minimize(avg_cost)

(precision, recall, f1_score, num_infer_chunks, num_label_chunks,
 num_correct_chunks) = fluid.layers.chunk_eval(
     input=crf_decode,
     label=target,
     chunk_scheme="IOB",
     num_chunk_types=int(math.ceil((label_dict_len - 1) / 2.0)))
chunk_evaluator = fluid.metrics.ChunkEvaluator()
chunk_evaluator.reset()
inference_program = fluid.default_main_program().clone()
with fluid.program_guard(inference_program):
    inference_program = fluid.default_main_program().clone(for_test=True)
train_reader_list = []
corpus_num = len(args.corpus_type_list)
for i in xrange(corpus_num):
    train_reader = paddle.batch(
        paddle.reader.shuffle(
            file_reader(args.traindata_dir,
                                        word2id_dict,
                                        label2id_dict,
                                        word_rep_dict,
                                        args.corpus_type_list[i]),
            buf_size=args.traindata_shuffle_buffer),
        batch_size=int(args.batch_size * args.corpus_proportion_list[i]))
    train_reader_list.append(train_reader)
test_reader = paddle.batch(
    file_reader(args.testdata_dir, word2id_dict, label2id_dict, word_rep_dict),
    batch_size=args.batch_size)
train_reader_itr_list = []
for train_reader in train_reader_list:
    cur_reader_itr = train_reader()
    train_reader_itr_list.append(cur_reader_itr)

place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
feeder = fluid.DataFeeder(feed_list=[word, target], place=place)
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
batch_id = 0
start_time = time.time()
eval_list = []
iter = 0
cnt_fail = 0
while iter <= 30002:
    full_batch = []
    cur_batch = []
    for i in xrange(corpus_num):
        reader_itr = train_reader_itr_list[i]
        try:
            cur_batch = next(reader_itr)
        except StopIteration:
            print(args.corpus_type_list[i] +
                  " corpus finish a pass of training")
            new_reader = train_reader_list[i]
            train_reader_itr_list[i] = new_reader()
            cur_batch = next(train_reader_itr_list[i])
        full_batch += cur_batch
    random.shuffle(full_batch)

    cost_var, nums_infer, nums_label, nums_correct = exe.run(
        fluid.default_main_program(),
        fetch_list=[
            avg_cost, num_infer_chunks, num_label_chunks,
            num_correct_chunks
        ],
        feed=feeder.feed(full_batch))
    print("batch_id:" + str(batch_id) + ", avg_cost:" + str(cost_var[0]))
    chunk_evaluator.update(nums_infer, nums_label, nums_correct)
    batch_id += 1

    if (batch_id % args.save_model_per_batchs == 1):
        save_exe = fluid.Executor(place)
        save_dirname = os.path.join(args.model_save_dir,
                                    "params_batch_%d" % batch_id)
        fluid.io.save_inference_model(save_dirname, ['word'], [crf_decode],
                                      save_exe, main_program=inference_program)
        precision, recall, f1_score = chunk_evaluator.eval()
        print("[train] batch_id:" + str(batch_id) + ", precision:" +
              str(precision) + ", recall:" + str(recall) + ", f1:" +
              str(f1_score))
        chunk_evaluator.reset()
        p, r, f1 = test(
            exe, chunk_evaluator, inference_program, test_reader, place,
            [num_infer_chunks, num_label_chunks, num_correct_chunks])
        print("[test] batch_id:" + str(batch_id) + ", precision:" +
              str(p) + ", recall:" + str(r) + ", f1:" + str(f1))
        end_time = time.time()
        print("cur_batch_id:" + str(batch_id) + ", last " +
              str(args.save_model_per_batchs) + " batchs, time_cost:" +
              str(end_time - start_time))
        start_time = time.time()

        if len(eval_list) < 2 * args.eval_window:
            eval_list.append(f1)
        else:
            eval_list.pop(0)
            eval_list.append(f1)
            last_avg_f1 = sum(
                eval_list[0:args.eval_window]) / args.eval_window
            cur_avg_f1 = sum(eval_list[
                args.eval_window:2 * args.eval_window]) / args.eval_window
            if cur_avg_f1 <= last_avg_f1:
                cnt_fail += 1
                print "cur_avg_fs <= last_avg_f1! "
                if cnt_fail > 10:
                    return
            else:
                cnt_fail = 0
                print "keep training!"
    iter += 1
    if (iter == args.num_iterations):
        return

if name == "main": args = parse_args() print_arguments(args) train(args)

sunflowerlyb commented 5 years ago

减小bs试下？这个调参问题 您可以试着按照一些策略调一下 或者如果可以，把代码发出来看下

我尝试了bs从40改为20或者50还是这样的。不收敛.

hjchen2 commented 5 years ago

看cost并没有变大，训练过程中肯定是会有一些波动的，可以增加一些数据量和多训练几轮看看结果。

sunflowerlyb commented 5 years ago

看cost并没有变大，训练过程中肯定是会有一些波动的，可以增加一些数据量和多训练几轮看看结果。

上面那个图cost的值显示的不全，是为了列举模型参数的，下面这个是整个cost的变化，最后cost值瞬间变大了很多，一直也没有下降

----------- Configuration Arguments ----------- base_learning_rate: 0.001 batch_size: 50 bigru_num: 2 corpus_proportion_list: [1] corpus_type_list: ['train_data'] crf_learning_rate: 0.2 emb_learning_rate: 5 eval_window: 20 grnn_hidden_dim: 256 label_dict_path: ./conf/label.txt model_save_dir: ./models num_iterations: 0 save_model_per_batchs: 5 testdata_dir: ./conf/test_data traindata_dir: ./conf/train_data traindata_shuffle_buffer: 200000 use_gpu: False word_dict_path: ./conf/word.txt word_emb_dim: 128 word_rep_dict_path: ./conf/q2b.dic

./conf/train_data 222 222 batch_id:0, avg_cost:2366.9077 [train] batch_id:1, precision:[0.08190622], recall:[0.04684721], f1:[0.05960351] ./conf/test_data 30 30 [test] batch_id:1, precision:[0.96754024], recall:[0.9740625], f1:[0.97079042] cur_batch_id:1, last 5 batchs, time_cost:18.5129520893 batch_id:1, avg_cost:199.21548 batch_id:2, avg_cost:189.14902 batch_id:3, avg_cost:185.65263 batch_id:4, avg_cost:183.33946 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:5, avg_cost:186.00348 [train] batch_id:6, precision:[0.96614719], recall:[0.97416145], f1:[0.97013777] ./conf/test_data 30 30 [test] batch_id:6, precision:[0.96754024], recall:[0.9740625], f1:[0.97079042] cur_batch_id:6, last 5 batchs, time_cost:51.4624049664 batch_id:6, avg_cost:182.72617 batch_id:7, avg_cost:173.66454 batch_id:8, avg_cost:189.37361 batch_id:9, avg_cost:178.70583 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:10, avg_cost:187.64755 [train] batch_id:11, precision:[0.96286026], recall:[0.97186584], f1:[0.96734209] ./conf/test_data 30 30 [test] batch_id:11, precision:[0.96754024], recall:[0.9740625], f1:[0.97079042] cur_batch_id:11, last 5 batchs, time_cost:48.7185890675 batch_id:11, avg_cost:222.07236 batch_id:12, avg_cost:197.66876 batch_id:13, avg_cost:181.45526 batch_id:14, avg_cost:191.00937 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:15, avg_cost:177.54623 [train] batch_id:16, precision:[0.96626462], recall:[0.97463532], f1:[0.97043192] ./conf/test_data 30 30 [test] batch_id:16, precision:[0.96754024], recall:[0.9740625], f1:[0.97079042] cur_batch_id:16, last 5 batchs, time_cost:51.9831688404 batch_id:16, avg_cost:169.08972 batch_id:17, avg_cost:176.76445 batch_id:18, avg_cost:183.57555 batch_id:19, avg_cost:194.19347 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:20, avg_cost:187.96387 [train] batch_id:21, precision:[0.96436877], recall:[0.97276946], f1:[0.9685509] ./conf/test_data 30 30 [test] batch_id:21, precision:[0.96754024], recall:[0.9740625], f1:[0.97079042] cur_batch_id:21, last 5 batchs, time_cost:50.2134890556 batch_id:21, avg_cost:175.13324 batch_id:22, avg_cost:175.83594 batch_id:23, avg_cost:175.81049 batch_id:24, avg_cost:176.49579 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:25, avg_cost:169.30278 [train] batch_id:26, precision:[0.965802], recall:[0.97411694], f1:[0.96994165] ./conf/test_data 30 30 [test] batch_id:26, precision:[0.96754024], recall:[0.9740625], f1:[0.97079042] cur_batch_id:26, last 5 batchs, time_cost:51.0059669018 batch_id:26, avg_cost:168.10968 batch_id:27, avg_cost:177.42542 batch_id:28, avg_cost:183.82002 batch_id:29, avg_cost:199.70189 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:30, avg_cost:183.59383 [train] batch_id:31, precision:[0.96451317], recall:[0.97327286], f1:[0.96887322] ./conf/test_data 30 30 [test] batch_id:31, precision:[0.96754024], recall:[0.9740625], f1:[0.97079042] cur_batch_id:31, last 5 batchs, time_cost:50.4568808079 batch_id:31, avg_cost:165.80646 batch_id:32, avg_cost:176.89014 batch_id:33, avg_cost:172.32672 batch_id:34, avg_cost:178.21706 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:35, avg_cost:170.43794 [train] batch_id:36, precision:[0.96469718], recall:[0.97320933], f1:[0.96893456] ./conf/test_data 30 30 [test] batch_id:36, precision:[0.96754024], recall:[0.9740625], f1:[0.97079042] cur_batch_id:36, last 5 batchs, time_cost:50.0860080719 batch_id:36, avg_cost:164.17354 batch_id:37, avg_cost:179.29102 batch_id:38, avg_cost:165.40787 batch_id:39, avg_cost:186.3816 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:40, avg_cost:172.85657 [train] batch_id:41, precision:[0.9642167], recall:[0.97289302], f1:[0.96853543] ./conf/test_data 30 30 [test] batch_id:41, precision:[0.96754024], recall:[0.9740625], f1:[0.97079042] cur_batch_id:41, last 5 batchs, time_cost:49.9946949482 batch_id:41, avg_cost:170.59883 batch_id:42, avg_cost:161.02394 batch_id:43, avg_cost:172.59984 batch_id:44, avg_cost:160.27277 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:45, avg_cost:169.38324 [train] batch_id:46, precision:[0.96572385], recall:[0.97394707], f1:[0.96981803] ./conf/test_data 30 30 [test] batch_id:46, precision:[0.96754024], recall:[0.9740625], f1:[0.97079042] cur_batch_id:46, last 5 batchs, time_cost:50.7684149742 batch_id:46, avg_cost:176.985 batch_id:47, avg_cost:171.23367 batch_id:48, avg_cost:160.24301 batch_id:49, avg_cost:166.57253 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:50, avg_cost:153.29279 [train] batch_id:51, precision:[0.96397269], recall:[0.97281875], f1:[0.96837552] ./conf/test_data 30 30 [test] batch_id:51, precision:[0.96754024], recall:[0.9740625], f1:[0.97079042] cur_batch_id:51, last 5 batchs, time_cost:48.6295690536 batch_id:51, avg_cost:166.3755 batch_id:52, avg_cost:162.75645 batch_id:53, avg_cost:185.17558 batch_id:54, avg_cost:185.07697 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:55, avg_cost:164.40323 [train] batch_id:56, precision:[0.96523988], recall:[0.97376103], f1:[0.96948173] ./conf/test_data 30 30 [test] batch_id:56, precision:[0.96754024], recall:[0.9740625], f1:[0.97079042] cur_batch_id:56, last 5 batchs, time_cost:51.604197979 batch_id:56, avg_cost:174.60631 batch_id:57, avg_cost:149.17494 batch_id:58, avg_cost:166.43422 batch_id:59, avg_cost:159.94733 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:60, avg_cost:157.74432 [train] batch_id:61, precision:[0.96581643], recall:[0.97374619], f1:[0.9697651] ./conf/test_data 30 30 [test] batch_id:61, precision:[0.96754024], recall:[0.9740625], f1:[0.97079042] cur_batch_id:61, last 5 batchs, time_cost:51.6987860203 batch_id:61, avg_cost:158.1448 batch_id:62, avg_cost:152.98149 batch_id:63, avg_cost:154.99283 batch_id:64, avg_cost:158.43834 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:65, avg_cost:159.70451 [train] batch_id:66, precision:[0.96431844], recall:[0.97321879], f1:[0.96874817] ./conf/test_data 30 30 [test] batch_id:66, precision:[0.96754024], recall:[0.9740625], f1:[0.97079042] cur_batch_id:66, last 5 batchs, time_cost:50.2143101692 batch_id:66, avg_cost:153.51575 batch_id:67, avg_cost:147.52202 batch_id:68, avg_cost:158.41492 batch_id:69, avg_cost:159.33286 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:70, avg_cost:154.5351 [train] batch_id:71, precision:[0.96626306], recall:[0.97413173], f1:[0.97018144] ./conf/test_data 30 30 [test] batch_id:71, precision:[0.96754024], recall:[0.9740625], f1:[0.97079042] cur_batch_id:71, last 5 batchs, time_cost:51.4869060516 batch_id:71, avg_cost:145.62756 batch_id:72, avg_cost:156.73358 batch_id:73, avg_cost:151.06348 batch_id:74, avg_cost:171.03955 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:75, avg_cost:134.05615 [train] batch_id:76, precision:[0.96509046], recall:[0.97279443], f1:[0.96892714] ./conf/test_data 30 30 [test] batch_id:76, precision:[0.96814128], recall:[0.9740625], f1:[0.97109286] cur_batch_id:76, last 5 batchs, time_cost:49.93364501 batch_id:76, avg_cost:145.03711 batch_id:77, avg_cost:144.68495 batch_id:78, avg_cost:143.76091 batch_id:79, avg_cost:150.68063 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:80, avg_cost:135.34462 [train] batch_id:81, precision:[0.96495097], recall:[0.97287146], f1:[0.96889503] ./conf/test_data 30 30 [test] batch_id:81, precision:[0.97024234], recall:[0.97232143], f1:[0.97128077] cur_batch_id:81, last 5 batchs, time_cost:50.2912061214 batch_id:81, avg_cost:156.80211 batch_id:82, avg_cost:146.9292 batch_id:83, avg_cost:161.64514 batch_id:84, avg_cost:141.74074 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:85, avg_cost:158.67883 [train] batch_id:86, precision:[0.96722507], recall:[0.97289414], f1:[0.97005132] ./conf/test_data 30 30 [test] batch_id:86, precision:[0.9706656], recall:[0.97200893], f1:[0.9713368] cur_batch_id:86, last 5 batchs, time_cost:51.6507320404 batch_id:86, avg_cost:130.99527 batch_id:87, avg_cost:146.67703 batch_id:88, avg_cost:143.35762 batch_id:89, avg_cost:140.0539 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:90, avg_cost:143.0884 [train] batch_id:91, precision:[0.96648392], recall:[0.97115558], f1:[0.96881412] ./conf/test_data 30 30 [test] batch_id:91, precision:[0.97070887], recall:[0.97200893], f1:[0.97135847] cur_batch_id:91, last 5 batchs, time_cost:49.9286980629 batch_id:91, avg_cost:140.99673 batch_id:92, avg_cost:129.1026 batch_id:93, avg_cost:131.31128 batch_id:94, avg_cost:143.67125 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:95, avg_cost:132.11763 [train] batch_id:96, precision:[0.96799667], recall:[0.97201265], f1:[0.9700005] ./conf/test_data 30 30 [test] batch_id:96, precision:[0.9707154], recall:[0.97223214], f1:[0.97147318] cur_batch_id:96, last 5 batchs, time_cost:50.5543680191 batch_id:96, avg_cost:137.9944 batch_id:97, avg_cost:133.00458 batch_id:98, avg_cost:126.90342 batch_id:99, avg_cost:135.00247 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:100, avg_cost:141.71329 [train] batch_id:101, precision:[0.96818828], recall:[0.97181692], f1:[0.96999921] ./conf/test_data 30 30 [test] batch_id:101, precision:[0.97117962], recall:[0.9703125], f1:[0.97074587] cur_batch_id:101, last 5 batchs, time_cost:51.1195399761 batch_id:101, avg_cost:137.1569 batch_id:102, avg_cost:131.69618 batch_id:103, avg_cost:139.79424 batch_id:104, avg_cost:143.74799 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:105, avg_cost:125.914795 [train] batch_id:106, precision:[0.96661185], recall:[0.96970623], f1:[0.96815657] ./conf/test_data 30 30 [test] batch_id:106, precision:[0.9711723], recall:[0.9715625], f1:[0.97136736] cur_batch_id:106, last 5 batchs, time_cost:48.3584570885 batch_id:106, avg_cost:134.02904 batch_id:107, avg_cost:127.80123 batch_id:108, avg_cost:137.84654 batch_id:109, avg_cost:115.88592 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:110, avg_cost:140.92398 [train] batch_id:111, precision:[0.96921637], recall:[0.97199761], f1:[0.970605] ./conf/test_data 30 30 [test] batch_id:111, precision:[0.9712015], recall:[0.97107143], f1:[0.97113646] cur_batch_id:111, last 5 batchs, time_cost:52.2464079857 batch_id:111, avg_cost:136.65617 batch_id:112, avg_cost:127.37205 batch_id:113, avg_cost:151.91225 batch_id:114, avg_cost:121.55056 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:115, avg_cost:124.295654 [train] batch_id:116, precision:[0.96808163], recall:[0.97119173], f1:[0.96963419] ./conf/test_data 30 30 [test] batch_id:116, precision:[0.97121821], recall:[0.97165179], f1:[0.97143495] cur_batch_id:116, last 5 batchs, time_cost:50.9729940891 batch_id:116, avg_cost:122.452194 batch_id:117, avg_cost:120.91547 batch_id:118, avg_cost:130.54568 batch_id:119, avg_cost:117.72769 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:120, avg_cost:122.71166 [train] batch_id:121, precision:[0.96823034], recall:[0.96964038], f1:[0.96893485] ./conf/test_data 30 30 [test] batch_id:121, precision:[0.97106038], recall:[0.9721875], f1:[0.97162361] cur_batch_id:121, last 5 batchs, time_cost:49.8376870155 batch_id:121, avg_cost:115.46645 batch_id:122, avg_cost:114.38584 batch_id:123, avg_cost:126.516365 batch_id:124, avg_cost:129.04584 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:125, avg_cost:149.60551 [train] batch_id:126, precision:[0.97009008], recall:[0.97230857], f1:[0.97119806] ./conf/test_data 30 30 [test] batch_id:126, precision:[0.97109852], recall:[0.97200893], f1:[0.97155351] cur_batch_id:126, last 5 batchs, time_cost:52.4700520039 batch_id:126, avg_cost:124.91243 batch_id:127, avg_cost:128.5589 batch_id:128, avg_cost:112.76416 batch_id:129, avg_cost:109.08898 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:130, avg_cost:124.06509 [train] batch_id:131, precision:[0.96803193], recall:[0.97061081], f1:[0.96931966] ./conf/test_data 30 30 [test] batch_id:131, precision:[0.97076545], recall:[0.97098214], f1:[0.97087379] cur_batch_id:131, last 5 batchs, time_cost:50.0998620987 batch_id:131, avg_cost:112.28147 batch_id:132, avg_cost:114.07932 batch_id:133, avg_cost:139.74754 batch_id:134, avg_cost:135.13922 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:135, avg_cost:120.11338 [train] batch_id:136, precision:[0.96885375], recall:[0.97157377], f1:[0.97021186] ./conf/test_data 30 30 [test] batch_id:136, precision:[0.97096084], recall:[0.97174107], f1:[0.9713508] cur_batch_id:136, last 5 batchs, time_cost:51.129817009 batch_id:136, avg_cost:122.05933 batch_id:137, avg_cost:115.35651 batch_id:138, avg_cost:115.11463 batch_id:139, avg_cost:105.489624 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:140, avg_cost:117.21934 [train] batch_id:141, precision:[0.96932249], recall:[0.97112238], f1:[0.9702216] ./conf/test_data 30 30 [test] batch_id:141, precision:[0.97138776], recall:[0.97151786], f1:[0.9714528] cur_batch_id:141, last 5 batchs, time_cost:51.0500259399 batch_id:141, avg_cost:114.800575 batch_id:142, avg_cost:113.59616 batch_id:143, avg_cost:100.19049 batch_id:144, avg_cost:102.4298 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:145, avg_cost:112.11459 [train] batch_id:146, precision:[0.96859387], recall:[0.97019285], f1:[0.9693927] ./conf/test_data 30 30 [test] batch_id:146, precision:[0.97237371], recall:[0.97107143], f1:[0.97172214] cur_batch_id:146, last 5 batchs, time_cost:48.7377319336 batch_id:146, avg_cost:110.82226 batch_id:147, avg_cost:106.55925 batch_id:148, avg_cost:110.10191 batch_id:149, avg_cost:99.507744 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:150, avg_cost:102.3245 [train] batch_id:151, precision:[0.97106569], recall:[0.97203387], f1:[0.97154954] ./conf/test_data 30 30 [test] batch_id:151, precision:[0.97187626], recall:[0.97191964], f1:[0.97189795] cur_batch_id:151, last 5 batchs, time_cost:51.2562000751 batch_id:151, avg_cost:104.880905 batch_id:152, avg_cost:106.83875 batch_id:153, avg_cost:101.41776 batch_id:154, avg_cost:98.15506 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:155, avg_cost:98.97391 [train] batch_id:156, precision:[0.97058775], recall:[0.97162588], f1:[0.97110654] ./conf/test_data 30 30 [test] batch_id:156, precision:[0.97146806], recall:[0.9728125], f1:[0.97213981] cur_batch_id:156, last 5 batchs, time_cost:50.9179019928 batch_id:156, avg_cost:100.2788 batch_id:157, avg_cost:112.74755 batch_id:158, avg_cost:127.55057 batch_id:159, avg_cost:178.00343 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:160, avg_cost:114.01898 [train] batch_id:161, precision:[0.96986581], recall:[0.97043921], f1:[0.97015243] ./conf/test_data 30 30 [test] batch_id:161, precision:[0.97168213], recall:[0.97272321], f1:[0.97220239] cur_batch_id:161, last 5 batchs, time_cost:51.0771420002 batch_id:161, avg_cost:97.420876 batch_id:162, avg_cost:100.41996 batch_id:163, avg_cost:95.663635 batch_id:164, avg_cost:112.74008 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:165, avg_cost:106.313065 [train] batch_id:166, precision:[0.96976718], recall:[0.9701168], f1:[0.96994196] ./conf/test_data 30 30 [test] batch_id:166, precision:[0.97118297], recall:[0.9734375], f1:[0.97230893] cur_batch_id:166, last 5 batchs, time_cost:49.2930881977 batch_id:166, avg_cost:97.56251 batch_id:167, avg_cost:103.393196 batch_id:168, avg_cost:96.88614 batch_id:169, avg_cost:89.71152 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:170, avg_cost:102.11901 [train] batch_id:171, precision:[0.97313085], recall:[0.97309982], f1:[0.97311534] ./conf/test_data 30 30 [test] batch_id:171, precision:[0.97054243], recall:[0.97370536], f1:[0.97212132] cur_batch_id:171, last 5 batchs, time_cost:52.616476059 batch_id:171, avg_cost:92.19833 batch_id:172, avg_cost:102.2205 batch_id:173, avg_cost:104.06466 batch_id:174, avg_cost:101.22338 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:175, avg_cost:93.973656 [train] batch_id:176, precision:[0.97184042], recall:[0.97176511], f1:[0.97180276] ./conf/test_data 30 30 [test] batch_id:176, precision:[0.97131275], recall:[0.9734375], f1:[0.97237397] cur_batch_id:176, last 5 batchs, time_cost:50.8310658932 batch_id:176, avg_cost:92.968445 batch_id:177, avg_cost:93.97635 batch_id:178, avg_cost:100.07751 batch_id:179, avg_cost:83.721695 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:180, avg_cost:89.787674 [train] batch_id:181, precision:[0.97114933], recall:[0.97104074], f1:[0.97109503] ./conf/test_data 30 30 [test] batch_id:181, precision:[0.97122879], recall:[0.97352679], f1:[0.97237643] cur_batch_id:181, last 5 batchs, time_cost:50.3950440884 batch_id:181, avg_cost:93.18135 batch_id:182, avg_cost:95.58863 batch_id:183, avg_cost:91.57173 batch_id:184, avg_cost:86.61145 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:185, avg_cost:85.46055 [train] batch_id:186, precision:[0.97164658], recall:[0.97188791], f1:[0.97176723] ./conf/test_data 30 30 [test] batch_id:186, precision:[0.97114614], recall:[0.97366071], f1:[0.9724018] cur_batch_id:186, last 5 batchs, time_cost:49.5702321529 batch_id:186, avg_cost:97.30125 batch_id:187, avg_cost:89.375046 batch_id:188, avg_cost:89.99459 batch_id:189, avg_cost:77.53957 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:190, avg_cost:87.27847 [train] batch_id:191, precision:[0.97198255], recall:[0.97177417], f1:[0.97187835] ./conf/test_data 30 30 [test] batch_id:191, precision:[0.97119067], recall:[0.97370536], f1:[0.97244639] cur_batch_id:191, last 5 batchs, time_cost:50.2202010155 batch_id:191, avg_cost:81.548225 batch_id:192, avg_cost:83.719574 batch_id:193, avg_cost:94.56268 batch_id:194, avg_cost:74.29131 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:195, avg_cost:72.77294 [train] batch_id:196, precision:[0.9732816], recall:[0.97304565], f1:[0.97316361] ./conf/test_data 30 30 [test] batch_id:196, precision:[0.97278122], recall:[0.97325893], f1:[0.97302002] cur_batch_id:196, last 5 batchs, time_cost:49.7227330208 batch_id:196, avg_cost:82.67382 batch_id:197, avg_cost:104.598465 batch_id:198, avg_cost:95.791374 batch_id:199, avg_cost:96.14575 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:200, avg_cost:79.37234 [train] batch_id:201, precision:[0.97265693], recall:[0.97254282], f1:[0.97259987] ./conf/test_data 30 30 [test] batch_id:201, precision:[0.97399115], recall:[0.97299107], f1:[0.97349085] cur_batch_id:201, last 5 batchs, time_cost:51.0217759609 keep training! batch_id:201, avg_cost:70.041725 batch_id:202, avg_cost:80.34682 batch_id:203, avg_cost:72.5716 batch_id:204, avg_cost:94.58445 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:205, avg_cost:73.909325 [train] batch_id:206, precision:[0.97651271], recall:[0.97489114], f1:[0.97570125] ./conf/test_data 30 30 [test] batch_id:206, precision:[0.97347859], recall:[0.97334821], f1:[0.9734134] cur_batch_id:206, last 5 batchs, time_cost:53.1445858479 keep training! batch_id:206, avg_cost:74.01509 batch_id:207, avg_cost:82.40077 batch_id:208, avg_cost:90.30901 batch_id:209, avg_cost:78.05412 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:210, avg_cost:72.25984 [train] batch_id:211, precision:[0.97285755], recall:[0.97231], f1:[0.9725837] ./conf/test_data 30 30 [test] batch_id:211, precision:[0.97261862], recall:[0.97366071], f1:[0.97313939] cur_batch_id:211, last 5 batchs, time_cost:48.4145109653 keep training! batch_id:211, avg_cost:74.81678 batch_id:212, avg_cost:91.855804 batch_id:213, avg_cost:77.84203 batch_id:214, avg_cost:85.14333 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:215, avg_cost:71.034805 [train] batch_id:216, precision:[0.9743391], recall:[0.97356235], f1:[0.97395057] ./conf/test_data 30 30 [test] batch_id:216, precision:[0.9710203], recall:[0.97379464], f1:[0.97240549] cur_batch_id:216, last 5 batchs, time_cost:50.65863204 keep training! batch_id:216, avg_cost:80.846245 batch_id:217, avg_cost:76.78863 batch_id:218, avg_cost:66.29149 batch_id:219, avg_cost:63.70071 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:220, avg_cost:68.06963 [train] batch_id:221, precision:[0.976486], recall:[0.97615456], f1:[0.97632025] ./conf/test_data 30 30 [test] batch_id:221, precision:[0.97463833], recall:[0.97446429], f1:[0.9745513] cur_batch_id:221, last 5 batchs, time_cost:52.2119140625 keep training! batch_id:221, avg_cost:63.907528 batch_id:222, avg_cost:73.36685 batch_id:223, avg_cost:77.08213 batch_id:224, avg_cost:68.570915 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:225, avg_cost:67.08461 [train] batch_id:226, precision:[0.97681837], recall:[0.97590501], f1:[0.97636148] ./conf/test_data 30 30 [test] batch_id:226, precision:[0.97424337], recall:[0.97433036], f1:[0.97428686] cur_batch_id:226, last 5 batchs, time_cost:50.9714829922 keep training! batch_id:226, avg_cost:76.94973 batch_id:227, avg_cost:73.63173 batch_id:228, avg_cost:61.798912 batch_id:229, avg_cost:64.09221 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:230, avg_cost:65.42129 [train] batch_id:231, precision:[0.97723425], recall:[0.9767312], f1:[0.97698266] ./conf/test_data 30 30 [test] batch_id:231, precision:[0.97314538], recall:[0.97388393], f1:[0.97351451] cur_batch_id:231, last 5 batchs, time_cost:51.3494369984 keep training! batch_id:231, avg_cost:64.97014 batch_id:232, avg_cost:81.55357 batch_id:233, avg_cost:75.64045 batch_id:234, avg_cost:63.053165 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:235, avg_cost:69.62247 [train] batch_id:236, precision:[0.97554578], recall:[0.97531826], f1:[0.97543201] ./conf/test_data 30 30 [test] batch_id:236, precision:[0.97543656], recall:[0.97504464], f1:[0.97524056] cur_batch_id:236, last 5 batchs, time_cost:48.9641740322 keep training! batch_id:236, avg_cost:67.11469 batch_id:237, avg_cost:64.42642 batch_id:238, avg_cost:61.00353 batch_id:239, avg_cost:66.06465 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:240, avg_cost:61.14476 [train] batch_id:241, precision:[0.9784417], recall:[0.97759675], f1:[0.97801905] ./conf/test_data 30 30 [test] batch_id:241, precision:[0.97680965], recall:[0.9759375], f1:[0.97637338] cur_batch_id:241, last 5 batchs, time_cost:51.0931949615 keep training! batch_id:241, avg_cost:61.499577 batch_id:242, avg_cost:71.174904 batch_id:243, avg_cost:60.02257 batch_id:244, avg_cost:62.21637 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:245, avg_cost:56.790966 [train] batch_id:246, precision:[0.97900671], recall:[0.97820738], f1:[0.97860688] ./conf/test_data 30 30 [test] batch_id:246, precision:[0.97849414], recall:[0.97700893], f1:[0.97775097] cur_batch_id:246, last 5 batchs, time_cost:51.3685441017 keep training! batch_id:246, avg_cost:57.07731 batch_id:247, avg_cost:70.730675 batch_id:248, avg_cost:59.94163 batch_id:249, avg_cost:74.605675 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:250, avg_cost:62.61334 [train] batch_id:251, precision:[0.97776513], recall:[0.97670425], f1:[0.9772344] ./conf/test_data 30 30 [test] batch_id:251, precision:[0.97676601], recall:[0.9759375], f1:[0.97635158] cur_batch_id:251, last 5 batchs, time_cost:48.6456270218 keep training! batch_id:251, avg_cost:61.52506 batch_id:252, avg_cost:63.258602 batch_id:253, avg_cost:57.93417 batch_id:254, avg_cost:51.162758 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:255, avg_cost:60.52709 [train] batch_id:256, precision:[0.98021775], recall:[0.97919184], f1:[0.97970453] ./conf/test_data 30 30 [test] batch_id:256, precision:[0.9759289], recall:[0.97558036], f1:[0.9757546] cur_batch_id:256, last 5 batchs, time_cost:51.7606141567 keep training! batch_id:256, avg_cost:65.71584 batch_id:257, avg_cost:54.188076 batch_id:258, avg_cost:57.799706 batch_id:259, avg_cost:59.274868 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:260, avg_cost:55.659683 [train] batch_id:261, precision:[0.97874907], recall:[0.97760939], f1:[0.9781789] ./conf/test_data 30 30 [test] batch_id:261, precision:[0.97952524], recall:[0.97816964], f1:[0.97884697] cur_batch_id:261, last 5 batchs, time_cost:48.4133729935 keep training! batch_id:261, avg_cost:61.34455 batch_id:262, avg_cost:54.55279 batch_id:263, avg_cost:53.251198 batch_id:264, avg_cost:52.453934 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:265, avg_cost:51.70429 [train] batch_id:266, precision:[0.98173764], recall:[0.98035553], f1:[0.9810461] ./conf/test_data 30 30 [test] batch_id:266, precision:[0.98175068], recall:[0.97986607], f1:[0.98080747] cur_batch_id:266, last 5 batchs, time_cost:51.9654259682 keep training! batch_id:266, avg_cost:57.183014 batch_id:267, avg_cost:55.00469 batch_id:268, avg_cost:54.953625 batch_id:269, avg_cost:51.436985 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:270, avg_cost:54.90588 [train] batch_id:271, precision:[0.9812169], recall:[0.97978463], f1:[0.98050024] ./conf/test_data 30 30 [test] batch_id:271, precision:[0.98254955], recall:[0.9803125], f1:[0.98142975] cur_batch_id:271, last 5 batchs, time_cost:51.2433509827 keep training! batch_id:271, avg_cost:47.92301 batch_id:272, avg_cost:55.750507 batch_id:273, avg_cost:54.789936 batch_id:274, avg_cost:65.23421 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:275, avg_cost:65.01375 [train] batch_id:276, precision:[0.97947495], recall:[0.97887796], f1:[0.97917637] ./conf/test_data 30 30 [test] batch_id:276, precision:[0.98245143], recall:[0.97973214], f1:[0.9810899] cur_batch_id:276, last 5 batchs, time_cost:52.2189991474 keep training! batch_id:276, avg_cost:53.591743 batch_id:277, avg_cost:61.194992 batch_id:278, avg_cost:52.796253 batch_id:279, avg_cost:46.783005 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:280, avg_cost:53.962833 [train] batch_id:281, precision:[0.97897596], recall:[0.97815053], f1:[0.97856307] ./conf/test_data 30 30 [test] batch_id:281, precision:[0.98407373], recall:[0.98200893], f1:[0.98304024] cur_batch_id:281, last 5 batchs, time_cost:50.7135751247 keep training! batch_id:281, avg_cost:53.274166 batch_id:282, avg_cost:49.798965 batch_id:283, avg_cost:52.68145 batch_id:284, avg_cost:58.564022 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:285, avg_cost:54.299232 [train] batch_id:286, precision:[0.98172511], recall:[0.98064591], f1:[0.98118521] ./conf/test_data 30 30 [test] batch_id:286, precision:[0.98394023], recall:[0.98191964], f1:[0.9829289] cur_batch_id:286, last 5 batchs, time_cost:50.3793311119 keep training! batch_id:286, avg_cost:51.056595 batch_id:287, avg_cost:44.68227 batch_id:288, avg_cost:56.001476 batch_id:289, avg_cost:49.332542 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:290, avg_cost:53.87721 [train] batch_id:291, precision:[0.98187895], recall:[0.98068115], f1:[0.98127968] ./conf/test_data 30 30 [test] batch_id:291, precision:[0.98433303], recall:[0.98169643], f1:[0.98301296] cur_batch_id:291, last 5 batchs, time_cost:50.8300189972 keep training! batch_id:291, avg_cost:56.944603 batch_id:292, avg_cost:44.062576 batch_id:293, avg_cost:51.290817 batch_id:294, avg_cost:64.834724 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:295, avg_cost:47.997604 [train] batch_id:296, precision:[0.98194494], recall:[0.98068602], f1:[0.98131508] ./conf/test_data 30 30 [test] batch_id:296, precision:[0.98291669], recall:[0.98120536], f1:[0.98206028] cur_batch_id:296, last 5 batchs, time_cost:50.691229105 keep training! batch_id:296, avg_cost:44.41853 batch_id:297, avg_cost:48.93032 batch_id:298, avg_cost:52.45603 batch_id:299, avg_cost:50.670597 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:300, avg_cost:46.049965 [train] batch_id:301, precision:[0.98374083], recall:[0.98237238], f1:[0.98305613] ./conf/test_data 30 30 [test] batch_id:301, precision:[0.98425267], recall:[0.9821875], f1:[0.983219] cur_batch_id:301, last 5 batchs, time_cost:50.6825911999 keep training! batch_id:301, avg_cost:51.308037 batch_id:302, avg_cost:48.995895 batch_id:303, avg_cost:43.02427 batch_id:304, avg_cost:54.198086 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:305, avg_cost:53.531948 [train] batch_id:306, precision:[0.98389571], recall:[0.98235589], f1:[0.9831252] ./conf/test_data 30 30 [test] batch_id:306, precision:[0.97818702], recall:[0.97696429], f1:[0.97757527] cur_batch_id:306, last 5 batchs, time_cost:50.7405459881 keep training! batch_id:306, avg_cost:42.6889 batch_id:307, avg_cost:51.86658 batch_id:308, avg_cost:50.877007 batch_id:309, avg_cost:61.974148 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:310, avg_cost:60.58248 [train] batch_id:311, precision:[0.98259469], recall:[0.98133029], f1:[0.98196208] ./conf/test_data 30 30 [test] batch_id:311, precision:[0.97962466], recall:[0.97875], f1:[0.97918714] cur_batch_id:311, last 5 batchs, time_cost:51.0374088287 keep training! batch_id:311, avg_cost:47.560863 batch_id:312, avg_cost:43.313446 batch_id:313, avg_cost:41.627045 batch_id:314, avg_cost:44.7456 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:315, avg_cost:43.51214 [train] batch_id:316, precision:[0.98516299], recall:[0.98373617], f1:[0.98444906] ./conf/test_data 30 30 [test] batch_id:316, precision:[0.98461745], recall:[0.98299107], f1:[0.98380359] cur_batch_id:316, last 5 batchs, time_cost:50.0787789822 keep training! batch_id:316, avg_cost:40.706238 batch_id:317, avg_cost:47.14232 batch_id:318, avg_cost:44.897106 batch_id:319, avg_cost:61.547585 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:320, avg_cost:48.35003 [train] batch_id:321, precision:[0.98515802], recall:[0.98370731], f1:[0.98443213] ./conf/test_data 30 30 [test] batch_id:321, precision:[0.98416745], recall:[0.98236607], f1:[0.98326594] cur_batch_id:321, last 5 batchs, time_cost:50.8340480328 keep training! batch_id:321, avg_cost:40.430782 batch_id:322, avg_cost:43.17542 batch_id:323, avg_cost:45.145824 batch_id:324, avg_cost:54.395714 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:325, avg_cost:45.4014 [train] batch_id:326, precision:[0.98556279], recall:[0.98423829], f1:[0.98490009] ./conf/test_data 30 30 [test] batch_id:326, precision:[0.98582036], recall:[0.98388393], f1:[0.98485119] cur_batch_id:326, last 5 batchs, time_cost:50.9573531151 keep training! batch_id:326, avg_cost:45.085224 batch_id:327, avg_cost:48.11266 batch_id:328, avg_cost:40.294056 batch_id:329, avg_cost:38.78737 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:330, avg_cost:43.61987 [train] batch_id:331, precision:[0.98611158], recall:[0.98438739], f1:[0.98524873] ./conf/test_data 30 30 [test] batch_id:331, precision:[0.98475023], recall:[0.98303571], f1:[0.98389223] cur_batch_id:331, last 5 batchs, time_cost:49.736978054 keep training! batch_id:331, avg_cost:45.0493 batch_id:332, avg_cost:43.88135 batch_id:333, avg_cost:47.24461 batch_id:334, avg_cost:44.93334 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:335, avg_cost:45.452755 [train] batch_id:336, precision:[0.98587331], recall:[0.98439906], f1:[0.98513563] ./conf/test_data 30 30 [test] batch_id:336, precision:[0.98225064], recall:[0.98080357], f1:[0.98152657] cur_batch_id:336, last 5 batchs, time_cost:50.764786005 keep training! batch_id:336, avg_cost:47.70652 batch_id:337, avg_cost:44.479664 batch_id:338, avg_cost:36.714207 batch_id:339, avg_cost:49.690258 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:340, avg_cost:48.02865 [train] batch_id:341, precision:[0.98539388], recall:[0.98410093], f1:[0.98474698] ./conf/test_data 30 30 [test] batch_id:341, precision:[0.98376928], recall:[0.98223214], f1:[0.98300011] cur_batch_id:341, last 5 batchs, time_cost:51.245882988 keep training! batch_id:341, avg_cost:44.377037 batch_id:342, avg_cost:46.325974 batch_id:343, avg_cost:37.22873 batch_id:344, avg_cost:37.797596 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:345, avg_cost:45.14303 [train] batch_id:346, precision:[0.98638468], recall:[0.98475999], f1:[0.98557166] ./conf/test_data 30 30 [test] batch_id:346, precision:[0.98784249], recall:[0.98665179], f1:[0.98724678] cur_batch_id:346, last 5 batchs, time_cost:50.2956669331 keep training! batch_id:346, avg_cost:41.582157 batch_id:347, avg_cost:34.989323 batch_id:348, avg_cost:42.772236 batch_id:349, avg_cost:46.76128 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:350, avg_cost:46.747944 [train] batch_id:351, precision:[0.98719949], recall:[0.98609273], f1:[0.9866458] ./conf/test_data 30 30 [test] batch_id:351, precision:[0.98779506], recall:[0.98638393], f1:[0.98708899] cur_batch_id:351, last 5 batchs, time_cost:51.0594229698 keep training! batch_id:351, avg_cost:38.097233 batch_id:352, avg_cost:43.111374 batch_id:353, avg_cost:35.95972 batch_id:354, avg_cost:44.24218 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:355, avg_cost:40.97456 [train] batch_id:356, precision:[0.98797111], recall:[0.98680311], f1:[0.98738676] ./conf/test_data 30 30 [test] batch_id:356, precision:[0.98819954], recall:[0.98696429], f1:[0.98758152] cur_batch_id:356, last 5 batchs, time_cost:51.1623930931 keep training! batch_id:356, avg_cost:43.85615 batch_id:357, avg_cost:40.60841 batch_id:358, avg_cost:37.544342 batch_id:359, avg_cost:41.98435 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:360, avg_cost:38.193607 [train] batch_id:361, precision:[0.98791552], recall:[0.98688714], f1:[0.98740106] ./conf/test_data 30 30 [test] batch_id:361, precision:[0.98792972], recall:[0.9865625], f1:[0.98724564] cur_batch_id:361, last 5 batchs, time_cost:51.6551270485 keep training! batch_id:361, avg_cost:50.175747 batch_id:362, avg_cost:36.332317 batch_id:363, avg_cost:45.856964 batch_id:364, avg_cost:32.778225 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:365, avg_cost:38.845375 [train] batch_id:366, precision:[0.98733763], recall:[0.98618623], f1:[0.98676159] ./conf/test_data 30 30 [test] batch_id:366, precision:[0.98811917], recall:[0.98763393], f1:[0.98787649] cur_batch_id:366, last 5 batchs, time_cost:49.9659819603 keep training! batch_id:366, avg_cost:41.152027 batch_id:367, avg_cost:35.60906 batch_id:368, avg_cost:42.067642 batch_id:369, avg_cost:42.257877 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:370, avg_cost:47.85316 [train] batch_id:371, precision:[0.9875173], recall:[0.98686199], f1:[0.98718954] ./conf/test_data 30 30 [test] batch_id:371, precision:[0.98859673], recall:[0.98691964], f1:[0.98775747] cur_batch_id:371, last 5 batchs, time_cost:50.6341798306 keep training! batch_id:371, avg_cost:41.460426 batch_id:372, avg_cost:32.35874 batch_id:373, avg_cost:41.534256 batch_id:374, avg_cost:35.06027 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:375, avg_cost:40.156292 [train] batch_id:376, precision:[0.98813581], recall:[0.98720149], f1:[0.98766843] ./conf/test_data 30 30 [test] batch_id:376, precision:[0.98896385], recall:[0.988125], f1:[0.98854425] cur_batch_id:376, last 5 batchs, time_cost:50.5755829811 keep training! batch_id:376, avg_cost:36.86848 batch_id:377, avg_cost:41.084625 batch_id:378, avg_cost:38.896915 batch_id:379, avg_cost:32.955776 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:380, avg_cost:43.745567 [train] batch_id:381, precision:[0.98876174], recall:[0.98819798], f1:[0.98847978] ./conf/test_data 30 30 [test] batch_id:381, precision:[0.98968243], recall:[0.98919643], f1:[0.98943937] cur_batch_id:381, last 5 batchs, time_cost:50.7046711445 keep training! batch_id:381, avg_cost:38.309155 batch_id:382, avg_cost:38.157665 batch_id:383, avg_cost:38.212566 batch_id:384, avg_cost:26.830517 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:385, avg_cost:40.9202 [train] batch_id:386, precision:[0.98890276], recall:[0.98830514], f1:[0.98860386] ./conf/test_data 30 30 [test] batch_id:386, precision:[0.98914791], recall:[0.98879464], f1:[0.98897124] cur_batch_id:386, last 5 batchs, time_cost:50.9420449734 keep training! batch_id:386, avg_cost:33.534256 batch_id:387, avg_cost:43.4758 batch_id:388, avg_cost:38.50392 batch_id:389, avg_cost:30.315224 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:390, avg_cost:36.973248 [train] batch_id:391, precision:[0.98947386], recall:[0.98868781], f1:[0.98908068] ./conf/test_data 30 30 [test] batch_id:391, precision:[0.98821271], recall:[0.98808036], f1:[0.98814653] cur_batch_id:391, last 5 batchs, time_cost:51.7508370876 keep training! batch_id:391, avg_cost:47.56221 batch_id:392, avg_cost:41.220043 batch_id:393, avg_cost:28.342375 batch_id:394, avg_cost:27.79406 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:395, avg_cost:35.466564 [train] batch_id:396, precision:[0.98851053], recall:[0.98809402], f1:[0.98830223] ./conf/test_data 30 30 [test] batch_id:396, precision:[0.98990801], recall:[0.98964286], f1:[0.98977542] cur_batch_id:396, last 5 batchs, time_cost:49.3899149895 keep training! batch_id:396, avg_cost:32.71554 batch_id:397, avg_cost:44.95441 batch_id:398, avg_cost:33.443756 batch_id:399, avg_cost:37.66003 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:400, avg_cost:34.193283 [train] batch_id:401, precision:[0.99020757], recall:[0.98990548], f1:[0.9900565] ./conf/test_data 30 30 [test] batch_id:401, precision:[0.98883779], recall:[0.98870536], f1:[0.98877157] cur_batch_id:401, last 5 batchs, time_cost:52.4226970673 keep training! batch_id:401, avg_cost:39.549805 batch_id:402, avg_cost:39.766598 batch_id:403, avg_cost:34.444645 batch_id:404, avg_cost:38.828686 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:405, avg_cost:30.874933 [train] batch_id:406, precision:[0.98865934], recall:[0.98805197], f1:[0.98835556] ./conf/test_data 30 30 [test] batch_id:406, precision:[0.98906103], recall:[0.98892857], f1:[0.9889948] cur_batch_id:406, last 5 batchs, time_cost:49.4613571167 keep training! batch_id:406, avg_cost:33.612587 batch_id:407, avg_cost:40.470768 batch_id:408, avg_cost:37.863422 batch_id:409, avg_cost:43.23653 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:410, avg_cost:33.66039 [train] batch_id:411, precision:[0.98973725], recall:[0.98949907], f1:[0.98961815] ./conf/test_data 30 30 [test] batch_id:411, precision:[0.98830148], recall:[0.988125], f1:[0.98821323] cur_batch_id:411, last 5 batchs, time_cost:51.1512610912 keep training! batch_id:411, avg_cost:37.62359 batch_id:412, avg_cost:36.291214 batch_id:413, avg_cost:40.340588 batch_id:414, avg_cost:31.644884 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:415, avg_cost:49.99988 [train] batch_id:416, precision:[0.98807976], recall:[0.98748376], f1:[0.98778167] ./conf/test_data 30 30 [test] batch_id:416, precision:[0.98999598], recall:[0.98959821], f1:[0.98979706] cur_batch_id:416, last 5 batchs, time_cost:48.7105898857 keep training! batch_id:416, avg_cost:32.06892 batch_id:417, avg_cost:26.349747 batch_id:418, avg_cost:39.725 batch_id:419, avg_cost:39.072723 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:420, avg_cost:41.310387 [train] batch_id:421, precision:[0.98976066], recall:[0.98962432], f1:[0.98969248] ./conf/test_data 30 30 [test] batch_id:421, precision:[0.99066589], recall:[0.99026786], f1:[0.99046683] cur_batch_id:421, last 5 batchs, time_cost:51.1821079254 keep training! batch_id:421, avg_cost:34.135975 batch_id:422, avg_cost:34.22553 batch_id:423, avg_cost:34.78476 batch_id:424, avg_cost:31.875519 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:425, avg_cost:40.0022 [train] batch_id:426, precision:[0.98969095], recall:[0.98918655], f1:[0.98943869] ./conf/test_data 30 30 [test] batch_id:426, precision:[0.99062374], recall:[0.99049107], f1:[0.9905574] cur_batch_id:426, last 5 batchs, time_cost:50.9367911816 keep training! batch_id:426, avg_cost:29.74993 batch_id:427, avg_cost:30.41559 batch_id:428, avg_cost:38.824722 batch_id:429, avg_cost:31.627748 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:430, avg_cost:37.261154 [train] batch_id:431, precision:[0.99007369], recall:[0.98981357], f1:[0.98994361] ./conf/test_data 30 30 [test] batch_id:431, precision:[0.99017462], recall:[0.98977679], f1:[0.98997566] cur_batch_id:431, last 5 batchs, time_cost:51.6790151596 keep training! batch_id:431, avg_cost:42.337074 batch_id:432, avg_cost:41.98433 batch_id:433, avg_cost:39.49742 batch_id:434, avg_cost:37.072147 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:435, avg_cost:33.77734 [train] batch_id:436, precision:[0.98836971], recall:[0.98802001], f1:[0.98819483] ./conf/test_data 30 30 [test] batch_id:436, precision:[0.9898629], recall:[0.98955357], f1:[0.98970821] cur_batch_id:436, last 5 batchs, time_cost:51.2071831226 keep training! batch_id:436, avg_cost:33.107365 batch_id:437, avg_cost:35.884914 batch_id:438, avg_cost:41.01536 batch_id:439, avg_cost:29.294573 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:440, avg_cost:42.625034 [train] batch_id:441, precision:[0.98932864], recall:[0.98921378], f1:[0.98927121] ./conf/test_data 30 30 [test] batch_id:441, precision:[0.9912927], recall:[0.99107143], f1:[0.99118205] cur_batch_id:441, last 5 batchs, time_cost:50.95480299 keep training! batch_id:441, avg_cost:36.303173 batch_id:442, avg_cost:30.168491 batch_id:443, avg_cost:27.789001 batch_id:444, avg_cost:27.282738 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:445, avg_cost:31.000183 [train] batch_id:446, precision:[0.99099756], recall:[0.99068707], f1:[0.99084229] ./conf/test_data 30 30 [test] batch_id:446, precision:[0.99022147], recall:[0.99004464], f1:[0.99013305] cur_batch_id:446, last 5 batchs, time_cost:52.1154391766 keep training! batch_id:446, avg_cost:32.818775 batch_id:447, avg_cost:38.615074 batch_id:448, avg_cost:32.944458 batch_id:449, avg_cost:48.573452 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:450, avg_cost:34.45762 [train] batch_id:451, precision:[0.98870016], recall:[0.9882963], f1:[0.98849819] ./conf/test_data 30 30 [test] batch_id:451, precision:[0.98897272], recall:[0.98892857], f1:[0.98895065] cur_batch_id:451, last 5 batchs, time_cost:48.7332491875 keep training! batch_id:451, avg_cost:32.20121 batch_id:452, avg_cost:33.986874 batch_id:453, avg_cost:36.012856 batch_id:454, avg_cost:25.402424 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:455, avg_cost:30.215614 [train] batch_id:456, precision:[0.99063816], recall:[0.99049122], f1:[0.99056469] ./conf/test_data 30 30 [test] batch_id:456, precision:[0.99080193], recall:[0.990625], f1:[0.99071346] cur_batch_id:456, last 5 batchs, time_cost:53.9731490612 keep training! batch_id:456, avg_cost:31.395864 batch_id:457, avg_cost:36.116703 batch_id:458, avg_cost:32.952183 batch_id:459, avg_cost:34.82959 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:460, avg_cost:26.333735 [train] batch_id:461, precision:[0.98996812], recall:[0.98957948], f1:[0.98977376] ./conf/test_data 30 30 [test] batch_id:461, precision:[0.98986517], recall:[0.98977679], f1:[0.98982097] cur_batch_id:461, last 5 batchs, time_cost:49.7182278633 keep training! batch_id:461, avg_cost:40.1506 batch_id:462, avg_cost:31.79365 batch_id:463, avg_cost:28.069094 batch_id:464, avg_cost:32.977528 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:465, avg_cost:36.455486 [train] batch_id:466, precision:[0.9896434], recall:[0.98954454], f1:[0.98959397] ./conf/test_data 30 30 [test] batch_id:466, precision:[0.99151634], recall:[0.99133929], f1:[0.99142781] cur_batch_id:466, last 5 batchs, time_cost:50.8688750267 keep training! batch_id:466, avg_cost:28.475576 batch_id:467, avg_cost:27.404346 batch_id:468, avg_cost:37.63287 batch_id:469, avg_cost:25.1958 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:470, avg_cost:32.538322 [train] batch_id:471, precision:[0.99087529], recall:[0.99061927], f1:[0.99074726] ./conf/test_data 30 30 [test] batch_id:471, precision:[0.99178425], recall:[0.99160714], f1:[0.99169569] cur_batch_id:471, last 5 batchs, time_cost:53.678082943 keep training! batch_id:471, avg_cost:35.66035 batch_id:472, avg_cost:25.908989 batch_id:473, avg_cost:32.06723 batch_id:474, avg_cost:33.74988 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:475, avg_cost:27.138376 [train] batch_id:476, precision:[0.9906248], recall:[0.99036894], f1:[0.99049686] ./conf/test_data 30 30 [test] batch_id:476, precision:[0.99084699], recall:[0.99071429], f1:[0.99078063] cur_batch_id:476, last 5 batchs, time_cost:48.9769129753 keep training! batch_id:476, avg_cost:30.191929 batch_id:477, avg_cost:38.55315 batch_id:478, avg_cost:31.387794 batch_id:479, avg_cost:26.826637 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:480, avg_cost:27.574995 [train] batch_id:481, precision:[0.99043281], recall:[0.99020083], f1:[0.99031681] ./conf/test_data 30 30 [test] batch_id:481, precision:[0.9905791], recall:[0.99044643], f1:[0.99051276] cur_batch_id:481, last 5 batchs, time_cost:51.6501629353 keep training! batch_id:481, avg_cost:33.391926 batch_id:482, avg_cost:31.64903 batch_id:483, avg_cost:29.620674 batch_id:484, avg_cost:45.26167 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:485, avg_cost:34.35822 [train] batch_id:486, precision:[0.98927241], recall:[0.98888863], f1:[0.98908048] ./conf/test_data 30 30 [test] batch_id:486, precision:[0.98897321], recall:[0.98897321], f1:[0.98897321] cur_batch_id:486, last 5 batchs, time_cost:48.7119679451 keep training! batch_id:486, avg_cost:31.437187 batch_id:487, avg_cost:32.536137 batch_id:488, avg_cost:31.476694 batch_id:489, avg_cost:23.243135 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:490, avg_cost:31.147678 [train] batch_id:491, precision:[0.98997042], recall:[0.98988153], f1:[0.98992597] ./conf/test_data 30 30 [test] batch_id:491, precision:[0.99133813], recall:[0.99120536], f1:[0.99127174] cur_batch_id:491, last 5 batchs, time_cost:50.5393738747 keep training! batch_id:491, avg_cost:29.408503 batch_id:492, avg_cost:25.594505 batch_id:493, avg_cost:26.78594 batch_id:494, avg_cost:43.09981 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:495, avg_cost:38.673237 [train] batch_id:496, precision:[0.99085414], recall:[0.99046145], f1:[0.99065776] ./conf/test_data 30 30 [test] batch_id:496, precision:[0.98696429], recall:[0.98696429], f1:[0.98696429] cur_batch_id:496, last 5 batchs, time_cost:51.5540299416 keep training! batch_id:496, avg_cost:40.310516 batch_id:497, avg_cost:29.799105 batch_id:498, avg_cost:25.337786 batch_id:499, avg_cost:29.421186 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:500, avg_cost:25.580124 [train] batch_id:501, precision:[0.99029638], recall:[0.99015761], f1:[0.99022699] ./conf/test_data 30 30 [test] batch_id:501, precision:[0.99098093], recall:[0.99084821], f1:[0.99091457] cur_batch_id:501, last 5 batchs, time_cost:50.5659809113 keep training! batch_id:501, avg_cost:28.946619 batch_id:502, avg_cost:22.860413 batch_id:503, avg_cost:38.31851 batch_id:504, avg_cost:35.904877 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:505, avg_cost:32.459976 [train] batch_id:506, precision:[0.99018168], recall:[0.98993022], f1:[0.99005593] ./conf/test_data 30 30 [test] batch_id:506, precision:[0.99062374], recall:[0.99049107], f1:[0.9905574] cur_batch_id:506, last 5 batchs, time_cost:51.0606851578 keep training! batch_id:506, avg_cost:31.050083 batch_id:507, avg_cost:29.867037 batch_id:508, avg_cost:24.495726 batch_id:509, avg_cost:25.959206 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:510, avg_cost:26.413464 [train] batch_id:511, precision:[0.99093781], recall:[0.99072952], f1:[0.99083366] ./conf/test_data 30 30 [test] batch_id:511, precision:[0.99115874], recall:[0.9909375], f1:[0.99104811] cur_batch_id:511, last 5 batchs, time_cost:50.9872350693 keep training! batch_id:511, avg_cost:30.61478 batch_id:512, avg_cost:30.251797 batch_id:513, avg_cost:29.41216 batch_id:514, avg_cost:26.050142 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:515, avg_cost:22.854948 [train] batch_id:516, precision:[0.99083359], recall:[0.9905782], f1:[0.99070588] ./conf/test_data 30 30 [test] batch_id:516, precision:[0.9905791], recall:[0.99044643], f1:[0.99051276] cur_batch_id:516, last 5 batchs, time_cost:51.4235868454 keep training! batch_id:516, avg_cost:31.4004 batch_id:517, avg_cost:31.836365 batch_id:518, avg_cost:28.30924 batch_id:519, avg_cost:40.40398 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:520, avg_cost:29.510872 [train] batch_id:521, precision:[0.98954964], recall:[0.9893321], f1:[0.98944086] ./conf/test_data 30 30 [test] batch_id:521, precision:[0.98977587], recall:[0.9896875], f1:[0.98973168] cur_batch_id:521, last 5 batchs, time_cost:49.0402829647 keep training! batch_id:521, avg_cost:30.388575 batch_id:522, avg_cost:26.060581 batch_id:523, avg_cost:30.135473 batch_id:524, avg_cost:30.562244 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:525, avg_cost:31.8513 [train] batch_id:526, precision:[0.99083204], recall:[0.99067014], f1:[0.99075108] ./conf/test_data 30 30 [test] batch_id:526, precision:[0.99089082], recall:[0.99066964], f1:[0.99078022] cur_batch_id:526, last 5 batchs, time_cost:51.7804219723 keep training! batch_id:526, avg_cost:29.4428 batch_id:527, avg_cost:24.459627 batch_id:528, avg_cost:26.473133 batch_id:529, avg_cost:28.943415 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:530, avg_cost:31.443005 [train] batch_id:531, precision:[0.99072573], recall:[0.99044538], f1:[0.99058554] ./conf/test_data 30 30 [test] batch_id:531, precision:[0.99098053], recall:[0.99080357], f1:[0.99089204] cur_batch_id:531, last 5 batchs, time_cost:51.1053171158 keep training! batch_id:531, avg_cost:26.972666 batch_id:532, avg_cost:32.081726 batch_id:533, avg_cost:27.920721 batch_id:534, avg_cost:28.163176 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:535, avg_cost:27.672426 [train] batch_id:536, precision:[0.99087445], recall:[0.9904417], f1:[0.99065803] ./conf/test_data 30 30 [test] batch_id:536, precision:[0.99089164], recall:[0.99075893], f1:[0.99082528] cur_batch_id:536, last 5 batchs, time_cost:51.1027581692 keep training! batch_id:536, avg_cost:32.271538 batch_id:537, avg_cost:21.90371 batch_id:538, avg_cost:31.39706 batch_id:539, avg_cost:27.576984 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:540, avg_cost:27.665623 [train] batch_id:541, precision:[0.99050696], recall:[0.99032007], f1:[0.99041351] ./conf/test_data 30 30 [test] batch_id:541, precision:[0.99245367], recall:[0.99223214], f1:[0.9923429] cur_batch_id:541, last 5 batchs, time_cost:49.6584150791 keep training! batch_id:541, avg_cost:27.263958 batch_id:542, avg_cost:21.717016 batch_id:543, avg_cost:29.60586 batch_id:544, avg_cost:32.841625 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:545, avg_cost:31.959116 [train] batch_id:546, precision:[0.99140218], recall:[0.99127313], f1:[0.99133765] ./conf/test_data 30 30 [test] batch_id:546, precision:[0.98611545], recall:[0.98607143], f1:[0.98609344] cur_batch_id:546, last 5 batchs, time_cost:51.9816951752 keep training! batch_id:546, avg_cost:38.39094 batch_id:547, avg_cost:22.850645 batch_id:548, avg_cost:30.674456 batch_id:549, avg_cost:32.83472 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:550, avg_cost:29.075424 [train] batch_id:551, precision:[0.98969031], recall:[0.9894201], f1:[0.98955519] ./conf/test_data 30 30 [test] batch_id:551, precision:[0.9921411], recall:[0.99191964], f1:[0.99203036] cur_batch_id:551, last 5 batchs, time_cost:49.6433110237 keep training! batch_id:551, avg_cost:25.100933 batch_id:552, avg_cost:26.879168 batch_id:553, avg_cost:28.180256 batch_id:554, avg_cost:23.380322 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:555, avg_cost:29.751842 [train] batch_id:556, precision:[0.99188069], recall:[0.99167223], f1:[0.99177645] ./conf/test_data 30 30 [test] batch_id:556, precision:[0.99115914], recall:[0.99098214], f1:[0.99107063] cur_batch_id:556, last 5 batchs, time_cost:53.731470108 keep training! batch_id:556, avg_cost:25.293892 batch_id:557, avg_cost:26.375626 batch_id:558, avg_cost:26.059658 batch_id:559, avg_cost:35.618076 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:560, avg_cost:36.515564 [train] batch_id:561, precision:[0.99038546], recall:[0.9900299], f1:[0.99020765] ./conf/test_data 30 30 [test] batch_id:561, precision:[0.9865613], recall:[0.98647321], f1:[0.98651726] cur_batch_id:561, last 5 batchs, time_cost:48.499792099 keep training! batch_id:561, avg_cost:40.86869 batch_id:562, avg_cost:41.323315 batch_id:563, avg_cost:36.013046 batch_id:564, avg_cost:28.907124 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:565, avg_cost:24.7443 [train] batch_id:566, precision:[0.98905921], recall:[0.98895245], f1:[0.98900583] ./conf/test_data 30 30 [test] batch_id:566, precision:[0.99124883], recall:[0.99111607], f1:[0.99118245] cur_batch_id:566, last 5 batchs, time_cost:52.6075308323 keep training! batch_id:566, avg_cost:30.286997 batch_id:567, avg_cost:31.690891 batch_id:568, avg_cost:24.201653 batch_id:569, avg_cost:22.710518 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:570, avg_cost:19.127213 [train] batch_id:571, precision:[0.99104536], recall:[0.99067132], f1:[0.99085831] ./conf/test_data 30 30 [test] batch_id:571, precision:[0.99115953], recall:[0.99102679], f1:[0.99109315] cur_batch_id:571, last 5 batchs, time_cost:49.9665210247 keep training! batch_id:571, avg_cost:26.767197 batch_id:572, avg_cost:28.736502 batch_id:573, avg_cost:24.582369 batch_id:574, avg_cost:40.686584 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:575, avg_cost:24.259905 [train] batch_id:576, precision:[0.99093558], recall:[0.99065988], f1:[0.99079771] ./conf/test_data 30 30 [test] batch_id:576, precision:[0.99031163], recall:[0.99022321], f1:[0.99026742] cur_batch_id:576, last 5 batchs, time_cost:49.4741899967 keep training! batch_id:576, avg_cost:25.223083 batch_id:577, avg_cost:24.409954 batch_id:578, avg_cost:25.23522 batch_id:579, avg_cost:45.327164 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:580, avg_cost:67.00339 [train] batch_id:581, precision:[0.98030642], recall:[0.98027018], f1:[0.9802883] ./conf/test_data 30 30 [test] batch_id:581, precision:[0.98297891], recall:[0.98227679], f1:[0.98262772] cur_batch_id:581, last 5 batchs, time_cost:52.9658210278 cur_avg_fs <= last_avg_f1! batch_id:581, avg_cost:131.23073 batch_id:582, avg_cost:416.50293 batch_id:583, avg_cost:284.7473 batch_id:584, avg_cost:412.12747 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:585, avg_cost:495.7599 [train] batch_id:586, precision:[0.95076351], recall:[0.92058774], f1:[0.93543233] ./conf/test_data 30 30 [test] batch_id:586, precision:[0.96754024], recall:[0.9740625], f1:[0.97079042] cur_batch_id:586, last 5 batchs, time_cost:49.8440830708 cur_avg_fs <= last_avg_f1! batch_id:586, avg_cost:193.64848 batch_id:587, avg_cost:524.20795 batch_id:588, avg_cost:171.49594 batch_id:589, avg_cost:1837.8544 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:590, avg_cost:1264.7365 [train] batch_id:591, precision:[0.90261737], recall:[0.90545709], f1:[0.904035] ./conf/test_data 30 30 [test] batch_id:591, precision:[0.02571948], recall:[0.02589286], f1:[0.02580588] cur_batch_id:591, last 5 batchs, time_cost:51.6086521149 cur_avg_fs <= last_avg_f1! batch_id:591, avg_cost:4168.049 batch_id:592, avg_cost:2415.5732 batch_id:593, avg_cost:1715.0652 batch_id:594, avg_cost:1271.2507 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:595, avg_cost:581.77936 [train] batch_id:596, precision:[0.84934683], recall:[0.85685104], f1:[0.85308243] ./conf/test_data 30 30 [test] batch_id:596, precision:[0.42307692], recall:[0.00392857], f1:[0.00778485] cur_batch_id:596, last 5 batchs, time_cost:79.0023081303 cur_avg_fs <= last_avg_f1! batch_id:596, avg_cost:9716.86 batch_id:597, avg_cost:8762.354 batch_id:598, avg_cost:8775.369 batch_id:599, avg_cost:7858.473 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:600, avg_cost:6874.813 [train] batch_id:601, precision:[0.96320221], recall:[0.8482704], f1:[0.90209029] ./conf/test_data 30 30 [test] batch_id:601, precision:[0.96754024], recall:[0.9740625], f1:[0.97079042] cur_batch_id:601, last 5 batchs, time_cost:49.4042160511 cur_avg_fs <= last_avg_f1! batch_id:601, avg_cost:5805.404 batch_id:602, avg_cost:6248.8667 batch_id:603, avg_cost:5219.3535 batch_id:604, avg_cost:4730.429 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:605, avg_cost:4093.8132 [train] batch_id:606, precision:[0.96377082], recall:[0.97258115], f1:[0.96815594] ./conf/test_data 30 30 [test] batch_id:606, precision:[0.96754024], recall:[0.9740625], f1:[0.97079042] cur_batch_id:606, last 5 batchs, time_cost:109.015568972 cur_avg_fs <= last_avg_f1! batch_id:606, avg_cost:3346.4417 batch_id:607, avg_cost:2446.231 batch_id:608, avg_cost:1986.445 batch_id:609, avg_cost:1576.2915 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:610, avg_cost:1088.1302 [train] batch_id:611, precision:[0.96453489], recall:[0.97310254], f1:[0.96879977] ./conf/test_data 30 30 [test] batch_id:611, precision:[0.00940091], recall:[0.00946429], f1:[0.00943249] cur_batch_id:611, last 5 batchs, time_cost:66.0644118786 cur_avg_fs <= last_avg_f1! batch_id:611, avg_cost:10075.178 batch_id:612, avg_cost:8546.415 batch_id:613, avg_cost:8683.717 batch_id:614, avg_cost:7911.358 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:615, avg_cost:7565.28 [train] batch_id:616, precision:[0.8482514], recall:[0.85535281], f1:[0.8517873] ./conf/test_data 30 30 [test] batch_id:616, precision:[0.96754024], recall:[0.9740625], f1:[0.97079042] cur_batch_id:616, last 5 batchs, time_cost:51.3674058914 cur_avg_fs <= last_avg_f1! batch_id:616, avg_cost:6652.714 batch_id:617, avg_cost:6108.9663 batch_id:618, avg_cost:6408.4727 batch_id:619, avg_cost:6220.4844 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:620, avg_cost:5154.8735 [train] batch_id:621, precision:[0.96380714], recall:[0.97272296], f1:[0.96824452] ./conf/test_data 30 30 [test] batch_id:621, precision:[0.96754024], recall:[0.9740625], f1:[0.97079042] cur_batch_id:621, last 5 batchs, time_cost:52.5047721863 cur_avg_fs <= last_avg_f1! batch_id:621, avg_cost:4169.627 batch_id:622, avg_cost:4121.6465 batch_id:623, avg_cost:3499.6968 batch_id:624, avg_cost:2861.6892 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:625, avg_cost:2720.2324 [train] batch_id:626, precision:[0.96637509], recall:[0.97412936], f1:[0.97023673] ./conf/test_data 30 30 [test] batch_id:626, precision:[0.96754024], recall:[0.9740625], f1:[0.97079042] cur_batch_id:626, last 5 batchs, time_cost:184.488799095 cur_avg_fs <= last_avg_f1! batch_id:626, avg_cost:1652.3354 batch_id:627, avg_cost:1311.3981 batch_id:628, avg_cost:974.034 batch_id:629, avg_cost:3742.7444 train_data corpus finish a pass of training ./conf/train_data 222 222 batch_id:630, avg_cost:9479.485 [train] batch_id:631, precision:[0.92830623], recall:[0.92017417], f1:[0.92422231] ./conf/test_data 30 30 [test] batch_id:631, precision:[0.96754024], recall:[0.9740625], f1:[0.97079042] cur_batch_id:631, last 5 batchs, time_cost:51.9951021671 cur_avg_fs <= last_avg_f1!

hjchen2 commented 5 years ago

是不是有异常数据影响了训练的收敛性，建议先排查一下数据集是否有异常数据，另外也可以尝试用gradent clip策略，防止梯度突然爆炸。可以直接调用fluid.clip.set_gradient_clip(xx)接口进行设置clip。

paddle-bot-old[bot] commented 4 years ago

Since you haven\'t replied for more than a year, we have closed this issue/pr. If the problem is not solved or there is a follow-up one, please reopen it at any time and we will continue to follow up. 由于您超过一年未回复，我们将关闭这个issue/pr。 若问题未解决或有后续问题，请随时重新打开，我们会继续跟进。