jacbson commented 2 years ago

paddle版本： paddlehub 2.1.0 pypi_0 pypi paddlenlp 2.0.8 pypi_0 pypi paddlepaddle-gpu 2.1.3 pypi_0 pypi

语句： import paddlehub as hub import paddle label_list = ['B-assist','B-cellno','B-city','B-community','B-devzone','B-distance','B-district','B-floorno','B-houseno', 'B-intersection','B-poi','B-prov','B-road','B-roadno','B-subpoi','B-town','B-village_group','E-assist','E-cellno', 'E-city','E-community','E-devzone','E-distance','E-district','E-floorno','E-houseno','E-intersection','E-poi','E-prov', 'E-road','E-roadno','E-subpoi','E-town','E-village_group','I-assist','I-cellno','I-city','I-community','I-devzone', 'I-distance','I-district','I-floorno','I-houseno','I-intersection','I-poi','I-prov','I-road','I-roadno','I-subpoi', 'I-town','I-village_group','O','S-assist','S-intersection','S-poi','S-district','S-community'] label_map = {idx: label for idx, label in enumerate(label_list)}

选择所需要的模型

model = hub.Module(name='ernie', task='token-cls', label_map=label_map)

出错信息： TypeError Traceback (most recent call last) ~\AppData\Local\Temp/ipykernel_1104/1273845971.py in 1 # 选择所需要的模型 ----> 2 model = hub.Module(name='ernie', task='token-cls', label_map=label_map)

~\Anaconda3\envs\paddle-env\lib\site-packages\paddlehub\module\module.py in new(cls, name, directory, version, source, update, branch, ignore_env_mismatch, **kwargs) 386 # This branch come from hub.Module(name='xxx') or hub.Module(directory='xxx') 387 if name: --> 388 module = cls.init_with_name( 389 name=name, 390 version=version,

~\Anaconda3\envs\paddle-env\lib\site-packages\paddlehub\module\module.py in init_with_name(cls, name, version, source, update, branch, ignore_env_mismatch, kwargs) 502 ) 503 user_module = user_module_cls(directory=directory) --> 504 user_module._initialize(kwargs) 505 return user_module 506

~\Anaconda3\envs\paddle-env\lib\site-packages\paddlehub\compat\paddle_utils.py in runner(*args, kwargs) 218 def runner(*args, *kwargs): 219 with static_mode_guard(): --> 220 return func(args, kwargs) 221 222 return runner

TypeError: _initialize() got an unexpected keyword argument 'task'

linjieccc commented 2 years ago

可以卸载重装paddlehub至最新版本再试下，我这边运行是没有问题的

jacbson commented 2 years ago

现在模型加载是没问题了，但是模型训练到eval阶段的时候，notebook就会莫名其妙崩溃，不知道是什么原因？

语句： import paddlehub as hub import paddle label_list = ['B-assist','B-cellno','B-city','B-community','B-devzone','B-distance','B-district','B-floorno','B-houseno', 'B-intersection','B-poi','B-prov','B-road','B-roadno','B-subpoi','B-town','B-village_group','E-assist','E-cellno', 'E-city','E-community','E-devzone','E-distance','E-district','E-floorno','E-houseno','E-intersection','E-poi','E-prov', 'E-road','E-roadno','E-subpoi','E-town','E-village_group','I-assist','I-cellno','I-city','I-community','I-devzone', 'I-distance','I-district','I-floorno','I-houseno','I-intersection','I-poi','I-prov','I-road','I-roadno','I-subpoi', 'I-town','I-village_group','O','S-assist','S-intersection','S-poi','S-district','S-community'] label_map = {idx: label for idx, label in enumerate(label_list)}

选择所需要的模型

model = hub.Module(name='ernie_tiny', version='2.0.1', task='token-cls', label_map=label_map) from paddlehub.datasets.base_nlp_dataset import SeqLabelingDataset

class ExpressNER(SeqLabelingDataset):

数据集存放目录

base_path = 'express_ner'
# 数据集的标签列表
label_list = label_list
label_map = {idx: label for idx, label in enumerate(label_list)}
# 数据文件使用的分隔符
split_char = '\002'

def __init__(self, tokenizer, max_seq_len: int = 128, mode: str = 'train'):
    if mode == 'train':
        data_file = 'train.txt'
    elif mode == 'test':
        data_file = 'test.txt'
    else:
        data_file = 'dev.txt'
    super().__init__(
                base_path=self.base_path,
                tokenizer=tokenizer,
                max_seq_len=max_seq_len,
                mode=mode,
                data_file=data_file,
                label_file=None,
                label_list=self.label_list,
                split_char=self.split_char,
                is_file_with_header=True)

tokenizer = model.get_tokenizer()

获取数据集

train_dataset = ExpressNER(tokenizer=tokenizer, max_seq_len=64, mode='train') dev_dataset = ExpressNER(tokenizer=tokenizer, max_seq_len=64, mode='dev') test_dataset = ExpressNER(tokenizer=tokenizer, max_seq_len=64, mode='test') optimizer = paddle.optimizer.Adam(learning_rate=5e-5, parameters=model.parameters()) # 优化器的选择和参数配置 trainer = hub.Trainer(model, optimizer, checkpoint_dir='./ckpt', use_gpu=True) # fine-tune任务的执行者

trainer.train(train_dataset, epochs=10, batch_size=8, eval_dataset=dev_dataset, save_interval=5) # 配置训练参数，启动训练，并指定验证集

执行日志： [2021-09-26 10:13:49,732] [ WARNING] - Compatibility Warning: The params of ChunkEvaluator.compute has been modified. The old version is inputs, lengths, predictions, labels while the current version is lengths, predictions, labels. Please update the usage. [2021-09-26 10:13:50,352] [ TRAIN] - Epoch=1/10, Step=10/1107 loss=1.0749 f1_score=0.0119 lr=0.000050 step/sec=15.29 | ETA 00:12:03 [2021-09-26 10:13:50,931] [ TRAIN] - Epoch=1/10, Step=20/1107 loss=0.7263 f1_score=0.1215 lr=0.000050 step/sec=17.28 | ETA 00:11:22 [2021-09-26 10:13:51,492] [ TRAIN] - Epoch=1/10, Step=30/1107 loss=0.5467 f1_score=0.3325 lr=0.000050 step/sec=17.80 | ETA 00:11:02 [2021-09-26 10:13:52,062] [ TRAIN] - Epoch=1/10, Step=40/1107 loss=0.4233 f1_score=0.3977 lr=0.000050 step/sec=17.54 | ETA 00:10:54 [2021-09-26 10:13:52,644] [ TRAIN] - Epoch=1/10, Step=50/1107 loss=0.2870 f1_score=0.5122 lr=0.000050 step/sec=17.19 | ETA 00:10:52 [2021-09-26 10:13:53,211] [ TRAIN] - Epoch=1/10, Step=60/1107 loss=0.2535 f1_score=0.5397 lr=0.000050 step/sec=17.65 | ETA 00:10:48 [2021-09-26 10:13:53,785] [ TRAIN] - Epoch=1/10, Step=70/1107 loss=0.2207 f1_score=0.5924 lr=0.000050 step/sec=17.41 | ETA 00:10:46 [2021-09-26 10:13:54,376] [ TRAIN] - Epoch=1/10, Step=80/1107 loss=0.2168 f1_score=0.6086 lr=0.000050 step/sec=16.91 | ETA 00:10:47 [2021-09-26 10:13:54,954] [ TRAIN] - Epoch=1/10, Step=90/1107 loss=0.1816 f1_score=0.6798 lr=0.000050 step/sec=17.30 | ETA 00:10:46 [2021-09-26 10:13:55,526] [ TRAIN] - Epoch=1/10, Step=100/1107 loss=0.1617 f1_score=0.7098 lr=0.000050 step/sec=17.51 | ETA 00:10:45 [2021-09-26 10:13:56,099] [ TRAIN] - Epoch=1/10, Step=110/1107 loss=0.1782 f1_score=0.6805 lr=0.000050 step/sec=17.45 | ETA 00:10:44 [2021-09-26 10:13:56,685] [ TRAIN] - Epoch=1/10, Step=120/1107 loss=0.1483 f1_score=0.6792 lr=0.000050 step/sec=17.05 | ETA 00:10:44 [2021-09-26 10:13:57,255] [ TRAIN] - Epoch=1/10, Step=130/1107 loss=0.1415 f1_score=0.7202 lr=0.000050 step/sec=17.55 | ETA 00:10:43 [2021-09-26 10:13:57,830] [ TRAIN] - Epoch=1/10, Step=140/1107 loss=0.1232 f1_score=0.7417 lr=0.000050 step/sec=17.38 | ETA 00:10:43 [2021-09-26 10:13:58,406] [ TRAIN] - Epoch=1/10, Step=150/1107 loss=0.1524 f1_score=0.7394 lr=0.000050 step/sec=17.35 | ETA 00:10:42 [2021-09-26 10:13:58,981] [ TRAIN] - Epoch=1/10, Step=160/1107 loss=0.1236 f1_score=0.7412 lr=0.000050 step/sec=17.39 | ETA 00:10:42 [2021-09-26 10:13:59,556] [ TRAIN] - Epoch=1/10, Step=170/1107 loss=0.1092 f1_score=0.7471 lr=0.000050 step/sec=17.41 | ETA 00:10:41 [2021-09-26 10:14:00,132] [ TRAIN] - Epoch=1/10, Step=180/1107 loss=0.1238 f1_score=0.7777 lr=0.000050 step/sec=17.35 | ETA 00:10:41 [2021-09-26 10:14:00,722] [ TRAIN] - Epoch=1/10, Step=190/1107 loss=0.1291 f1_score=0.7318 lr=0.000050 step/sec=16.95 | ETA 00:10:42 [2021-09-26 10:14:01,288] [ TRAIN] - Epoch=1/10, Step=200/1107 loss=0.1074 f1_score=0.7615 lr=0.000050 step/sec=17.68 | ETA 00:10:41 [2021-09-26 10:14:01,859] [ TRAIN] - Epoch=1/10, Step=210/1107 loss=0.0953 f1_score=0.7656 lr=0.000050 step/sec=17.51 | ETA 00:10:41 [2021-09-26 10:14:02,434] [ TRAIN] - Epoch=1/10, Step=220/1107 loss=0.1252 f1_score=0.7735 lr=0.000050 step/sec=17.38 | ETA 00:10:40 [2021-09-26 10:14:03,017] [ TRAIN] - Epoch=1/10, Step=230/1107 loss=0.0898 f1_score=0.7540 lr=0.000050 step/sec=17.15 | ETA 00:10:41 [2021-09-26 10:14:03,576] [ TRAIN] - Epoch=1/10, Step=240/1107 loss=0.1129 f1_score=0.7521 lr=0.000050 step/sec=17.89 | ETA 00:10:40 [2021-09-26 10:14:04,163] [ TRAIN] - Epoch=1/10, Step=250/1107 loss=0.1445 f1_score=0.6939 lr=0.000050 step/sec=17.05 | ETA 00:10:40 [2021-09-26 10:14:04,742] [ TRAIN] - Epoch=1/10, Step=260/1107 loss=0.0918 f1_score=0.7885 lr=0.000050 step/sec=17.26 | ETA 00:10:40 [2021-09-26 10:14:05,310] [ TRAIN] - Epoch=1/10, Step=270/1107 loss=0.1078 f1_score=0.7795 lr=0.000050 step/sec=17.61 | ETA 00:10:40 [2021-09-26 10:14:05,890] [ TRAIN] - Epoch=1/10, Step=280/1107 loss=0.1294 f1_score=0.7531 lr=0.000050 step/sec=17.24 | ETA 00:10:40 [2021-09-26 10:14:06,449] [ TRAIN] - Epoch=1/10, Step=290/1107 loss=0.0998 f1_score=0.7799 lr=0.000050 step/sec=17.90 | ETA 00:10:39 [2021-09-26 10:14:07,015] [ TRAIN] - Epoch=1/10, Step=300/1107 loss=0.1139 f1_score=0.7874 lr=0.000050 step/sec=17.66 | ETA 00:10:38 [2021-09-26 10:14:07,605] [ TRAIN] - Epoch=1/10, Step=310/1107 loss=0.0912 f1_score=0.7754 lr=0.000050 step/sec=16.95 | ETA 00:10:39 [2021-09-26 10:14:08,164] [ TRAIN] - Epoch=1/10, Step=320/1107 loss=0.0831 f1_score=0.8122 lr=0.000050 step/sec=17.89 | ETA 00:10:38 [2021-09-26 10:14:08,745] [ TRAIN] - Epoch=1/10, Step=330/1107 loss=0.1168 f1_score=0.7299 lr=0.000050 step/sec=17.20 | ETA 00:10:38 [2021-09-26 10:14:09,327] [ TRAIN] - Epoch=1/10, Step=340/1107 loss=0.0952 f1_score=0.7856 lr=0.000050 step/sec=17.20 | ETA 00:10:39 [2021-09-26 10:14:09,912] [ TRAIN] - Epoch=1/10, Step=350/1107 loss=0.0991 f1_score=0.7940 lr=0.000050 step/sec=17.09 | ETA 00:10:39 [2021-09-26 10:14:10,477] [ TRAIN] - Epoch=1/10, Step=360/1107 loss=0.0865 f1_score=0.7916 lr=0.000050 step/sec=17.67 | ETA 00:10:38 [2021-09-26 10:14:11,060] [ TRAIN] - Epoch=1/10, Step=370/1107 loss=0.1127 f1_score=0.7517 lr=0.000050 step/sec=17.17 | ETA 00:10:39 [2021-09-26 10:14:11,642] [ TRAIN] - Epoch=1/10, Step=380/1107 loss=0.0929 f1_score=0.8108 lr=0.000050 step/sec=17.18 | ETA 00:10:39 [2021-09-26 10:14:12,202] [ TRAIN] - Epoch=1/10, Step=390/1107 loss=0.0873 f1_score=0.7847 lr=0.000050 step/sec=17.86 | ETA 00:10:38 [2021-09-26 10:14:12,790] [ TRAIN] - Epoch=1/10, Step=400/1107 loss=0.0790 f1_score=0.8151 lr=0.000050 step/sec=16.99 | ETA 00:10:39 [2021-09-26 10:14:13,366] [ TRAIN] - Epoch=1/10, Step=410/1107 loss=0.0917 f1_score=0.8065 lr=0.000050 step/sec=17.37 | ETA 00:10:39 [2021-09-26 10:14:13,943] [ TRAIN] - Epoch=1/10, Step=420/1107 loss=0.0819 f1_score=0.8414 lr=0.000050 step/sec=17.34 | ETA 00:10:39 [2021-09-26 10:14:14,532] [ TRAIN] - Epoch=1/10, Step=430/1107 loss=0.0875 f1_score=0.8011 lr=0.000050 step/sec=16.98 | ETA 00:10:39 [2021-09-26 10:14:15,097] [ TRAIN] - Epoch=1/10, Step=440/1107 loss=0.0756 f1_score=0.8358 lr=0.000050 step/sec=17.70 | ETA 00:10:39 [2021-09-26 10:14:15,701] [ TRAIN] - Epoch=1/10, Step=450/1107 loss=0.0628 f1_score=0.8590 lr=0.000050 step/sec=16.56 | ETA 00:10:39 [2021-09-26 10:14:16,295] [ TRAIN] - Epoch=1/10, Step=460/1107 loss=0.0689 f1_score=0.8161 lr=0.000050 step/sec=16.84 | ETA 00:10:40 [2021-09-26 10:14:16,864] [ TRAIN] - Epoch=1/10, Step=470/1107 loss=0.0869 f1_score=0.7955 lr=0.000050 step/sec=17.55 | ETA 00:10:39 [2021-09-26 10:14:17,437] [ TRAIN] - Epoch=1/10, Step=480/1107 loss=0.0887 f1_score=0.8041 lr=0.000050 step/sec=17.47 | ETA 00:10:39 [2021-09-26 10:14:18,019] [ TRAIN] - Epoch=1/10, Step=490/1107 loss=0.0743 f1_score=0.8346 lr=0.000050 step/sec=17.17 | ETA 00:10:39 [2021-09-26 10:14:18,594] [ TRAIN] - Epoch=1/10, Step=500/1107 loss=0.0607 f1_score=0.8667 lr=0.000050 step/sec=17.40 | ETA 00:10:39 [2021-09-26 10:14:19,175] [ TRAIN] - Epoch=1/10, Step=510/1107 loss=0.0644 f1_score=0.8859 lr=0.000050 step/sec=17.23 | ETA 00:10:39 [2021-09-26 10:14:19,744] [ TRAIN] - Epoch=1/10, Step=520/1107 loss=0.0755 f1_score=0.8462 lr=0.000050 step/sec=17.57 | ETA 00:10:39 [2021-09-26 10:14:20,331] [ TRAIN] - Epoch=1/10, Step=530/1107 loss=0.1151 f1_score=0.7464 lr=0.000050 step/sec=17.02 | ETA 00:10:39 [2021-09-26 10:14:20,909] [ TRAIN] - Epoch=1/10, Step=540/1107 loss=0.1050 f1_score=0.8303 lr=0.000050 step/sec=17.30 | ETA 00:10:39 [2021-09-26 10:14:21,488] [ TRAIN] - Epoch=1/10, Step=550/1107 loss=0.0973 f1_score=0.8079 lr=0.000050 step/sec=17.29 | ETA 00:10:39 [2021-09-26 10:14:22,065] [ TRAIN] - Epoch=1/10, Step=560/1107 loss=0.0889 f1_score=0.8158 lr=0.000050 step/sec=17.33 | ETA 00:10:39 [2021-09-26 10:14:22,631] [ TRAIN] - Epoch=1/10, Step=570/1107 loss=0.0605 f1_score=0.8503 lr=0.000050 step/sec=17.67 | ETA 00:10:39 [2021-09-26 10:14:23,206] [ TRAIN] - Epoch=1/10, Step=580/1107 loss=0.0819 f1_score=0.8182 lr=0.000050 step/sec=17.38 | ETA 00:10:39 [2021-09-26 10:14:23,789] [ TRAIN] - Epoch=1/10, Step=590/1107 loss=0.0694 f1_score=0.8314 lr=0.000050 step/sec=17.15 | ETA 00:10:39 [2021-09-26 10:14:24,382] [ TRAIN] - Epoch=1/10, Step=600/1107 loss=0.0963 f1_score=0.7898 lr=0.000050 step/sec=16.87 | ETA 00:10:39 [2021-09-26 10:14:24,967] [ TRAIN] - Epoch=1/10, Step=610/1107 loss=0.0984 f1_score=0.7980 lr=0.000050 step/sec=17.11 | ETA 00:10:40 [2021-09-26 10:14:25,527] [ TRAIN] - Epoch=1/10, Step=620/1107 loss=0.0787 f1_score=0.8035 lr=0.000050 step/sec=17.84 | ETA 00:10:39 [2021-09-26 10:14:26,104] [ TRAIN] - Epoch=1/10, Step=630/1107 loss=0.0719 f1_score=0.8514 lr=0.000050 step/sec=17.33 | ETA 00:10:39 [2021-09-26 10:14:26,678] [ TRAIN] - Epoch=1/10, Step=640/1107 loss=0.0752 f1_score=0.8572 lr=0.000050 step/sec=17.41 | ETA 00:10:39 [2021-09-26 10:14:27,252] [ TRAIN] - Epoch=1/10, Step=650/1107 loss=0.0595 f1_score=0.8543 lr=0.000050 step/sec=17.43 | ETA 00:10:39 [2021-09-26 10:14:27,836] [ TRAIN] - Epoch=1/10, Step=660/1107 loss=0.0813 f1_score=0.8287 lr=0.000050 step/sec=17.11 | ETA 00:10:39 [2021-09-26 10:14:28,410] [ TRAIN] - Epoch=1/10, Step=670/1107 loss=0.0726 f1_score=0.8305 lr=0.000050 step/sec=17.45 | ETA 00:10:39 [2021-09-26 10:14:28,997] [ TRAIN] - Epoch=1/10, Step=680/1107 loss=0.0580 f1_score=0.8565 lr=0.000050 step/sec=17.03 | ETA 00:10:39 [2021-09-26 10:14:29,575] [ TRAIN] - Epoch=1/10, Step=690/1107 loss=0.1327 f1_score=0.7626 lr=0.000050 step/sec=17.29 | ETA 00:10:39 [2021-09-26 10:14:30,140] [ TRAIN] - Epoch=1/10, Step=700/1107 loss=0.0765 f1_score=0.8242 lr=0.000050 step/sec=17.72 | ETA 00:10:39 [2021-09-26 10:14:30,716] [ TRAIN] - Epoch=1/10, Step=710/1107 loss=0.0859 f1_score=0.8277 lr=0.000050 step/sec=17.35 | ETA 00:10:39 [2021-09-26 10:14:31,299] [ TRAIN] - Epoch=1/10, Step=720/1107 loss=0.0721 f1_score=0.7828 lr=0.000050 step/sec=17.16 | ETA 00:10:39 [2021-09-26 10:14:31,878] [ TRAIN] - Epoch=1/10, Step=730/1107 loss=0.1005 f1_score=0.7547 lr=0.000050 step/sec=17.26 | ETA 00:10:39 [2021-09-26 10:14:32,448] [ TRAIN] - Epoch=1/10, Step=740/1107 loss=0.0729 f1_score=0.8090 lr=0.000050 step/sec=17.56 | ETA 00:10:39 [2021-09-26 10:14:33,022] [ TRAIN] - Epoch=1/10, Step=750/1107 loss=0.0793 f1_score=0.8189 lr=0.000050 step/sec=17.42 | ETA 00:10:39 [2021-09-26 10:14:33,596] [ TRAIN] - Epoch=1/10, Step=760/1107 loss=0.0724 f1_score=0.8305 lr=0.000050 step/sec=17.42 | ETA 00:10:39 [2021-09-26 10:14:34,183] [ TRAIN] - Epoch=1/10, Step=770/1107 loss=0.0692 f1_score=0.8112 lr=0.000050 step/sec=17.04 | ETA 00:10:39 [2021-09-26 10:14:34,767] [ TRAIN] - Epoch=1/10, Step=780/1107 loss=0.0534 f1_score=0.8490 lr=0.000050 step/sec=17.12 | ETA 00:10:39 [2021-09-26 10:14:35,339] [ TRAIN] - Epoch=1/10, Step=790/1107 loss=0.0831 f1_score=0.8360 lr=0.000050 step/sec=17.47 | ETA 00:10:39 [2021-09-26 10:14:35,912] [ TRAIN] - Epoch=1/10, Step=800/1107 loss=0.1037 f1_score=0.8189 lr=0.000050 step/sec=17.48 | ETA 00:10:39 [2021-09-26 10:14:36,494] [ TRAIN] - Epoch=1/10, Step=810/1107 loss=0.0616 f1_score=0.8628 lr=0.000050 step/sec=17.16 | ETA 00:10:39 [2021-09-26 10:14:37,069] [ TRAIN] - Epoch=1/10, Step=820/1107 loss=0.0646 f1_score=0.8349 lr=0.000050 step/sec=17.41 | ETA 00:10:39 [2021-09-26 10:14:37,643] [ TRAIN] - Epoch=1/10, Step=830/1107 loss=0.0734 f1_score=0.8387 lr=0.000050 step/sec=17.42 | ETA 00:10:39 [2021-09-26 10:14:38,217] [ TRAIN] - Epoch=1/10, Step=840/1107 loss=0.0618 f1_score=0.8299 lr=0.000050 step/sec=17.42 | ETA 00:10:39 [2021-09-26 10:14:38,808] [ TRAIN] - Epoch=1/10, Step=850/1107 loss=0.0609 f1_score=0.8262 lr=0.000050 step/sec=16.93 | ETA 00:10:39 [2021-09-26 10:14:39,390] [ TRAIN] - Epoch=1/10, Step=860/1107 loss=0.0692 f1_score=0.8139 lr=0.000050 step/sec=17.16 | ETA 00:10:39 [2021-09-26 10:14:39,959] [ TRAIN] - Epoch=1/10, Step=870/1107 loss=0.0538 f1_score=0.8665 lr=0.000050 step/sec=17.58 | ETA 00:10:39 [2021-09-26 10:14:40,540] [ TRAIN] - Epoch=1/10, Step=880/1107 loss=0.0468 f1_score=0.9024 lr=0.000050 step/sec=17.22 | ETA 00:10:39 [2021-09-26 10:14:41,130] [ TRAIN] - Epoch=1/10, Step=890/1107 loss=0.0818 f1_score=0.8407 lr=0.000050 step/sec=16.95 | ETA 00:10:39 [2021-09-26 10:14:41,704] [ TRAIN] - Epoch=1/10, Step=900/1107 loss=0.0627 f1_score=0.8346 lr=0.000050 step/sec=17.40 | ETA 00:10:39 [2021-09-26 10:14:42,280] [ TRAIN] - Epoch=1/10, Step=910/1107 loss=0.0958 f1_score=0.7635 lr=0.000050 step/sec=17.38 | ETA 00:10:39 [2021-09-26 10:14:42,855] [ TRAIN] - Epoch=1/10, Step=920/1107 loss=0.0740 f1_score=0.8080 lr=0.000050 step/sec=17.39 | ETA 00:10:39 [2021-09-26 10:14:43,445] [ TRAIN] - Epoch=1/10, Step=930/1107 loss=0.0692 f1_score=0.8396 lr=0.000050 step/sec=16.95 | ETA 00:10:39 [2021-09-26 10:14:44,031] [ TRAIN] - Epoch=1/10, Step=940/1107 loss=0.0692 f1_score=0.8300 lr=0.000050 step/sec=17.07 | ETA 00:10:39 [2021-09-26 10:14:44,608] [ TRAIN] - Epoch=1/10, Step=950/1107 loss=0.0733 f1_score=0.8697 lr=0.000050 step/sec=17.32 | ETA 00:10:39 [2021-09-26 10:14:45,180] [ TRAIN] - Epoch=1/10, Step=960/1107 loss=0.0861 f1_score=0.7876 lr=0.000050 step/sec=17.48 | ETA 00:10:39 [2021-09-26 10:14:45,751] [ TRAIN] - Epoch=1/10, Step=970/1107 loss=0.0785 f1_score=0.8208 lr=0.000050 step/sec=17.53 | ETA 00:10:39 [2021-09-26 10:14:46,326] [ TRAIN] - Epoch=1/10, Step=980/1107 loss=0.0734 f1_score=0.8327 lr=0.000050 step/sec=17.38 | ETA 00:10:39 [2021-09-26 10:14:46,909] [ TRAIN] - Epoch=1/10, Step=990/1107 loss=0.0589 f1_score=0.8380 lr=0.000050 step/sec=17.15 | ETA 00:10:39 [2021-09-26 10:14:47,484] [ TRAIN] - Epoch=1/10, Step=1000/1107 loss=0.0614 f1_score=0.8710 lr=0.000050 step/sec=17.40 | ETA 00:10:39 [2021-09-26 10:14:48,073] [ TRAIN] - Epoch=1/10, Step=1010/1107 loss=0.0541 f1_score=0.8793 lr=0.000050 step/sec=16.98 | ETA 00:10:39 [2021-09-26 10:14:48,641] [ TRAIN] - Epoch=1/10, Step=1020/1107 loss=0.0726 f1_score=0.8331 lr=0.000050 step/sec=17.62 | ETA 00:10:39 [2021-09-26 10:14:49,235] [ TRAIN] - Epoch=1/10, Step=1030/1107 loss=0.0737 f1_score=0.8378 lr=0.000050 step/sec=16.82 | ETA 00:10:39 [2021-09-26 10:14:49,808] [ TRAIN] - Epoch=1/10, Step=1040/1107 loss=0.0720 f1_score=0.8175 lr=0.000050 step/sec=17.45 | ETA 00:10:39 [2021-09-26 10:14:50,382] [ TRAIN] - Epoch=1/10, Step=1050/1107 loss=0.0635 f1_score=0.8070 lr=0.000050 step/sec=17.42 | ETA 00:10:39 [2021-09-26 10:14:50,964] [ TRAIN] - Epoch=1/10, Step=1060/1107 loss=0.0831 f1_score=0.8336 lr=0.000050 step/sec=17.19 | ETA 00:10:39 [2021-09-26 10:14:51,546] [ TRAIN] - Epoch=1/10, Step=1070/1107 loss=0.0705 f1_score=0.8299 lr=0.000050 step/sec=17.19 | ETA 00:10:39 [2021-09-26 10:14:52,113] [ TRAIN] - Epoch=1/10, Step=1080/1107 loss=0.0819 f1_score=0.8032 lr=0.000050 step/sec=17.63 | ETA 00:10:39 [2021-09-26 10:14:52,703] [ TRAIN] - Epoch=1/10, Step=1090/1107 loss=0.0693 f1_score=0.8148 lr=0.000050 step/sec=16.96 | ETA 00:10:39 [2021-09-26 10:14:53,269] [ TRAIN] - Epoch=1/10, Step=1100/1107 loss=0.0655 f1_score=0.8263 lr=0.000050 step/sec=17.64 | ETA 00:10:39

linjieccc commented 2 years ago

麻烦发下报错信息谢谢

jacbson commented 2 years ago

没有报错，到了Step=1100/1107以后就不运行下去了，一直不动。

linjieccc commented 2 years ago

方便告知下数据的获取方式，我这边尝试复现下问题

linjieccc commented 2 years ago

用demo提供的数据集跑会hang住吗？

jacbson commented 2 years ago

https://aistudio.baidu.com/bdvgpu/user/189371/2178752/notebooks/2178752.ipynb 我在aistudio上跑了，也是报错 2021-09-26 10:46:22,144 - WARNING - DataLoader reader thread raised an exception. Exception in thread Thread-8: Traceback (most recent call last): File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/threading.py", line 926, in _bootstrap_inner self.run() File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/threading.py", line 870, in run self._target(*self._args, *self._kwargs) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/dataloader/dataloader_iter.py", line 346, in _thread_loop six.reraise(sys.exc_info()) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/six.py", line 703, in reraise raise value File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/dataloader/dataloader_iter.py", line 317, in _thread_loop batch = self._dataset_fetcher.fetch(indices) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/dataloader/fetcher.py", line 60, in fetch data = [self.dataset[idx] for idx in batch_indices] File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/dataloader/fetcher.py", line 60, in data = [self.dataset[idx] for idx in batch_indices] File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlehub/datasets/base_nlp_dataset.py", line 381, in getitem return np.array(record['input_ids']), np.array(record['segment_ids']), np.array(record['seq_len']), np.array(record['label'], dtype=np.int64) KeyError: 'segment_ids'

---------------------------------------------------------------------------SystemError Traceback (most recent call last) in ----> 1 trainer.train(train_dataset, epochs=10, batch_size=32, eval_dataset=dev_dataset, save_interval=5) # 配置训练参数，启动训练，并指定验证集 /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlehub/finetune/trainer.py in train(self, train_dataset, epochs, batch_size, num_workers, eval_dataset, log_interval, save_interval, collate_fn) 196 self.model.train() 197 --> 198 for batch_idx, batch in enumerate(loader): 199 loss, metrics = self.training_step(batch, batch_idx) 200 self.optimizer_step(self.current_epoch, batch_idx, self.optimizer, loss) /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/dataloader/dataloader_iter.py in next(self) 349 try: 350 if in_dygraph_mode(): --> 351 return self._reader.read_next_var_list() 352 else: 353 if self._returnlist: SystemError: (Fatal) Blocking queue is killed because the data reader raises an exception. [Hint: Expected killed != true, but received killed_:1 == true:1.] (at /paddle/paddle/fluid/operators/reader/blocking_queue.h:158) 是我的数据集有问题吗？

linjieccc commented 2 years ago

你给的项目链接我没有权限访问，方便的话可以设置成公开项目：点击项目右上角的分享->复制链接

链接格式： https://aistudio.baidu.com/aistudio/projectdetail/xxxxx

jacbson commented 2 years ago

https://aistudio.baidu.com/aistudio/projectdetail/2178752?contributionType=1&shared=1

linjieccc commented 2 years ago

项目里的label_list和数据集文件train.txt的标签对不上。我用默认的express_ner数据集验证没有问题，可以参考下demo数据集文件的格式检查一下你的数据集文件

jacbson commented 2 years ago

好的

rrryan2016 commented 1 year ago

paddlepaddle 2.4.1 cpu 做NER任务遇到了类似的问题，具体的报错如下

[2023-01-01 09:44:46,575] [ WARNING] - Compatibility Warning: The params of ChunkEvaluator.compute has been modified. The old version is `inputs`, `lengths`, `predictions`, `labels` while the current version is `lengths`, `predictions`, `labels`.  Please update the 
usage.
Traceback (most recent call last):
  File "d:/Doc/Personal/OceanneDLG/mine/21 paddle text/ner_2.py", line 56, in <module>
    trainer.train(train_dataset, epochs=num_epoch, batch_size=batch_size, eval_dataset=dev_dataset,
  File "D:\Program\Anaconda3\lib\site-packages\paddlehub\finetune\trainer.py", line 212, in train
    loss, metrics = self.training_step(batch, batch_idx)
  File "D:\Program\Anaconda3\lib\site-packages\paddlehub\finetune\trainer.py", line 352, in training_step
    result = self.model.training_step(batch, batch_idx)
  File "D:\Program\Anaconda3\lib\site-packages\paddlehub\module\nlp_module.py", line 504, in training_step
    predictions, avg_loss, metric = self(
  File "D:\Program\Anaconda3\lib\site-packages\paddle\fluid\dygraph\layers.py", line 948, in __call__
    return self.forward(*inputs, **kwargs)
  File "C:\Users\ghostInSh3ll\.paddlehub\modules\ernie_tiny\module.py", line 130, in forward
    self.metric.compute(None, seq_lengths, preds, labels)
  File "D:\Program\Anaconda3\lib\site-packages\paddlenlp\metrics\chunk.py", line 118, in compute
    unpad_labels = [
  File "D:\Program\Anaconda3\lib\site-packages\paddlenlp\metrics\chunk.py", line 119, in <listcomp>
    [self.id2label_dict[index] for index in labels[sent_index][: lengths[sent_index]]]
  File "D:\Program\Anaconda3\lib\site-packages\paddlenlp\metrics\chunk.py", line 119, in <listcomp>
    [self.id2label_dict[index] for index in labels[sent_index][: lengths[sent_index]]]
KeyError: -100

代码里的label_list和数据集文件的标签已检查是对应的，模型的具体输出如下

[2023-01-01 09:44:04,915] [    INFO] - Model config ErnieConfig {
  "attention_probs_dropout_prob": 0.1,
  "enable_recompute": false,
  "fuse": false,
  "hidden_act": "relu",
  "hidden_dropout_prob": 0.1,
  "hidden_size": 1024,
  "id2label": {
    "0": "LABEL_0",
    "1": "LABEL_1",
    "2": "LABEL_2",
    "3": "LABEL_3",
    "4": "LABEL_4",
    "5": "LABEL_5",
    "6": "LABEL_6",
    "7": "LABEL_7",
    "8": "LABEL_8"
  },
  "initializer_range": 0.02,
  "intermediate_size": 4096,
  "label2id": {
    "LABEL_0": 0,
    "LABEL_1": 1,
    "LABEL_2": 2,
    "LABEL_3": 3,
    "LABEL_4": 4,
    "LABEL_5": 5,
    "LABEL_6": 6,
    "LABEL_7": 7,
    "LABEL_8": 8
  },
  "layer_norm_eps": 1e-12,
  "max_position_embeddings": 600,
  "model_type": "ernie",
  "num_attention_heads": 16,
  "num_hidden_layers": 3,
  "pad_token_id": 0,
  "paddlenlp_version": null,
  "pool_act": "tanh",
  "task_id": 0,
  "task_type_vocab_size": 3,
  "type_vocab_size": 2,
  "use_task_id": true,
  "vocab_size": 50006
}

suntao2015005848 commented 1 year ago

一样的报错：在执行model = hub.Module(name='ernie', task='token-cls', label_map=label_map)时候总是报：TypeError: _initialize() got an unexpected keyword argument 'task'

PaddlePaddle / PaddleHub

序列标注加载模型报错 #1621

选择所需要的模型

选择所需要的模型

数据集存放目录

获取数据集