作者,你好,我是按照run.sh脚本处理demo.json文件得到all.txt文件,然后用all.txt去执行run.sh出现了下面的错误,这是什么原因呀?
Traceback (most recent call last):
File "train.py", line 277, in
train_model()
File "train.py", line 133, in train_model
fit(model, train_data, dev_data)
File "train.py", line 144, in fit
train_dataset = ListDataset(X_train, args.sub_graph, args.max_groups, args.max_psglen, tokenizer, args.dataset_script_dir, args.dataset_cache_dir)
File "/root/laotang/Socialformer/model/list_dataset_all.py", line 40, in init
'pid': datasets.Value('string'),
File "/root/miniconda3/envs/so/lib/python3.7/site-packages/datasets/load.py", line 852, in load_dataset
use_auth_token=use_auth_token,
File "/root/miniconda3/envs/so/lib/python3.7/site-packages/datasets/builder.py", line 616, in download_and_prepare
dl_manager=dl_manager, verify_infos=verify_infos, download_and_prepare_kwargs
File "/root/miniconda3/envs/so/lib/python3.7/site-packages/datasets/builder.py", line 693, in _download_and_prepare
self._prepare_split(split_generator, prepare_split_kwargs)
File "/root/miniconda3/envs/so/lib/python3.7/site-packages/datasets/builder.py", line 1163, in _prepare_split
generator, unit=" tables", leave=False, disable=bool(logging.get_verbosity() == logging.NOTSET)
File "/root/miniconda3/envs/so/lib/python3.7/site-packages/tqdm/std.py", line 1195, in iter
for obj in iterable:
File "/root/.cache/huggingface/modules/datasets_modules/datasets/json/654bf9d299c640b28eb532b11086abb730c9e5eb50e2e29e463c3fa5a8559887/json.py", line 87, in _generate_tables
dataset = json.load(f)
File "/root/miniconda3/envs/so/lib/python3.7/json/init.py", line 296, in load
parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, **kw)
File "/root/miniconda3/envs/so/lib/python3.7/json/init.py", line 348, in loads
return _default_decoder.decode(s)
File "/root/miniconda3/envs/so/lib/python3.7/json/decoder.py", line 340, in decode
raise JSONDecodeError("Extra data", s, end)
json.decoder.JSONDecodeError: Extra data: line 2 column 1 (char 37609)
作者,你好,我是按照run.sh脚本处理demo.json文件得到all.txt文件,然后用all.txt去执行run.sh出现了下面的错误,这是什么原因呀? Traceback (most recent call last): File "train.py", line 277, in
train_model()
File "train.py", line 133, in train_model
fit(model, train_data, dev_data)
File "train.py", line 144, in fit
train_dataset = ListDataset(X_train, args.sub_graph, args.max_groups, args.max_psglen, tokenizer, args.dataset_script_dir, args.dataset_cache_dir)
File "/root/laotang/Socialformer/model/list_dataset_all.py", line 40, in init
'pid': datasets.Value('string'),
File "/root/miniconda3/envs/so/lib/python3.7/site-packages/datasets/load.py", line 852, in load_dataset
use_auth_token=use_auth_token,
File "/root/miniconda3/envs/so/lib/python3.7/site-packages/datasets/builder.py", line 616, in download_and_prepare
dl_manager=dl_manager, verify_infos=verify_infos, download_and_prepare_kwargs
File "/root/miniconda3/envs/so/lib/python3.7/site-packages/datasets/builder.py", line 693, in _download_and_prepare
self._prepare_split(split_generator, prepare_split_kwargs)
File "/root/miniconda3/envs/so/lib/python3.7/site-packages/datasets/builder.py", line 1163, in _prepare_split
generator, unit=" tables", leave=False, disable=bool(logging.get_verbosity() == logging.NOTSET)
File "/root/miniconda3/envs/so/lib/python3.7/site-packages/tqdm/std.py", line 1195, in iter
for obj in iterable:
File "/root/.cache/huggingface/modules/datasets_modules/datasets/json/654bf9d299c640b28eb532b11086abb730c9e5eb50e2e29e463c3fa5a8559887/json.py", line 87, in _generate_tables
dataset = json.load(f)
File "/root/miniconda3/envs/so/lib/python3.7/json/init.py", line 296, in load
parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, **kw)
File "/root/miniconda3/envs/so/lib/python3.7/json/init.py", line 348, in loads
return _default_decoder.decode(s)
File "/root/miniconda3/envs/so/lib/python3.7/json/decoder.py", line 340, in decode
raise JSONDecodeError("Extra data", s, end)
json.decoder.JSONDecodeError: Extra data: line 2 column 1 (char 37609)