[Question] 运行cli.py文件时出现ValueError: `bos_token_id` has to be defined when no `input_ids` are provided.

[20191864218]

Question

我在运行cli.py文件进行推理时，出现以下错误： Traceback (most recent call last): File "/root/miniconda3/lib/python3.10/runpy.py", line 196, in _run_module_as_main return _run_code(code, main_globals, None, File "/root/miniconda3/lib/python3.10/runpy.py", line 86, in _run_code exec(code, run_globals) File "/root/LLaVA/llava/serve/cli.py", line 137, in <module> main(args) File "/root/LLaVA/llava/serve/cli.py", line 106, in main output_ids = model.generate( File "/root/miniconda3/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context return func(*args, **kwargs) File "/root/LLaVA/llava/model/language_model/llava_Taiyi.py", line 133, in generate return super().generate( File "/root/LLaVA/llava/model/language_model/Taiyi/modeling_qwen.py", line 1111, in generate return super().generate( File "/root/miniconda3/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context return func(*args, **kwargs) File "/root/miniconda3/lib/python3.10/site-packages/transformers/generation/utils.py", line 1330, in generate inputs_tensor, model_input_name, model_kwargs = self._prepare_model_inputs( File "/root/miniconda3/lib/python3.10/site-packages/transformers/generation/utils.py", line 402, in _prepare_model_inputs model_kwargs["input_ids"] = self._maybe_initialize_input_ids_for_generation( File "/root/miniconda3/lib/python3.10/site-packages/transformers/generation/utils.py", line 431, in _maybe_initialize_input_ids_for_generation raise ValueError("bos_token_idhas to be defined when noinput_idsare provided.") ValueError:bos_token_idhas to be defined when noinput_idsare provided.

但是input_ids打印出来长这个样子

cli.py文件内容为： `import argparse import torch

from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN from llava.conversation import conv_templates, SeparatorStyle from llava.model.builder import load_pretrained_model from llava.utils import disable_torch_init from llava.mm_utils import process_images, tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria

from PIL import Image

import requests from PIL import Image from io import BytesIO from transformers import TextStreamer

def load_image(image_file): if image_file.startswith('http://') or image_file.startswith('https://'): response = requests.get(image_file) image = Image.open(BytesIO(response.content)).convert('RGB') else: image = Image.open(image_file).convert('RGB') return image

def main(args):

Model

disable_torch_init()

model_name = get_model_name_from_path(args.model_path)
tokenizer, model, image_processor, context_len = load_pretrained_model(args.model_path, args.model_base, model_name, args.load_8bit, args.load_4bit, device=args.device)
if "llama-2" in model_name.lower():
    conv_mode = "llava_llama_2"
elif "mistral" in model_name.lower():
    conv_mode = "mistral_instruct"
elif "v1.6-34b" in model_name.lower():
    conv_mode = "chatml_direct"
elif "v1" in model_name.lower():
    conv_mode = "llava_v1"
elif "mpt" in model_name.lower():
    conv_mode = "mpt"
else:
    conv_mode = "llava_v0"

if args.conv_mode is not None and conv_mode != args.conv_mode:
    print('[WARNING] the auto inferred conversation mode is {}, while `--conv-mode` is {}, using {}'.format(conv_mode, args.conv_mode, args.conv_mode))
else:
    args.conv_mode = conv_mode

conv = conv_templates[args.conv_mode].copy()
if "mpt" in model_name.lower():
    roles = ('user', 'assistant')
else:
    roles = conv.roles

image = load_image(args.image_file)
image_size = image.size
# Similar operation in model_worker.py
image_tensor = process_images([image], image_processor, model.config)
if type(image_tensor) is list:
    image_tensor = [image.to(model.device, dtype=torch.float16) for image in image_tensor]
else:
    image_tensor = image_tensor.to(model.device, dtype=torch.float16)

while True:
    try:
        inp = input(f"{roles[0]}: ")
    except EOFError:
        inp = ""
    if not inp:
        print("exit...")
        break

    print(f"{roles[1]}: ", end="")

    if image is not None:
        # first message
        if model.config.mm_use_im_start_end:
            inp = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\n' + inp
        else:
            inp = DEFAULT_IMAGE_TOKEN + '\n' + inp
        conv.append_message(conv.roles[0], inp)
        image = None
    else:
        # later messages
        conv.append_message(conv.roles[0], inp)
    conv.append_message(conv.roles[1], None)
    prompt = conv.get_prompt()
    print('prompt:', prompt)

    input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).to(model.device)
    print('input_ids:', input_ids)
    print('input_ids.shape:', input_ids.shape)
    print('input_ids.shape[1]:', input_ids.shape[1])
    # bos_token_id = torch.tensor([[tokenizer.bos_token_id]], dtype=torch.long).to(model.device)
    # eos_token_id = torch.tensor([[tokenizer.eos_token_id]], dtype=torch.long).to(model.device)
    # user_input_ids = torch.concat([bos_token_id,input_ids, eos_token_id], dim=1).to(model.device)
    stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
    print('stop_str:', stop_str)
    keywords = [stop_str]
    stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
    streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)

    with torch.inference_mode():
        output_ids = model.generate(
            input_ids,
            images=image_tensor,
            image_sizes=[image_size],
            do_sample=True if args.temperature > 0 else False,
            temperature=args.temperature,
            max_new_tokens=args.max_new_tokens,
            streamer=streamer,
            stopping_criteria=[stopping_criteria])

    outputs = tokenizer.decode(output_ids[0]).strip()
    conv.messages[-1][-1] = outputs

    # 如果启用了调试模式（args.debug为真），则打印出提示和生成的输出，方便调试和检查模型的行为。
    if args.debug:
        print("\n", {"prompt": prompt, "outputs": outputs}, "\n")

if name == "main": parser = argparse.ArgumentParser() parser.add_argument("--model-path", type=str, default="facebook/opt-350m") parser.add_argument("--model-base", type=str, default=None) parser.add_argument("--image-file", type=str, required=True) parser.add_argument("--device", type=str, default="cuda") parser.add_argument("--conv-mode", type=str, default=None) parser.add_argument("--temperature", type=float, default=0.2) parser.add_argument("--max-new-tokens", type=int, default=512) parser.add_argument("--load-8bit", action="store_true") parser.add_argument("--load-4bit", action="store_true") parser.add_argument("--debug", action="store_true") args = parser.parse_args() main(args) ` 请问各位大佬，为什么出现这个错误，感谢大佬的回复

[JosephPai]

Same problem...