Generation stop；torch.cuda.OutOfMemoryError: CUDA out of memory.

[Essence9999]

ubuntu 22.04 server; GPU A10 24G; attention_sinks 0.4.0

`import torch from transformers import AutoTokenizer, TextStreamer, GenerationConfig from attention_sinks import AutoModelForCausalLM

model_id = "/home/work/projects/model/Qwen-7B" model = AutoModelForCausalLM.from_pretrained( model_id, device_map="auto", torch_dtype=torch.float16, attention_sink_size=4, attention_sink_window_size=252, trust_remote_code=True, use_flash_attn=False ) model.eval() tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True) tokenizer.pad_token_id = tokenizer.eos_token_id

text = "保持身体健康有多种方式" input_ids = tokenizer.encode(text, return_tensors="pt").to(model.device)

streamer = TextStreamer(tokenizer) generation_config=GenerationConfig( use_cache=True, min_new_tokens=100_000, max_new_tokens=1_000_000, penalty_alpha=0.6, top_k=5, pad_token_id=tokenizer.pad_token_id, eos_token_id=tokenizer.eos_token_id, ) generated_tokens = model.generate( input_ids, generation_config, streamer=streamer, )

output_text = tokenizer.decode(generated_tokens[0], skip_special_tokens=True)`

The model generation is over normally，do not generate long text output:

But when I use the finetuned Qwen-base model to generate, after generating some text, the error is coming:

My code is as follows:

import torch from transformers import AutoTokenizer, TextStreamer, GenerationConfig from attention_sinks import AutoModelForCausalLM

model_id = "/home/work/projects/LLaMA-Factory/output/qwen_base"

model = AutoModelForCausalLM.from_pretrained( model_id,

for efficiency:

device_map="auto",
torch_dtype=torch.float16,
attention_sink_size=16,
attention_sink_window_size=1024, 
trust_remote_code=True,
use_flash_attn=False

)

model.eval() tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True) tokenizer.pad_token_id = tokenizer.eos_token_id

text = "写一篇文章，主题为：5G网络架构探讨"

input_ids = tokenizer.encode(text, return_tensors="pt").to(model.device)

with torch.no_grad(): streamer = TextStreamer(tokenizer) generated_tokens = model.generate( input_ids, generation_config=GenerationConfig( use_cache=True, min_new_tokens=100_000, max_new_tokens=1_000_000, penalty_alpha=0.6, top_k=5, pad_token_id=tokenizer.pad_token_id, eos_token_id=tokenizer.eos_token_id, ), streamer=streamer, ) output_text = tokenizer.decode(generated_tokens[0], skip_special_tokens=True)