This happens when I try to use a context size over 2048, e.g. 4096+. (I have not found the exact point at which it starts, necessarily).
Here is the traceback (that's what this is called, right?) it dumps:
File "aiserver.py", line 3906, in generate
genout, already_generated = tpool.execute(model.core_generate, txt, found_entries, gen_mode=gen_mode)
File "/opt/koboldai/runtime/envs/koboldai/lib/python3.8/site-packages/eventlet/tpool.py", line 132, in execute
six.reraise(c, e, tb)
File "/opt/koboldai/runtime/envs/koboldai/lib/python3.8/site-packages/six.py", line 719, in reraise
raise value
File "/opt/koboldai/runtime/envs/koboldai/lib/python3.8/site-packages/eventlet/tpool.py", line 86, in tworker
rv = meth(*args, **kwargs)
File "/opt/koboldai/modeling/inference_model.py", line 356, in core_generate
result = self.raw_generate(
File "/opt/koboldai/modeling/inference_model.py", line 629, in raw_generate
result = self._raw_generate(
File "/opt/koboldai/modeling/inference_models/exllama/class.py", line 308, in _raw_generate
self.generator.gen_begin_reuse(gen_in)
File "/opt/koboldai/runtime/envs/koboldai/lib/python3.8/site-packages/exllama/generator.py", line 223, in gen_begin_reuse
if reuse < in_tokens.shape[-1]: self.gen_feed_tokens(in_tokens[:, reuse:], mask = mask)
File "/opt/koboldai/runtime/envs/koboldai/lib/python3.8/site-packages/exllama/generator.py", line 243, in gen_feed_tokens
self.model.forward(self.sequence[:, start : -1], self.cache, preprocess_only = True, lora = self.lora, input_mask = mask)
File "/opt/koboldai/runtime/envs/koboldai/lib/python3.8/site-packages/exllama/model.py", line 966, in forward
r = self._forward(input_ids[:, chunk_begin : chunk_end],
File "/opt/koboldai/runtime/envs/koboldai/lib/python3.8/site-packages/exllama/model.py", line 1052, in _forward
hidden_states = decoder_layer.forward(hidden_states, cache, buffers[device], lora)
File "/opt/koboldai/runtime/envs/koboldai/lib/python3.8/site-packages/exllama/model.py", line 530, in forward
hidden_states = self.self_attn.forward(hidden_states, cache, buffer, lora)
File "/opt/koboldai/runtime/envs/koboldai/lib/python3.8/site-packages/exllama/model.py", line 434, in forward
new_keys = cache.key_states[self.index].narrow(2, past_len, q_len).narrow(0, 0, bsz)
RuntimeError: start (958) + length (1624) exceeds dimension size (2048).
This happens when I try to use a context size over 2048, e.g. 4096+. (I have not found the exact point at which it starts, necessarily).
Here is the traceback (that's what this is called, right?) it dumps:
File "aiserver.py", line 3906, in generate genout, already_generated = tpool.execute(model.core_generate, txt, found_entries, gen_mode=gen_mode) File "/opt/koboldai/runtime/envs/koboldai/lib/python3.8/site-packages/eventlet/tpool.py", line 132, in execute six.reraise(c, e, tb) File "/opt/koboldai/runtime/envs/koboldai/lib/python3.8/site-packages/six.py", line 719, in reraise raise value File "/opt/koboldai/runtime/envs/koboldai/lib/python3.8/site-packages/eventlet/tpool.py", line 86, in tworker rv = meth(*args, **kwargs) File "/opt/koboldai/modeling/inference_model.py", line 356, in core_generate result = self.raw_generate( File "/opt/koboldai/modeling/inference_model.py", line 629, in raw_generate result = self._raw_generate( File "/opt/koboldai/modeling/inference_models/exllama/class.py", line 308, in _raw_generate self.generator.gen_begin_reuse(gen_in) File "/opt/koboldai/runtime/envs/koboldai/lib/python3.8/site-packages/exllama/generator.py", line 223, in gen_begin_reuse if reuse < in_tokens.shape[-1]: self.gen_feed_tokens(in_tokens[:, reuse:], mask = mask) File "/opt/koboldai/runtime/envs/koboldai/lib/python3.8/site-packages/exllama/generator.py", line 243, in gen_feed_tokens self.model.forward(self.sequence[:, start : -1], self.cache, preprocess_only = True, lora = self.lora, input_mask = mask) File "/opt/koboldai/runtime/envs/koboldai/lib/python3.8/site-packages/exllama/model.py", line 966, in forward r = self._forward(input_ids[:, chunk_begin : chunk_end], File "/opt/koboldai/runtime/envs/koboldai/lib/python3.8/site-packages/exllama/model.py", line 1052, in _forward hidden_states = decoder_layer.forward(hidden_states, cache, buffers[device], lora) File "/opt/koboldai/runtime/envs/koboldai/lib/python3.8/site-packages/exllama/model.py", line 530, in forward hidden_states = self.self_attn.forward(hidden_states, cache, buffer, lora) File "/opt/koboldai/runtime/envs/koboldai/lib/python3.8/site-packages/exllama/model.py", line 434, in forward new_keys = cache.key_states[self.index].narrow(2, past_len, q_len).narrow(0, 0, bsz) RuntimeError: start (958) + length (1624) exceeds dimension size (2048).