Curiosity007
commented
1 year ago I solved this by adding n_ctx and max_tokens = 256.
However, this brings to new error -
llama_tokenize: too many tokens
Traceback (most recent call last):
File "/home/user/CASALIOY/customLLM.py", line 54, in <module>
main()
File "/home/user/CASALIOY/customLLM.py", line 39, in main
res = qa(query)
File "/root/miniconda3/envs/vicuna/lib/python3.9/site-packages/langchain/chains/base.py", line 140, in __call__
raise e
File "/root/miniconda3/envs/vicuna/lib/python3.9/site-packages/langchain/chains/base.py", line 134, in __call__
self._call(inputs, run_manager=run_manager)
File "/root/miniconda3/envs/vicuna/lib/python3.9/site-packages/langchain/chains/retrieval_qa/base.py", line 120, in _call
answer = self.combine_documents_chain.run(
File "/root/miniconda3/envs/vicuna/lib/python3.9/site-packages/langchain/chains/base.py", line 239, in run
return self(kwargs, callbacks=callbacks)[self.output_keys[0]]
File "/root/miniconda3/envs/vicuna/lib/python3.9/site-packages/langchain/chains/base.py", line 140, in __call__
raise e
File "/root/miniconda3/envs/vicuna/lib/python3.9/site-packages/langchain/chains/base.py", line 134, in __call__
self._call(inputs, run_manager=run_manager)
File "/root/miniconda3/envs/vicuna/lib/python3.9/site-packages/langchain/chains/combine_documents/base.py", line 84, in _call
output, extra_return_dict = self.combine_docs(
File "/root/miniconda3/envs/vicuna/lib/python3.9/site-packages/langchain/chains/combine_documents/stuff.py", line 87, in combine_docs
return self.llm_chain.predict(callbacks=callbacks, **inputs), {}
File "/root/miniconda3/envs/vicuna/lib/python3.9/site-packages/langchain/chains/llm.py", line 213, in predict
return self(kwargs, callbacks=callbacks)[self.output_key]
File "/root/miniconda3/envs/vicuna/lib/python3.9/site-packages/langchain/chains/base.py", line 140, in __call__
raise e
File "/root/miniconda3/envs/vicuna/lib/python3.9/site-packages/langchain/chains/base.py", line 134, in __call__
self._call(inputs, run_manager=run_manager)
File "/root/miniconda3/envs/vicuna/lib/python3.9/site-packages/langchain/chains/llm.py", line 69, in _call
response = self.generate([inputs], run_manager=run_manager)
File "/root/miniconda3/envs/vicuna/lib/python3.9/site-packages/langchain/chains/llm.py", line 79, in generate
return self.llm.generate_prompt(
File "/root/miniconda3/envs/vicuna/lib/python3.9/site-packages/langchain/llms/base.py", line 127, in generate_prompt
return self.generate(prompt_strings, stop=stop, callbacks=callbacks)
File "/root/miniconda3/envs/vicuna/lib/python3.9/site-packages/langchain/llms/base.py", line 176, in generate
raise e
File "/root/miniconda3/envs/vicuna/lib/python3.9/site-packages/langchain/llms/base.py", line 170, in generate
self._generate(prompts, stop=stop, run_manager=run_manager)
File "/root/miniconda3/envs/vicuna/lib/python3.9/site-packages/langchain/llms/base.py", line 377, in _generate
self._call(prompt, stop=stop, run_manager=run_manager)
File "/root/miniconda3/envs/vicuna/lib/python3.9/site-packages/langchain/llms/llamacpp.py", line 228, in _call
for token in self.stream(prompt=prompt, stop=stop, run_manager=run_manager):
File "/root/miniconda3/envs/vicuna/lib/python3.9/site-packages/langchain/llms/llamacpp.py", line 277, in stream
for chunk in result:
File "/root/miniconda3/envs/vicuna/lib/python3.9/site-packages/llama_cpp/llama.py", line 591, in _create_completion
prompt_tokens: List[llama_cpp.llama_token] = self.tokenize(
File "/root/miniconda3/envs/vicuna/lib/python3.9/site-packages/llama_cpp/llama.py", line 200, in tokenize
raise RuntimeError(f'Failed to tokenize: text="{text}" n_tokens={n_tokens}')
RuntimeError: Failed to tokenize: text="b" Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer.\n\n5TAYMEOUEQCYBVY2: DF778R1NYEBBI6CT\n1997-07-26: 2009-10-10\nRamiro: Angelina\nStover: Deboer\nisiah2@gmail.com: tashina18@yahoo.com\n\n5TAYMEOUEQCYBVY2: STBYB9ANQYQKHDXF\n1997-07-26: 2020-01-01\nRamiro: Vickey\nStover: Welch\nisiah2@gmail.com: isa_lewis@started.sumoto.hyogo.jp\n\n5TAYMEOUEQCYBVY2: 9YO6R9J0A3BESV2E\n1997-07-26: 2017-11-18\nRamiro: Bev\nStover: Satterfield\nisiah2@gmail.com: emeliabuxton2806@gmail.com\n\n5TAYMEOUEQCYBVY2: O4IMC2SQ4EL3UPBM\n1997-07-26: 2022-09-05\nRamiro: Shawnta\nStover: Everson\nisiah2@gmail.com: gisela-albright@rolling.hanawa.fukushima.jp\n\nQuestion: hi\nHelpful Answer:"" n_tokens=-415this is the code I am using for customLLM.py
from langchain.chains import RetrievalQA
from langchain.embeddings import LlamaCppEmbeddings
from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
from langchain.vectorstores import Qdrant
import qdrant_client
from langchain.llms import LlamaCpp
def main():
# Load stored vectorstore
llama = LlamaCppEmbeddings(model_path='models/ggml-model-q4_0.bin')
# Load ggml-formatted model
local_path = 'models/ggml-vic-7b-uncensored.bin'
client = qdrant_client.QdrantClient(
path="./db", prefer_grpc=True
)
qdrant = Qdrant(
client=client, collection_name="test",
embeddings=llama
)
# Prepare the LLM chain
callbacks = [StreamingStdOutCallbackHandler()]
#llm = GPT4All(model=local_path, callbacks=callbacks, verbose=True, backend='gptj')
llm = LlamaCpp(
model_path=local_path, callbacks=callbacks, verbose=True, n_ctx = 256, max_tokens = 256)
qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=qdrant.as_retriever(search_type="mmr"), return_source_documents=True)
# Interactive questions and answers
while True:
query = input("\nEnter a query: ")
if query == "exit":
break
# Get the answer from the chain
res = qa(query)
answer, docs = res['result'], res['source_documents']
# Print the result
print("\n\n> Question:")
print(query)
print("\n> Answer:")
print(answer)
# Print the relevant sources used for the answer
for document in docs:
print("\n> " + document.metadata["source"] + ":")
print(document.page_content)
if __name__ == "__main__":
main()
hippalectryon-0
su77ungr
alxspiker
Error Stack Trace