ConversationalRetrievalQAChain "I dont know responses"

[mickmedical]

Can anyone help me identify why my ConversationalRetrievalQAChain is replying with "I don't know, or information is not provided" I can see it pulling the relevant docs in my terminal but it doesn't seem to use the information. Any help would be greatly appreciated!

import { PineconeClient } from "@pinecone-database/pinecone"; import { LangChainStream, StreamingTextResponse } from "ai"; import { ConversationalRetrievalQAChain } from "langchain/chains"; import {PromptTemplate,} from "langchain/prompts"; import { OpenAI } from "langchain/llms/openai"; import { OpenAIEmbeddings } from "langchain/embeddings/openai"; import { AIChatMessage, HumanChatMessage, SystemChatMessage } from "langchain/schema"; import { PineconeStore } from "langchain/vectorstores/pinecone"; import z from "zod";

const ChatSchema = z.object({ messages: z.array( z.object({ role: z.enum(["system", "user", "assistant"]), content: z.string(), id: z.string().optional(), createdAt: z.date().optional(), }) ), });

export const runtime = "edge";

let pinecone: PineconeClient | null = null;

const initPineconeClient = async () => { pinecone = new PineconeClient(); await pinecone.init({ apiKey: process.env.PINECONE_API_KEY!, environment: process.env.PINECONE_ENVIRONMENT!, }); };

export async function POST(req: Request) { console.log("POST request received");

const QA_PROMPT = PromptTemplate.fromTemplate( `Ignore all previous instructions. I want you to act as a document that I am having a conversation with. Only provide responses based on the given information. Never breach character.

Question: {question}
=========
{context}
=========
Answer in Markdown:`
);

const body = await req.json(); console.log("Request body:", body);

try { const { messages } = ChatSchema.parse(body); console.log("Parsed messages:", messages);

if (pinecone == null) {
  await initPineconeClient();
  console.log("Pinecone client initialized");
}

const pineconeIndex = pinecone!.Index(process.env.PINECONE_INDEX_NAME!);

const vectorStore = await PineconeStore.fromExistingIndex(new OpenAIEmbeddings(), {
  pineconeIndex,
  textKey: "text",
});
console.log("Vector store created");

const pastMessages = messages.map((m) => {
  if (m.role === "user") {
    return new HumanChatMessage(m.content);
  }
  if (m.role === "system") {
    return new SystemChatMessage(m.content);
  }
  return new AIChatMessage(m.content);
});
console.log("Past messages:", pastMessages);

const { stream, handlers } = LangChainStream();
console.log("LangChainStream created");

const model = new OpenAI({
  streaming: true,
  modelName: "gpt-4",
});
console.log("ChatOpenAI model created");

const questionModel = new OpenAI({
  modelName: "gpt-4",
});
console.log("Question model created");

const chain = ConversationalRetrievalQAChain.fromLLM(model, vectorStore.asRetriever(), {
  returnSourceDocuments: true,
  qaTemplate: QA_PROMPT.template,

  verbose: true,
  questionGeneratorChainOptions: {
    llm: questionModel,
  },
});
console.log("ConversationalRetrievalQAChain created");

const question = messages[messages.length - 1].content;
console.log("Question:", question);

console.log("Calling chain.call");
chain
  .call(
    {
      question,
      chat_history: pastMessages,
    },
    [handlers]
  )
  .catch((e) => {
    console.error("Error in chain.call:", e.message);
  });

console.log("Returning StreamingTextResponse");
return new StreamingTextResponse(stream);

} catch (error) { console.log("Error in POST handler:", error);

if (error instanceof z.ZodError) {
  console.log("Validation issues:", error.issues);
  return new Response(JSON.stringify(error.issues), { status: 422 });
}

console.log("Returning 500 response");
return new Response(null, { status: 500 });

} }