useChat() streaming working in localhost but not docker deployment with nginx

[apensotti]

Description

AI Page

const { messages, input, handleInputChange, handleSubmit } = useChat();

/api/chat/route.ts

 import { LangChainAdapter, Message } from 'ai';
 import { unstable_noStore as noStore } from 'next/cache';

 // Allow streaming responses up to 30 seconds
 export const dynamic = 'force-dynamic';
 export const maxDuration = 30;

 noStore();
 export async function POST(req: Request) {
   const { messages }: { messages: Message[] } = await req.json();

   // Forward the request to your FastAPI backend
   const response = await fetch("https://www.moviewizard.com/wizapi/chat/stream", {
     method: "POST",
     headers: {
       "Content-Type": "application/json",
     },
     body: JSON.stringify({
       messages,  // Forward the messages directly
       email: "youremail",  // Add email or other necessary fields
       session_id: ""  // Add the session_id if required
     }),
   });

   // Read the streaming response from FastAPI
   const reader = response.body?.getReader();
   const decoder = new TextDecoder();

   const stream = new ReadableStream({
     async start(controller) {
       if (!reader) {
         controller.close();
         return;
       }

       while (true) {
         const { done, value } = await reader.read();
         if (done) {
           controller.close();
           break;
         }

         // Decode and send chunks to the stream
         const chunk = decoder.decode(value);
        controller.enqueue(chunk);
       }
     }
   });

   // Use LangChainAdapter to convert the stream to a data stream response
   return LangChainAdapter.toDataStreamResponse(stream);
 }

Fast API endpoint code

class Chat:
    def __init__(self):
        self.session_id = ""
        self.messages = []
        self.full_message = ""
        self.query = ""

    async def handle_chat_stream(self, req):
        self.query = get_query(req)

        if req.session_id == "":
            self.messages = []
        else:
            self.messages = get_messages(req.session_id)

        chain = rag_chain()

        async def generator():
            async for chunk in chain.astream({"input": self.query, "chat_history": self.messages}):
                try:
                    answer = chunk.get("answer", "")
                    self.full_message += answer
                    yield str(answer)
                except Exception as e:
                    print(e)

        # Add the user query and the system's response to the chat history
        req.messages.append(HumanMessage(content=self.query))
        req.messages.append(SystemMessage(content=self.full_message))

        if req.session_id == "":
            await create_session(req)
        else:
            await update_session(req)

        # Return a StreamingResponse with properly formatted JSON chunks
        return StreamingResponse(generator(), media_type='text/plain')

Code example

No response

Additional context

https://youtu.be/nK7fAOeQWeE

first one is local host, works fine. The second is the deployed website, which doesn't work.

I tried https://sdk.vercel.ai/docs/troubleshooting/common-issues/streaming-not-working-on-vercel

That solution didn't work.

[MISTYKITTY]

Le sam. 28 sept. 2024 à 2:34 PM, Alex @.***> a écrit :

Description AI Page

const { messages, input, handleInputChange, handleSubmit } = useChat();

/api/chat/route.ts

import { LangChainAdapter, Message } from 'ai'; import { unstable_noStore as noStore } from 'next/cache';

// Allow streaming responses up to 30 seconds export const dynamic = 'force-dynamic'; export const maxDuration = 30;

noStore(); export async function POST(req: Request) { const { messages }: { messages: Message[] } = await req.json();

// Forward the request to your FastAPI backend const response = await fetch("https://www.moviewizard.com/wizapi/chat/stream", { method: "POST", headers: { "Content-Type": "application/json", }, body: JSON.stringify({ messages, // Forward the messages directly email: "youremail", // Add email or other necessary fields session_id: "" // Add the session_id if required }), });

// Read the streaming response from FastAPI const reader = response.body?.getReader(); const decoder = new TextDecoder();

const stream = new ReadableStream({ async start(controller) { if (!reader) { controller.close(); return; }

   while (true) {
     const { done, value } = await reader.read();
     if (done) {
       controller.close();
       break;
     }

     // Decode and send chunks to the stream
     const chunk = decoder.decode(value);
    controller.enqueue(chunk);
   }
 }

});

// Use LangChainAdapter to convert the stream to a data stream response return LangChainAdapter.toDataStreamResponse(stream); }

Fast API endpoint code

class Chat: def init(self): self.session_id = "" self.messages = [] self.full_message = "" self.query = ""

async def handle_chat_stream(self, req):
    self.query = get_query(req)

    if req.session_id == "":
        self.messages = []
    else:
        self.messages = get_messages(req.session_id)

    chain = rag_chain()

    async def generator():
        async for chunk in chain.astream({"input": self.query, "chat_history": self.messages}):
            try:
                answer = chunk.get("answer", "")
                self.full_message += answer
                yield str(answer)
            except Exception as e:
                print(e)

    # Add the user query and the system's response to the chat history
    req.messages.append(HumanMessage(content=self.query))
    req.messages.append(SystemMessage(content=self.full_message))

    if req.session_id == "":
        await create_session(req)
    else:
        await update_session(req)

    # Return a StreamingResponse with properly formatted JSON chunks
    return StreamingResponse(generator(), media_type='text/plain')

Code example

No response Additional context

https://youtu.be/nK7fAOeQWeE

first one is local host, works fine. The second is the deployed website, which doesn't work.

I tried @.***/common-issues/streaming-not-working-on-vercel

That solution didn't work.

— Reply to this email directly, view it on GitHub, or unsubscribe. You are receiving this because you are subscribed to this thread.Message ID: @.***>

[bioshazard]

@apensotti got anything else between browser and docker? and can you share your Nginx config file?

[apensotti]

@bioshazard

No I don't have any middleware or anything like that.

server {
    listen 80;
    server_name mydomain.com www.mydomain.com;

    location / {
        proxy_pass http://frontend:3000/;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;

        # Disable caching
        add_header Cache-Control "no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0";
        add_header Pragma "no-cache";
        add_header Expires "0";

        # Ensure correct MIME type for GLSL files
        types {
            application/javascript glsl;
        }
    }

    location /wizapi/ {
        # Do not strip the /wizapi path
        proxy_pass http://backend:8000/;  # Ensure the path is passed intact
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;

        # Disable redirection by not modifying URL
        proxy_redirect off;

        # Disable caching for API
        add_header Cache-Control "no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0";
        add_header Pragma "no-cache";
        add_header Expires "0";
    }
}