node-fetch works - Deno fetch fails to make api calls to openai

[mfrancis107]

Version: Deno 2.0.3

I have tested this on both a Mac and Windows WSL

Using node-fetch always works. When using Deno's default fetch if we increase the payload size I will get an error.

failed with error: error sending request from x.x.x.x:53270 for https://api.openai.com/v1/chat/completions (172.66.0.243:443): client error (SendRequest): http2 error: stream error received: unspecific protocol error detected

If we reduce the promptLength length to 200; We will get successful requests.

I haven't seen this happen with other API's yet. It looks like OpenAI Proxy's through Cloudflare.

import { faker } from '@faker-js/faker';
import { encoding_for_model } from "tiktoken";
import {env} from "node:process";
import fetch from "node-fetch";

const enc = encoding_for_model("gpt-4o-mini");

const promptLength = 2000; // A lower multiplier like 200 will usually work. But increasing the payload causes errors.
const numRequests = 100; //

const prompt = `${faker.lorem.paragraph(promptLength)}\n\nCreate a react app.`;
const tokens = enc.encode(prompt);
console.log(tokens.length);

async function makeRequest(id) {
  try {
    const startTime = Date.now();
    const response = await fetch('https://api.openai.com/v1/chat/completions', {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'Authorization': `Bearer ${env["OPENAI_API_KEY"]}`
      },
      body: JSON.stringify({
        model: 'gpt-4o-mini',
        messages: [{ role: 'user', content: prompt }]
      })
    });

    if(response.status == 200){

    const data = await response.json();
    const elapsedTime = Date.now() - startTime;
    const tokensPerSecond = (data.usage.completion_tokens / elapsedTime) * 1000;
    return { success: true, elapsedTime, tokensPerSecond };
    } else {
        return { success:false, error: await response.text()  }
    }
  } catch (error) {
    return { success: false, error: error.message };
  }
}

async function main() {
  const numRequests = 100;
  console.log(`Starting performance test of ${numRequests} parallel requests...`);

  let remainingIds = [...Array(numRequests).keys()];
  const promises = [];

  for (let i = 0; i < numRequests; i++) {
    promises.push(
      new Promise(async (resolve) => {
        console.log(`request ${i} started`);
        const result = await makeRequest(i);
        remainingIds = remainingIds.filter(id => id !== i);
        if (result.success) {
            console.log(`request ${i} completed in ${result.elapsedTime}ms with ${result.tokensPerSecond.toFixed(2)} tokens/sec`);
        } else {
          console.log(`request ${i} failed with error: ${result.error}`);
          console.log(`Remaining requests: ${remainingIds.join(', ')}`);
        }
        resolve(result);
      })
    );
  }

  const results = await Promise.all(promises);

  const successfulRequests = results.filter(r => r.success);
  const failedRequests = results.filter(r => !r.success);

  const responseTimes = successfulRequests.map(r => r.elapsedTime);
  const tokenSpeeds = successfulRequests.map(r => r.tokensPerSecond);

  const avgTime = responseTimes.length ? responseTimes.reduce((a, b) => a + b, 0) / responseTimes.length : 0;
  const minTime = responseTimes.length ? Math.min(...responseTimes) : 0;
  const maxTime = responseTimes.length ? Math.max(...responseTimes) : 0;
  const avgTokensPerSecond = tokenSpeeds.length ? tokenSpeeds.reduce((a, b) => a + b, 0) / tokenSpeeds.length : 0;
  const minTokensPerSecond = tokenSpeeds.length ? Math.min(...tokenSpeeds) : 0;
  const maxTokensPerSecond = tokenSpeeds.length ? Math.max(...tokenSpeeds) : 0;

  console.log(`Performance results:
    Successful requests: ${successfulRequests.length}
    Failed requests: ${failedRequests.length}
    Average response time: ${avgTime.toFixed(2)}ms
    Minimum response time: ${minTime}ms
    Maximum response time: ${maxTime}ms
    Average tokens per second: ${avgTokensPerSecond.toFixed(2)}
    Minimum tokens per second: ${minTokensPerSecond.toFixed(2)}
    Maximum tokens per second: ${maxTokensPerSecond.toFixed(2)}
  `);

  if (failedRequests.length > 0) {
    console.log('\nError messages from failed requests:');
    failedRequests.forEach((req, index) => {
      console.log(`${index + 1}. ${req.error}`);
    });
  }
}

main().catch(console.error);

[marvinhagemeister]

FYI: As a workaround you might want to try using the native fetch implementation instead of node-fetch. You can do so by dropping the node-fetch import and keeping everything else as is. The fetch global is available in both Deno and Node.

[mfrancis107]

Native deno fetch is the one that does not work. node-fetch does work.

[marvinhagemeister]

Right, apologies. Looks like I misread the issue.