Having trouble using other models in node.js example.

[johnpg82]

I'm trying to use other models besides openai in your example code but I'm just getting a response back "invalid model". I've only tried with anthropic and perplexity so haven't set any of the other models supported in the array but those two models don't appear to be working. Note: It works fine if I use OpenAI models.

BadRequestError: 400 invalid model ID
    at APIError.generate (file:///Users/jgarland/Documents/GitHub/ai-api/node_modules/openai/error.mjs:41:20)
    at OpenAI.makeStatusError (file:///Users/jgarland/Documents/GitHub/ai-api/node_modules/openai/core.mjs:286:25)
    at OpenAI.makeRequest (file:///Users/jgarland/Documents/GitHub/ai-api/node_modules/openai/core.mjs:330:30)
    at process.processTicksAndRejections (node:internal/process/task_queues:105:5)
    at async main (file:///Users/jgarland/Documents/GitHub/ai-api/src/example.js:59:18) {
  status: 400,
  headers: {
    'alt-svc': 'h3=":443"; ma=86400',
    'cf-cache-status': 'DYNAMIC',
    'cf-ray': '8e2a1db4c819e64d-DEN',
    connection: 'keep-alive',
    'content-length': '149',
    'content-type': 'application/json; charset=utf-8',
    date: 'Thu, 14 Nov 2024 21:32:46 GMT',
    server: 'cloudflare',
    'set-cookie': '__cf_bm=-1731619966-1.0.1.1-.iEwnKF9.cvczeAyw; path=/; expires=Thu, 14-Nov-24 22:02:46 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None, _cfuvid=-1731619966293-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None',
    'strict-transport-security': 'max-age=31536000; includeSubDomains; preload',
    vary: 'Origin',
    'x-content-type-options': 'nosniff',
    'x-request-id': 'req_957384571a2399d617daa5ec52787002'
  },
  request_id: 'req_957384571a2399d617daa5ec52787002',
  error: {
    message: 'invalid model ID',
    type: 'invalid_request_error',
    param: null,
    code: null
  },
  code: null,
  param: null,
  type: 'invalid_request_error'
}

Here is some sample javascript code I'm using.

import OpenAI from 'openai';
import dotenv from 'dotenv';
dotenv.config({ path: '../.env' });

const modelApiKeys = {
  'openai': {
    apiKey: process.env.OPENAI_API_KEY,
    models: ['gpt-4o', 'gpt-4o-mini', 'gpt-4']
  },
  'anthropic': {
    apiKey: process.env.ANTHROPIC_API_KEY,
    models: ['anthropic.claude-v2']
  },
  'groq': {
    apiKey: process.env.GROQ_API_KEY,
    models: ['groq-model-1', 'groq-model-2']
  },
  'gemini': {
    apiKey: process.env.GEMINI_API_KEY,
    models: ['gemini-model-1', 'gemini-model-2']
  },
  'perplexity': {
    apiKey: process.env.PERPLEXITYAI_API_KEY,
    models: ['perplexity/mixtral-8x7b-instruct', 'perplexity/llama-3-sonar-large-32k-online']
  },
  'cohere': {
    apiKey: process.env.COHERE_API_KEY,
    models: ['cohere-model-1', 'cohere-model-2']
  },
  // Add other models and their corresponding API keys here
};

const model = 'perplexity/llama-3-sonar-large-32k-online'; // Change this to switch between models
//const model = 'gpt-4o-mini'; // Change this to switch between models
let apiKey;

for (const { apiKey: key, models } of Object.values(modelApiKeys)) {
  if (models.includes(model)) {
    apiKey = key;
    break;
  }
}

if (!apiKey) {
  throw new Error(`API key for model ${model} is not defined`);
}

const baseUrl = process.env.base_url || "https://open-assistant-ai.astra.datastax.com/v1";

async function main(apiKey, model) {
  console.log(baseUrl);
  const openai = new OpenAI({
    base_url: baseUrl,
    api_key: apiKey,
    default_headers: {
      "astra-api-token": process.env.ASTRA_DB_APPLICATION_TOKEN,
    },
  });

  const stream = await openai.chat.completions.create({
    model: model,
    messages: [{ role: 'user', content: "Can you generate a list of 10 ice cream flavors?" }],
    stream: true,
  });
  for await (const chunk of stream) {
    process.stdout.write(chunk.choices[0]?.delta?.content || '');
  }
}

main(apiKey, model);

[phact]

Hmm seems like you're still hitting openais endpoint?

Since this is js you'll have to do some things manually that we usually do for you in the python sdk wrapper / patch.

I'll take a look at your provided example as soon as I get a chance, don't see anything obviously wrong though.

In the meantime, this might help if you don't mind code diving:

https://github.com/datastax/astra-assistants-api/blob/main/client/astra_assistants/patch.py#L580

[johnpg82]

@phact So the var to change the url in openai is actually baseURL or the .env var OPENAI_BASE_URL. It seems like your example was going to OpenAI the entire time. But when I update the URL I'm getting a new error. I'm going to try just using the CURL method next. I'll post my update here.

BadRequestError: 400 status code (no body) at APIError.generate (file:///Users/jgarland/Documents/GitHub/ai-api/node_modules/openai/error.mjs:41:20) at OpenAI.makeStatusError (file:///Users/jgarland/Documents/GitHub/ai-api/node_modules/openai/core.mjs:286:25) at OpenAI.makeRequest (file:///Users/jgarland/Documents/GitHub/ai-api/node_modules/openai/core.mjs:330:30) at process.processTicksAndRejections (node:internal/process/task_queues:105:5) at async main (file:///Users/jgarland/Documents/GitHub/ai-api/src/example.js:68:18) { status: 400, headers: { connection: 'keep-alive', 'content-length': '287', 'content-type': 'application/json', date: 'Thu, 14 Nov 2024 22:06:23 GMT' }, request_id: undefined, error: undefined, code: undefined, param: undefined, type: undefined }

[phact]

Thanks! It's entirely possible that the js example may be wrong or outdated.

Maybe sticking a breakpoint inside the SDK might give us some clues. I may be able to have a go at it later tonight.

[johnpg82]

@phact I got it working via Curl and yes the baseUrl just needs updated in the example. Do you have an array of the supported models?

Here is some JS to test all of the different models if it helps. I'd love to get a list of current models so I can know what models I can test with.

import * as dotenv from 'dotenv';
import { fileURLToPath } from 'url';
import { dirname, join } from 'path';

// Configure dotenv with the correct path
const __dirname = dirname(fileURLToPath(import.meta.url));
dotenv.config({ path: join(__dirname, '../.env') });

// Define constants and environment variables
const baseUrl = process.env.OPENAI_BASE_URL || 'https://open-assistant-ai.astra.datastax.com/v1';
const astraApiToken = process.env.ASTRA_DB_APPLICATION_TOKEN;

// Define model keys and API mappings
const modelApiKeys = {
  'openai': {
    apiKey: process.env.OPENAI_API_KEY,
    models: ['gpt-4o', 'gpt-4o-mini', 'gpt-4']
  },
  'anthropic': {
    apiKey: process.env.ANTHROPIC_API_KEY,
    models: ['anthropic.claude-v2']
  },
  'groq': {
    apiKey: process.env.GROQ_API_KEY,
    models: ['groq/llama3-8b-8192']
  },
  'gemini': {
    apiKey: process.env.GEMINI_API_KEY,
    models: ['gemini-model-1', 'gemini-model-2']
  },
  'perplexity': {
    apiKey: process.env.PERPLEXITYAI_API_KEY,
    models: ['perplexity/llama-3.1-sonar-small-128k-online', 'perplexity/llama-3.1-sonar-huge-128k-online']
  },
  'cohere': {
    apiKey: process.env.COHERE_API_KEY,
    models: ['cohere-model-1', 'cohere-model-2']
  }
};

// Test prompt to use for all models
const testPrompt = "What is 2+2? Reply with just the number.";

async function testModel(model) {
  // Find the API key for this model
  let apiKey;
  for (const { apiKey: key, models } of Object.values(modelApiKeys)) {
    if (models.includes(model)) {
      apiKey = key;
      break;
    }
  }

  if (!apiKey) {
    return { model, error: "API key not found" };
  }

  try {
    const response = await fetch(`${baseUrl}/chat/completions`, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'astra-api-token': astraApiToken,
        'Authorization': `Bearer ${apiKey}`,
        'OpenAI-Beta': 'assistants=v2'
      },
      body: JSON.stringify({
        model: model,
        messages: [
          { role: 'system', content: 'You are a helpful assistant.' },
          { role: 'user', content: testPrompt }
        ]
      })
    });

    const data = await response.json();

    if (!response.ok) {
      return {
        model,
        status: response.status,
        error: data
      };
    }

    return {
      model,
      status: response.status,
      message: data.choices[0].message.content
    };
  } catch (error) {
    return {
      model,
      error: error.message
    };
  }
}

async function testAllModels() {
  console.log('Testing all models with prompt:', testPrompt);
  console.log('-----------------------------------');

  // Get all unique models
  const allModels = Object.values(modelApiKeys)
    .flatMap(provider => provider.models);

  // Test each model
  const results = await Promise.all(allModels.map(testModel));

  // Display results in a formatted way
  results.forEach(result => {
    console.log(`\nModel: ${result.model}`);
    if (result.error) {
      console.log('Error:', result.error);
    } else {
      console.log('Status:', result.status);
      console.log('Response:', result.message);
    }
    console.log('-----------------------------------');
  });
}

// Run the tests
testAllModels().catch(error => {
  console.error('Test execution failed:', error);
}); 

[phact]

Everything supported by litellm, I've been pulling them from here:

https://github.com/langflow-ai/langflow/blob/7c048650e00208882fa58f2dee7e32c7c7883de5/src/backend/base/langflow/base/astra_assistants/util.py#L30

[johnpg82]

Thank You! FYI. It looks like the latest version of openai's node library doesn't support overriding the headers.

https://github.com/openai/openai-node/blob/master/src/index.ts

For now I'll use this example to stream.

https://github.com/openai/openai-node/blob/master/examples/stream-to-client-browser.ts

[johnpg82]

Here's' some sample code that will test each of the models.

import * as dotenv from 'dotenv';
import { fileURLToPath } from 'url';
import { dirname, join } from 'path';
import fetch from 'node-fetch';

// Configure dotenv with the correct path
const __dirname = dirname(fileURLToPath(import.meta.url));
dotenv.config({ path: join(__dirname, '../.env') });

const LITELLM_MODELS_URL = 'https://raw.githubusercontent.com/BerriAI/litellm/refs/heads/main/model_prices_and_context_window.json';
const baseUrl = process.env.BASE_URL || 'https://open-assistant-ai.astra.datastax.com/v1';

// Define API key mapping
const providerApiKeys = {
  'openai': process.env.OPENAI_API_KEY,
  'anthropic': process.env.ANTHROPIC_API_KEY,
  'groq': process.env.GROQ_API_KEY,
  'gemini': process.env.GEMINI_API_KEY,
  'perplexity': process.env.PERPLEXITY_API_KEY,
  'cohere': process.env.COHERE_API_KEY,
};

async function fetchModelData() {
    try {
        const response = await fetch(LITELLM_MODELS_URL);
        const data = await response.json();
        return data;
    } catch (error) {
        console.error('Error fetching model data:', error);
        throw error;
    }
}

async function testModel(model, modelData) {
    const modelInfo = modelData[model];
    if (!modelInfo) return null;

    const provider = modelInfo.litellm_provider;
    const apiKey = providerApiKeys[provider];
    if (!apiKey) return null;

    try {
        const response = await fetch(`${baseUrl}/chat/completions`, {
            method: 'POST',
            headers: {
                'Content-Type': 'application/json',
                'astra-api-token': process.env.ASTRA_DB_APPLICATION_TOKEN,
                'Authorization': `Bearer ${apiKey}`
            },
            body: JSON.stringify({
                model: model,
                messages: [
                    { role: 'system', content: 'You are a helpful assistant.' },
                    { role: 'user', content: 'What is 2+2? Reply with just the number.' }
                ]
            })
        });

        const data = await response.json();

        if (!response.ok) return null;

        return {
            model,
            provider,
            maxTokens: modelInfo.max_tokens,
            inputCost: modelInfo.input_cost_per_token,
            outputCost: modelInfo.output_cost_per_token
        };

    } catch (error) {
        return null;
    }
}

async function main() {
    try {
        console.log('Testing models...');
        const modelData = await fetchModelData();

        // Filter for chat models with matching providers
        const eligibleModels = Object.entries(modelData)
            .filter(([_, value]) => 
                value.mode === 'chat' && 
                value.litellm_provider && 
                providerApiKeys[value.litellm_provider]
            )
            .map(([key]) => key);

        // Test all models and collect working ones
        const results = await Promise.all(
            eligibleModels.map(model => testModel(model, modelData))
        );

        // Filter out null results and format for output
        const workingModels = results
            .filter(result => result !== null)
            .reduce((acc, model) => {
                acc[model.provider] = acc[model.provider] || { 
                    apiKey: `process.env.${model.provider.toUpperCase()}_API_KEY`,
                    models: []
                };
                acc[model.provider].models.push(model.model);
                return acc;
            }, {});

        // Output in a format ready to paste into code
        console.log('\nWorking models configuration:');
        console.log('const modelApiKeys = ' + 
            JSON.stringify(workingModels, null, 2)
            .replace(/"process\.env\./g, 'process.env.')
            .replace(/"/g, "'")
        );

    } catch (error) {
        console.error('Execution error:', error);
    }
}

main().catch(console.error);

[phact]

Thanks!

Hmm, I wonder if providing a custom fetch is the way to mess with headers. https://github.com/openai/openai-node/tree/master?tab=readme-ov-file#customizing-the-fetch-client