Function, call output error Tool has not been added

devanarchitect commented 1 month ago

I get this error even after the function has successfully executed.

I’m using client.addTool method in the body of the session.update method provided to us by the real-time api client.

It also makes the relay models server disconnect with an error mage saying that the item is was not found.

devanarchitect commented 1 month ago

MarvinBo commented 1 month ago

I fixed this as follows in /relay-server/lib/relay.js:

Replace this code:

// Relay: OpenAI Realtime API Event -> Browser Event
client.realtime.on('server.*', (event) => {
      this.log(`Relaying "${event.type}" to Client`);
      ws.send(JSON.stringify(event));
});

with this:

client.addTool(
  {
    name: 'get_weather',
    description: 'Retrieves the weather for a given lat, lng coordinate pair. Specify a label for the location.',
    parameters: {
      type: 'object',
      properties: {
        lat: {
          type: 'number',
          description: 'Latitude',
        },
        lng: {
          type: 'number',
          description: 'Longitude',
        },
        location: {
          type: 'string',
          description: 'Name of the location',
        },
      },
      required: ['lat', 'lng', 'location'],
    },
  },
  async ({ lat, lng, location }) => {
    const result = await fetch(
      `https://api.open-meteo.com/v1/forecast?latitude=${lat}&longitude=${lng}&current=temperature_2m,wind_speed_10m`
    );
    return result.json();
  }
);

client.addTool(
  {
    name: 'set_memory',
    description: 'Saves important data about the user into memory.',
    parameters: {
      type: 'object',
      properties: {
        key: {
          type: 'string',
          description: 'The key of the memory value. Always use lowercase and underscores, no other characters.',
        },
        value: {
          type: 'string',
          description: 'Value can be anything represented as a string',
        },
      },
      required: ['key', 'value'],
    },
  },
  async ({ key, value }) => {
    // In a real implementation, you'd store this data somewhere
    console.log(`Memory set: ${key} = ${value}`);
    return { ok: true };
  }
);

client.on('conversation.function_call_output', (event) => {
  this.log(`Function call output: ${JSON.stringify(event)}`);
  client.realtime.send('client.conversation.function_call_output', event);
});

// Relay: OpenAI Realtime API Event -> Browser Event
client.realtime.on('server.*', (event) => {
  this.log(`Relaying "${event.type}" to Client`);
  ws.send(JSON.stringify(event));
});

client.on('conversation.*', (event) => {
  this.log(`Relaying conversation event "${event.type}" to Client`);
  ws.send(JSON.stringify(event));
});

bjsllc commented 1 month ago

I've posted the solution here and I've also given credit to those that assisted with the bug. You guys are the real heroes!!!!! https://www.youtube.com/playlist?list=PLkNKgOpMwm0uvvtrnFn1hYrllN-d95FVV

devanarchitect commented 1 month ago

I fixed this as follows in /relay-server/lib/relay.js:

Replace this code:

// Relay: OpenAI Realtime API Event -> Browser Event
client.realtime.on('server.*', (event) => {
      this.log(`Relaying "${event.type}" to Client`);
      ws.send(JSON.stringify(event));
});

with this:

client.addTool(
  {
  name: 'get_weather',
  description: 'Retrieves the weather for a given lat, lng coordinate pair. Specify a label for the location.',
  parameters: {
    type: 'object',
    properties: {
      lat: {
        type: 'number',
        description: 'Latitude',
      },
      lng: {
        type: 'number',
        description: 'Longitude',
      },
      location: {
        type: 'string',
        description: 'Name of the location',
      },
    },
    required: ['lat', 'lng', 'location'],
  },
  },
  async ({ lat, lng, location }) => {
  const result = await fetch(
    `https://api.open-meteo.com/v1/forecast?latitude=${lat}&longitude=${lng}&current=temperature_2m,wind_speed_10m`
  );
  return result.json();
  }
);

client.addTool(
  {
  name: 'set_memory',
  description: 'Saves important data about the user into memory.',
  parameters: {
    type: 'object',
    properties: {
      key: {
        type: 'string',
        description: 'The key of the memory value. Always use lowercase and underscores, no other characters.',
      },
      value: {
        type: 'string',
        description: 'Value can be anything represented as a string',
      },
    },
    required: ['key', 'value'],
  },
  },
  async ({ key, value }) => {
  // In a real implementation, you'd store this data somewhere
  console.log(`Memory set: ${key} = ${value}`);
  return { ok: true };
  }
);

client.on('conversation.function_call_output', (event) => {
  this.log(`Function call output: ${JSON.stringify(event)}`);
  client.realtime.send('client.conversation.function_call_output', event);
});

// Relay: OpenAI Realtime API Event -> Browser Event
client.realtime.on('server.*', (event) => {
  this.log(`Relaying "${event.type}" to Client`);
  if (DEBUG) {
  console.log('Server event data:', JSON.stringify(removeDelta(event), null, 2));
  }
  ws.send(JSON.stringify(event));
});

client.on('conversation.*', (event) => {
  this.log(`Relaying conversation event "${event.type}" to Client`);
  ws.send(JSON.stringify(event));
});

@MarvinBo This solution Didn't work for me, what did you use for debug and removeDelta function and what do you have set for the console Page.

MarvinBo commented 1 month ago

@devanarchitect : I'm very sorry I forgot to remove some debugging code. You do not need the removeDelta function.

The .env vile in the project's root should contain this:

OPENAI_API_KEY=sk-xxx
REACT_APP_LOCAL_RELAY_SERVER_URL=http://localhost:8081

Do not forget to start the relay server with "npm run relay" before you start the console with "npm start".

However, if you are interested in debugging the relay server, here is the complete code of relay-server\lib\relay.js with the debug part. The debug output was too chatty printing a lot of base64 data, so I removed that part with removeDelta():

const DEBUG = true;

import { WebSocketServer } from 'ws';
import { RealtimeClient } from '@openai/realtime-api-beta';

// Function to remove 'delta' property from event object
function removeDelta(obj) {
  const newObj = { ...obj };
  delete newObj.delta;
  delete newObj.audio;
  return newObj;
}

export class RealtimeRelay {
  constructor(apiKey) {
    this.apiKey = apiKey;
    this.sockets = new WeakMap();
    this.wss = null;
  }

  listen(port) {
    this.wss = new WebSocketServer({ port });
    this.wss.on('connection', this.connectionHandler.bind(this));
    this.log(`Listening on ws://localhost:${port}`);
  }

  async connectionHandler(ws, req) {
    if (!req.url) {
      this.log('No URL provided, closing connection.');
      ws.close();
      return;
    }

    const url = new URL(req.url, `http://${req.headers.host}`);
    const pathname = url.pathname;

    if (pathname !== '/') {
      this.log(`Invalid pathname: "${pathname}"`);
      ws.close();
      return;
    }

    // Instantiate new client
    this.log(`Connecting with key "${this.apiKey.slice(0, 3)}..."`);
    const client = new RealtimeClient({ apiKey: this.apiKey });

    // Add this block to register the tools (BUGFIX MB & Claude)
    client.addTool(
      {
        name: 'get_weather',
        description: 'Retrieves the weather for a given lat, lng coordinate pair. Specify a label for the location.',
        parameters: {
          type: 'object',
          properties: {
            lat: {
              type: 'number',
              description: 'Latitude',
            },
            lng: {
              type: 'number',
              description: 'Longitude',
            },
            location: {
              type: 'string',
              description: 'Name of the location',
            },
          },
          required: ['lat', 'lng', 'location'],
        },
      },
      async ({ lat, lng, location }) => {
        const result = await fetch(
          `https://api.open-meteo.com/v1/forecast?latitude=${lat}&longitude=${lng}&current=temperature_2m,wind_speed_10m`
        );
        return result.json();
      }
    );

    client.addTool(
      {
        name: 'set_memory',
        description: 'Saves important data about the user into memory.',
        parameters: {
          type: 'object',
          properties: {
            key: {
              type: 'string',
              description: 'The key of the memory value. Always use lowercase and underscores, no other characters.',
            },
            value: {
              type: 'string',
              description: 'Value can be anything represented as a string',
            },
          },
          required: ['key', 'value'],
        },
      },
      async ({ key, value }) => {
        // In a real implementation, you'd store this data somewhere
        console.log(`Memory set: ${key} = ${value}`);
        return { ok: true };
      }
    );

    client.on('conversation.function_call_output', (event) => {
      this.log(`Function call output: ${JSON.stringify(event)}`);
      client.realtime.send('client.conversation.function_call_output', event);
    });
    // End bugix

    // Relay: OpenAI Realtime API Event -> Browser Event
    client.realtime.on('server.*', (event) => {
      this.log(`Relaying "${event.type}" to Client`);
      if (DEBUG) {
        console.log('Server event data:', JSON.stringify(removeDelta(event), null, 2));
      }
      ws.send(JSON.stringify(event));
    });

    // Bugfix MB & Claude
    client.on('conversation.*', (event) => {
      this.log(`Relaying conversation event "${event.type}" to Client`);
      ws.send(JSON.stringify(event));
    });
    // End bugfix

    client.realtime.on('close', () => ws.close());

    // Relay: Browser Event -> OpenAI Realtime API Event
    // We need to queue data waiting for the OpenAI connection
    const messageQueue = [];
    const messageHandler = (data) => {
      try {
        const event = JSON.parse(data);
        this.log(`Relaying "${event.type}" to OpenAI`);
        if (DEBUG) {
          console.log('Event data:', JSON.stringify(removeDelta(event), null, 2));          
        }
        client.realtime.send(event.type, event);
      } catch (e) {
        console.error(e.message);
        this.log(`Error parsing event from client: ${data}`);
      }
    };

    ws.on('message', (data) => {
      if (!client.isConnected()) {
        messageQueue.push(data);
      } else {
        messageHandler(data);
      }
    });
    ws.on('close', () => client.disconnect());

    // Connect to OpenAI Realtime API
    try {
      this.log(`Connecting to OpenAI...`);
      await client.connect();
    } catch (e) {
      this.log(`Error connecting to OpenAI: ${e.message}`);
      ws.close();
      return;
    }
    this.log(`Connected to OpenAI successfully!`);
    while (messageQueue.length) {
      messageHandler(messageQueue.shift());
    }
  }

  log(...args) {
    console.log(`[RealtimeRelay]`, ...args);
  }
}

devanarchitect commented 1 month ago

@devanarchitect : I'm very sorry I forgot to remove some debugging code. You do not need the removeDelta function.

The .env vile in the project's root should contain this:

OPENAI_API_KEY=sk-xxx
REACT_APP_LOCAL_RELAY_SERVER_URL=http://localhost:8081

Do not forget to start the relay server with "npm run relay" before you start the console with "npm start".

However, if you are interested in debugging the relay server, here is the complete code of relay-server\lib\relay.js with the debug part. The debug output was too chatty printing a lot of base64 data, so I removed that part with removeDelta():

const DEBUG = true;

import { WebSocketServer } from 'ws';
import { RealtimeClient } from '@openai/realtime-api-beta';

// Function to remove 'delta' property from event object
function removeDelta(obj) {
  const newObj = { ...obj };
  delete newObj.delta;
  delete newObj.audio;
  return newObj;
}

export class RealtimeRelay {
  constructor(apiKey) {
    this.apiKey = apiKey;
    this.sockets = new WeakMap();
    this.wss = null;
  }

  listen(port) {
    this.wss = new WebSocketServer({ port });
    this.wss.on('connection', this.connectionHandler.bind(this));
    this.log(`Listening on ws://localhost:${port}`);
  }

  async connectionHandler(ws, req) {
    if (!req.url) {
      this.log('No URL provided, closing connection.');
      ws.close();
      return;
    }

    const url = new URL(req.url, `http://${req.headers.host}`);
    const pathname = url.pathname;

    if (pathname !== '/') {
      this.log(`Invalid pathname: "${pathname}"`);
      ws.close();
      return;
    }

    // Instantiate new client
    this.log(`Connecting with key "${this.apiKey.slice(0, 3)}..."`);
    const client = new RealtimeClient({ apiKey: this.apiKey });

    // Add this block to register the tools (BUGFIX MB & Claude)
    client.addTool(
      {
        name: 'get_weather',
        description: 'Retrieves the weather for a given lat, lng coordinate pair. Specify a label for the location.',
        parameters: {
          type: 'object',
          properties: {
            lat: {
              type: 'number',
              description: 'Latitude',
            },
            lng: {
              type: 'number',
              description: 'Longitude',
            },
            location: {
              type: 'string',
              description: 'Name of the location',
            },
          },
          required: ['lat', 'lng', 'location'],
        },
      },
      async ({ lat, lng, location }) => {
        const result = await fetch(
          `https://api.open-meteo.com/v1/forecast?latitude=${lat}&longitude=${lng}&current=temperature_2m,wind_speed_10m`
        );
        return result.json();
      }
    );

    client.addTool(
      {
        name: 'set_memory',
        description: 'Saves important data about the user into memory.',
        parameters: {
          type: 'object',
          properties: {
            key: {
              type: 'string',
              description: 'The key of the memory value. Always use lowercase and underscores, no other characters.',
            },
            value: {
              type: 'string',
              description: 'Value can be anything represented as a string',
            },
          },
          required: ['key', 'value'],
        },
      },
      async ({ key, value }) => {
        // In a real implementation, you'd store this data somewhere
        console.log(`Memory set: ${key} = ${value}`);
        return { ok: true };
      }
    );

    client.on('conversation.function_call_output', (event) => {
      this.log(`Function call output: ${JSON.stringify(event)}`);
      client.realtime.send('client.conversation.function_call_output', event);
    });
    // End bugix

    // Relay: OpenAI Realtime API Event -> Browser Event
    client.realtime.on('server.*', (event) => {
      this.log(`Relaying "${event.type}" to Client`);
      if (DEBUG) {
        console.log('Server event data:', JSON.stringify(removeDelta(event), null, 2));
      }
      ws.send(JSON.stringify(event));
    });

    // Bugfix MB & Claude
    client.on('conversation.*', (event) => {
      this.log(`Relaying conversation event "${event.type}" to Client`);
      ws.send(JSON.stringify(event));
    });
    // End bugfix

    client.realtime.on('close', () => ws.close());

    // Relay: Browser Event -> OpenAI Realtime API Event
    // We need to queue data waiting for the OpenAI connection
    const messageQueue = [];
    const messageHandler = (data) => {
      try {
        const event = JSON.parse(data);
        this.log(`Relaying "${event.type}" to OpenAI`);
        if (DEBUG) {
          console.log('Event data:', JSON.stringify(removeDelta(event), null, 2));          
        }
        client.realtime.send(event.type, event);
      } catch (e) {
        console.error(e.message);
        this.log(`Error parsing event from client: ${data}`);
      }
    };

    ws.on('message', (data) => {
      if (!client.isConnected()) {
        messageQueue.push(data);
      } else {
        messageHandler(data);
      }
    });
    ws.on('close', () => client.disconnect());

    // Connect to OpenAI Realtime API
    try {
      this.log(`Connecting to OpenAI...`);
      await client.connect();
    } catch (e) {
      this.log(`Error connecting to OpenAI: ${e.message}`);
      ws.close();
      return;
    }
    this.log(`Connected to OpenAI successfully!`);
    while (messageQueue.length) {
      messageHandler(messageQueue.shift());
    }
  }

  log(...args) {
    console.log(`[RealtimeRelay]`, ...args);
  }
}

Thanks I'll give a shot now

openai / openai-realtime-console

Function, call output error Tool has not been added #53