anthropics / anthropic-sdk-typescript

Access to Anthropic's safety-first language model APIs
https://www.npmjs.com/package/@anthropic-ai/sdk
MIT License
731 stars 79 forks source link

invalid base64 data for the computer use api #603

Closed zfogg closed 1 week ago

zfogg commented 1 week ago

hello, i am getting this error:

Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.2.content.0.tool_result.content.0.image.source.base64: invalid base64 data"}}

i am programming with the computer use api. when i try to run my screenshot tool and continue the conversation with a tool_result message, i get back this error from the api.

here's the relevant code:

async function processComputerTool(
  content: any,
  messageHistory: MessageHistory
) {
  const action = (content.input as any).action;
  console.log("computer:action", action);

  if (action === "screenshot") {
    try {
      const screenshot = await takeScreenshot();
      const base64Data = screenshot;

      messageHistory.push({
        role: 'user',
        content: [{
          type: "tool_result",
          tool_use_id: content.id,
          content: [{
            type: "image",
            source: {
              type: "base64",
              media_type: "image/png",
              data: base64Data,
            },
          }],
        }],
      });
    } catch (error) {
      // Handle screenshot errors
      messageHistory.push({
        role: 'user',
        content: [{
          type: "tool_result",
          tool_use_id: content.id,
          content: `Error taking screenshot: ${error instanceof Error ? error.message : 'Unknown error'}`,
          is_error: true,
        }],
      });
    }
  }
}

async function takeScreenshot(): Promise<string> {
  try {
    // Execute PowerShell command to take screenshot via SSH
    const command = `powershell.exe -Command "& {ssh claude-testing-do 'DISPLAY=:0 scrot -o /tmp/screenshot.png && cat /tmp/screenshot.png | base64 && rm /tmp/screenshot.png'}"`;

    const { exec } = await import('child_process');

    return new Promise((resolve, reject) => {
      exec(command, { maxBuffer: 50 * 1024 * 1024 }, (error, stdout, stderr) => {
        if (error) {
          reject(new Error(`Screenshot failed: ${error.message}`));
          return;
        }
        if (stderr) {
          reject(new Error(`Screenshot failed: ${stderr}`));
          return;
        }
        resolve(stdout.toString().trim().replace(/\n+$/, ''));
      });
    });
  } catch (error) {
    if (error instanceof Error) {
      throw new Error(`Failed to take screenshot: ${error.message}`);
    }
    throw new Error('Failed to take screenshot: Unknown error');
  }
}
zfogg commented 1 week ago

i would like to first comment that my base64 data is valid. i validated it. but you can see in the takeScreenshot function that i just run scrot and cat it to base64 over ssh

zfogg commented 1 week ago

fixed by changing cat file.png | base64 to cat file.png | base64 -w

liamcharmer commented 1 week ago

I'm getting the same error when uploading via javascript. My base64 data is also valid but I get the same error. What does the -w symbol do?