Ollama LLaVa Model

[inzomiac]

I'm testing image processing using ollama model llava, seems that it can't handle image processing right now. Is there any plans to implement image processing besides gpt-4-vison-preview? also when adding ollama file handler doesn't recognive that llama can handle imageInput:

So I modified fetch-models.ts

export const fetchOllamaModels = async () => {
  try {
    const response = await fetch(
      process.env.NEXT_PUBLIC_OLLAMA_URL + "/api/tags"
    )

    if (!response.ok) {
      throw new Error(`Ollama server is not responding.`)
    }

    const data = await response.json()

    const localModels: LLM[] = data.models.map((model: any) => ({
      modelId: model.name as LLMID,
      modelName: model.name,
      provider: "ollama",
      hostedId: model.name,
      platformLink: "https://ollama.ai/library",
      imageInput: model.details.families ? model.details.families.includes("clip") : false
    }))

    localModels.forEach((ollama: LLM) => {
      const isDuplicate = OLLAMA_LLM_LIST.some((item) => item.modelId === ollama.modelId);
      if (!isDuplicate) {
        OLLAMA_LLM_LIST.push(ollama);
      }
    })
    LLM_LIST_MAP['ollama'] = OLLAMA_LLM_LIST
    return OLLAMA_LLM_LIST
  } catch (error) {
    console.warn("Error fetching Ollama models: " + error)
  }
}

added ollama-llm-list.ts in lib/models/llm

import { LLM } from "@/types"

const OPENAI_PLATORM_LINK = "https://ollama.ai/library"

export const OLLAMA_LLM_LIST: LLM[] = []

modified llm-list.ts

import { LLM } from "@/types"
import { ANTHROPIC_LLM_LIST } from "./anthropic-llm-list"
import { GOOGLE_LLM_LIST } from "./google-llm-list"
import { MISTRAL_LLM_LIST } from "./mistral-llm-list"
import { OPENAI_LLM_LIST } from "./openai-llm-list"
import { OLLAMA_LLM_LIST } from "./ollama-llm-list"
import { PERPLEXITY_LLM_LIST } from "./perplexity-llm-list"

export const LLM_LIST: LLM[] = [
  ...OPENAI_LLM_LIST,
  ...GOOGLE_LLM_LIST,
  ...MISTRAL_LLM_LIST,
  ...PERPLEXITY_LLM_LIST,
  ...ANTHROPIC_LLM_LIST,
  ...OLLAMA_LLM_LIST
]

export const LLM_LIST_MAP: Record<string, LLM[]> = {
  openai: OPENAI_LLM_LIST,
  azure: OPENAI_LLM_LIST,
  google: GOOGLE_LLM_LIST,
  mistral: MISTRAL_LLM_LIST,
  perplexity: PERPLEXITY_LLM_LIST,
  anthropic: ANTHROPIC_LLM_LIST,
  ollama: OLLAMA_LLM_LIST
}

modifield use-select-file-handler.tsx

  const handleFilesToAccept = () => {
    const model = chatSettings?.model
    let FULL_MODEL
    for (let key in LLM_LIST_MAP) {
      FULL_MODEL = LLM_LIST_MAP[key].find(llm => llm.modelId === model)
      if (FULL_MODEL) {
        break
      }
    }
    if (!FULL_MODEL) return

    setFilesToAccept(
      FULL_MODEL.imageInput
        ? `${ACCEPTED_FILE_TYPES},image/*`
        : ACCEPTED_FILE_TYPES
    )
  }

now my next issue is routing the message and image to ollama api /api/generate. Maybe someone is also doing this I just need a few help.

[Taikono-Himazin]

Any updates on this?