Thanks for great llm-inference engine. I made Nim bindings for it with help of Llama 405b.

[JohnClaw]

Hi. I used Llama 405b (https://cloud.sambanova.ai/) to make Nim bindings for chatllm.cpp. So llama 405b generated two files: main.nim and libchatllm.nim. To compile main.exe you should install nim from official site (https://nim-lang.org/install_windows.html). After installing Nim you should put main.nim, libchatllm.nim, libchatllm.dll and ggml.dll into the folder where your nim.exe is (C:\Nim\bin, for example). Then create .bat file with following command inside of it: nim.exe -d:release c main.nim and execute this bat. Main.nim and libchatllm.nim can be found in attached .zip archive. Nim_bindings_for_chatllm.cpp.zip

[JohnClaw]

Code of main.nim:

import system
import strutils
import os
import libchatllm

proc chatllm_print(user_data: pointer, print_type: PrintType, utf8_str: cstring) {.cdecl.} =
  case print_type
  of PRINT_CHAT_CHUNK:
    stdout.write(utf8_str)
  else:
    echo utf8_str
  stdout.flushFile()

proc chatllm_end(user_data: pointer) {.cdecl.} =
  echo ""

proc main() =
  let chat = chatllm_create()
  for i in 1 .. paramCount():
    chatllm_append_param(chat, paramStr(i).cstring)

  let r = chatllm_start(chat, chatllm_print, chatllm_end, nil)
  if r != 0:
    echo ">>> chatllm_start error: ", r
    quit(r)

  while true:
    stdout.write("You  > ")
    let input = stdin.readLine()
    if input.isEmptyOrWhitespace(): continue

    stdout.write("A.I. > ")
    let r = chatllm_user_input(chat, input.cstring)
    if r != 0:
      echo ">>> chatllm_user_input error: ", r
      break

main()

[JohnClaw]

Code of libchatllm.nim:

type
  PrintType* = enum
    PRINT_CHAT_CHUNK = 0,
    PRINTLN_META = 1,
    PRINTLN_ERROR = 2,
    PRINTLN_REF = 3,
    PRINTLN_REWRITTEN_QUERY = 4,
    PRINTLN_HISTORY_USER = 5,
    PRINTLN_HISTORY_AI = 6,
    PRINTLN_TOOL_CALLING = 7,
    PRINTLN_EMBEDDING = 8,
    PRINTLN_RANKING = 9,
    PRINTLN_TOKEN_IDS = 10

  ChatllmObj* = object

  ChatllmPrintProc* = proc (user_data: pointer, print_type: PrintType, utf8_str: cstring) {.cdecl.}
  ChatllmEndProc* = proc (user_data: pointer) {.cdecl.}

const
  libName = "libchatllm.dll"

proc chatllm_create*(): ptr ChatllmObj {.cdecl, dynlib: libName, importc.}

proc chatllm_append_param*(obj: ptr ChatllmObj, utf8_str: cstring) {.cdecl, dynlib: libName, importc.}

proc chatllm_start*(obj: ptr ChatllmObj, f_print: ChatllmPrintProc, f_end: ChatllmEndProc, user_data: pointer): cint {.cdecl, dynlib: libName, importc.}

proc chatllm_set_gen_max_tokens*(obj: ptr ChatllmObj, gen_max_tokens: cint) {.cdecl, dynlib: libName, importc.}

proc chatllm_restart*(obj: ptr ChatllmObj, utf8_sys_prompt: cstring) {.cdecl, dynlib: libName, importc.}

proc chatllm_user_input*(obj: ptr ChatllmObj, utf8_str: cstring): cint {.cdecl, dynlib: libName, importc.}

proc chatllm_set_ai_prefix*(obj: ptr ChatllmObj, utf8_str: cstring): cint {.cdecl, dynlib: libName, importc.}

proc chatllm_tool_input*(obj: ptr ChatllmObj, utf8_str: cstring): cint {.cdecl, dynlib: libName, importc.}

proc chatllm_tool_completion*(obj: ptr ChatllmObj, utf8_str: cstring): cint {.cdecl, dynlib: libName, importc.}

proc chatllm_text_tokenize*(obj: ptr ChatllmObj, utf8_str: cstring): cint {.cdecl, dynlib: libName, importc.}

proc chatllm_text_embedding*(obj: ptr ChatllmObj, utf8_str: cstring): cint {.cdecl, dynlib: libName, importc.}

proc chatllm_qa_rank*(obj: ptr ChatllmObj, utf8_str_q: cstring, utf8_str_a: cstring): cint {.cdecl, dynlib: libName, importc.}

proc chatllm_rag_select_store*(obj: ptr ChatllmObj, name: cstring): cint {.cdecl, dynlib: libName, importc.}

proc chatllm_abort_generation*(obj: ptr ChatllmObj) {.cdecl, dynlib: libName, importc.}

proc chatllm_show_statistics*(obj: ptr ChatllmObj) {.cdecl, dynlib: libName, importc.}

proc chatllm_save_session*(obj: ptr ChatllmObj, utf8_str: cstring): cint {.cdecl, dynlib: libName, importc.}

proc chatllm_load_session*(obj: ptr ChatllmObj, utf8_str: cstring): cint {.cdecl, dynlib: libName, importc.}

[foldl]

Nice. Could you make a PR for this?

A question: could you use this inference engine and another smaller model to generate the Nim code? Then, it seems that we are bootstrapping, :).

[JohnClaw]

Nice. Could you make a PR for this?

I'm sorry, i'm not a tech guy and English is not my native language. Due to these facts github interface isn't user-friendly for me. I don't know how to open PR.

[JohnClaw]

A question: could you use this inference engine and another smaller model to generate the Nim code? Then, it seems that we are bootstrapping

Small models are too weak for coding tasks. Even Qwen 2.5 Coder Instruct 32b failed my request to create properly working Nim bindings for chatllm.cpp. Only giant Llama 405b solved the problem correctly.