Use apply_chat_template to calculate tokens

[dixyes]

repro

# build it
# use THUDM/glm-4-9b-chat
python /path/to/chatglm.cpp/chatglm_cpp/convert.py \
    -i /path/to/models/THUDM/glm-4-9b-chat -t f16 -o models/glm-4-9b-chat.ggml
# use uvicorn create webapi
export MODEL=models/glm-4-9b-chat.ggml
python3 -m uvicorn chatglm_cpp.openai_api:app --host 0.0.0.0 --port 8000
# curl the api
curl localhost:8000/v1/chat/completions -H'Content-type: application/json' \
    -d '{"messages": [{"role": "user", "content": "帮我写一篇描述LLM基本工作原理的文章，要1000字以上"}]}'

traceback

2024-06-13 09:36:06,850 - httptools_impl - ERROR - Exception in ASGI application
Traceback (most recent call last):
  File "/usr/lib/python3/dist-packages/uvicorn/protocols/http/httptools_impl.py", line 375, in run_asgi
    result = await app(self.scope, self.receive, self.send)
  File "/usr/lib/python3/dist-packages/uvicorn/middleware/proxy_headers.py", line 75, in __call__
    return await self.app(scope, receive, send)
  File "/usr/local/lib/python3.10/dist-packages/fastapi/applications.py", line 1054, in __call__
    await super().__call__(scope, receive, send)
  File "/usr/local/lib/python3.10/dist-packages/starlette/applications.py", line 123, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/usr/local/lib/python3.10/dist-packages/starlette/middleware/errors.py", line 186, in __call__
    raise exc
  File "/usr/local/lib/python3.10/dist-packages/starlette/middleware/errors.py", line 164, in __call__
    await self.app(scope, receive, _send)
  File "/usr/local/lib/python3.10/dist-packages/starlette/middleware/cors.py", line 85, in __call__
    await self.app(scope, receive, send)
  File "/usr/local/lib/python3.10/dist-packages/starlette/middleware/exceptions.py", line 65, in __call__
    await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
  File "/usr/local/lib/python3.10/dist-packages/starlette/_exception_handler.py", line 64, in wrapped_app
    raise exc
  File "/usr/local/lib/python3.10/dist-packages/starlette/_exception_handler.py", line 53, in wrapped_app
    await app(scope, receive, sender)
  File "/usr/local/lib/python3.10/dist-packages/starlette/routing.py", line 756, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/usr/local/lib/python3.10/dist-packages/starlette/routing.py", line 776, in app
    await route.handle(scope, receive, send)
  File "/usr/local/lib/python3.10/dist-packages/starlette/routing.py", line 297, in handle
    await self.app(scope, receive, send)
  File "/usr/local/lib/python3.10/dist-packages/starlette/routing.py", line 77, in app
    await wrap_app_handling_exceptions(app, request)(scope, receive, send)
  File "/usr/local/lib/python3.10/dist-packages/starlette/_exception_handler.py", line 64, in wrapped_app
    raise exc
  File "/usr/local/lib/python3.10/dist-packages/starlette/_exception_handler.py", line 53, in wrapped_app
    await app(scope, receive, sender)
  File "/usr/local/lib/python3.10/dist-packages/starlette/routing.py", line 72, in app
    response = await func(request)
  File "/usr/local/lib/python3.10/dist-packages/fastapi/routing.py", line 278, in app
    raw_response = await run_endpoint_function(
  File "/usr/local/lib/python3.10/dist-packages/fastapi/routing.py", line 191, in run_endpoint_function
    return await dependant.call(**values)
  File "/usr/local/lib/python3.10/dist-packages/chatglm_cpp/openai_api.py", line 212, in create_chat_completion
    prompt_tokens = len(pipeline.tokenizer.encode_messages(messages, max_context_length))
AttributeError: 'chatglm_cpp._C.ChatGLM4Tokenizer' object has no attribute 'encode_messages'. Did you mean: 'decode_message'?

[li-plus]

Thanks! Merging this PR should close #308