janhq / cortex.cpp

Run and customize Local LLMs.
https://cortex.so
Apache License 2.0
1.94k stars 108 forks source link

feat: mapping mechanism from the HttpRequestPtr object to entity #453

Closed louis-jan closed 3 months ago

louis-jan commented 7 months ago

Problem Controllers are too heavy for now with data parsing

void llamaCPP::InferenceImpl(
    std::shared_ptr<Json::Value> jsonBody,
    std::function<void(const HttpResponsePtr&)>& callback) {
  std::string formatted_output = pre_prompt;

  json data;
  json stopWords;
  int no_images = 0;
  // To set default value

  if (jsonBody) {
    // Increase number of chats received and clean the prompt
    no_of_chats++;
    if (no_of_chats % clean_cache_threshold == 0) {
      LOG_INFO << "Clean cache threshold reached!";
      llama.kv_cache_clear();
      LOG_INFO << "Cache cleaned";
    }

    // Default values to enable auto caching
    data["cache_prompt"] = caching_enabled;
    data["n_keep"] = -1;

    // Passing load value
    data["repeat_last_n"] = this->repeat_last_n;

    data["stream"] = (*jsonBody).get("stream", false).asBool();
    data["n_predict"] = (*jsonBody).get("max_tokens", 500).asInt();
    data["top_p"] = (*jsonBody).get("top_p", 0.95).asFloat();
    data["temperature"] = (*jsonBody).get("temperature", 0.8).asFloat();
    data["frequency_penalty"] =
        (*jsonBody).get("frequency_penalty", 0).asFloat();
    data["presence_penalty"] = (*jsonBody).get("presence_penalty", 0).asFloat();
    const Json::Value& messages = (*jsonBody)["messages"];

Success Criteria Mapping Mechanic

// Define parser template somewhere else
namespace myapp{
struct User{
    std::string userName;
    std::string email;
    std::string address;
};
}
namespace drogon
{
template <>
inline myapp::User fromRequest(const HttpRequest &req)
{
    auto json = req.getJsonObject();
    myapp::User user;
    if(json)
    {
        user.userName = (*json)["name"].asString();
        user.email = (*json)["email"].asString();
        user.address = (*json)["address"].asString();
    }
    return user;
}

}

// Can use data entity directly from controller
// your declaration of processing function maybe like this:
 void newUser(const HttpRequestPtr &req,
        std::function<void (const HttpResponsePtr &)> &&callback,
        myapp::User &&pNewUser) const;

Additional context Add any other context or screenshots about the feature request here.

louis-jan commented 3 months ago

Completed