Closed louis-jan closed 3 months ago
Problem Controllers are too heavy for now with data parsing
void llamaCPP::InferenceImpl( std::shared_ptr<Json::Value> jsonBody, std::function<void(const HttpResponsePtr&)>& callback) { std::string formatted_output = pre_prompt; json data; json stopWords; int no_images = 0; // To set default value if (jsonBody) { // Increase number of chats received and clean the prompt no_of_chats++; if (no_of_chats % clean_cache_threshold == 0) { LOG_INFO << "Clean cache threshold reached!"; llama.kv_cache_clear(); LOG_INFO << "Cache cleaned"; } // Default values to enable auto caching data["cache_prompt"] = caching_enabled; data["n_keep"] = -1; // Passing load value data["repeat_last_n"] = this->repeat_last_n; data["stream"] = (*jsonBody).get("stream", false).asBool(); data["n_predict"] = (*jsonBody).get("max_tokens", 500).asInt(); data["top_p"] = (*jsonBody).get("top_p", 0.95).asFloat(); data["temperature"] = (*jsonBody).get("temperature", 0.8).asFloat(); data["frequency_penalty"] = (*jsonBody).get("frequency_penalty", 0).asFloat(); data["presence_penalty"] = (*jsonBody).get("presence_penalty", 0).asFloat(); const Json::Value& messages = (*jsonBody)["messages"];
Success Criteria Mapping Mechanic
// Define parser template somewhere else namespace myapp{ struct User{ std::string userName; std::string email; std::string address; }; } namespace drogon { template <> inline myapp::User fromRequest(const HttpRequest &req) { auto json = req.getJsonObject(); myapp::User user; if(json) { user.userName = (*json)["name"].asString(); user.email = (*json)["email"].asString(); user.address = (*json)["address"].asString(); } return user; } } // Can use data entity directly from controller // your declaration of processing function maybe like this: void newUser(const HttpRequestPtr &req, std::function<void (const HttpResponsePtr &)> &&callback, myapp::User &&pNewUser) const;
Additional context Add any other context or screenshots about the feature request here.
Completed
Problem Controllers are too heavy for now with data parsing
Success Criteria Mapping Mechanic
Additional context Add any other context or screenshots about the feature request here.