cui-llama.rn build error on iOS

[Strider-Alex]

The building of cui-llama.rn seems to be broken in ios/Mac. Since cui-llama.rn doesn't have a issue section I'll leave it here.

The error:

❌  (node_modules/cui-llama.rn/ios/RNLlamaContext.mm:8:5)

   6 | + (instancetype)initWithParams:(NSDictionary *)params {
   7 |     // llama_backend_init(false);
>  8 |     gpt_params defaultParams;
     |     ^ unknown type name 'gpt_params'; did you mean 'cpu_params'?
   9 | 
  10 |     NSString *modelPath = params[@"model"];
  11 |     BOOL isAsset = [params[@"is_model_asset"] boolValue];

❌  (node_modules/cui-llama.rn/ios/RNLlamaContext.mm:14:19)

  12 |     NSString *path = modelPath;
  13 |     if (isAsset) path = [[NSBundle mainBundle] pathForResource:modelPath ofType:nil];
> 14 |     defaultParams.model = [path UTF8String];
     |                   ^ no member named 'model' in 'cpu_params'
  15 | 
  16 |     if (params[@"embedding"] && [params[@"embedding"] boolValue]) {
  17 |         defaultParams.embedding = true;

❌  (node_modules/cui-llama.rn/ios/RNLlamaContext.mm:17:23)

  15 | 
  16 |     if (params[@"embedding"] && [params[@"embedding"] boolValue]) {
> 17 |         defaultParams.embedding = true;
     |                       ^ no member named 'embedding' in 'cpu_params'
  18 |     }
  19 | 
  20 |     if (params[@"n_ctx"]) defaultParams.n_ctx = [params[@"n_ctx"] intValue];

❌  (node_modules/cui-llama.rn/ios/RNLlamaContext.mm:20:41)

  18 |     }
  19 | 
> 20 |     if (params[@"n_ctx"]) defaultParams.n_ctx = [params[@"n_ctx"] intValue];
     |                                         ^ no member named 'n_ctx' in 'cpu_params'
  21 |     if (params[@"use_mlock"]) defaultParams.use_mlock = [params[@"use_mlock"]boolValue];
  22 | 
  23 |     BOOL isMetalEnabled = false;

❌  (node_modules/cui-llama.rn/ios/RNLlamaContext.mm:21:45)

  19 | 
  20 |     if (params[@"n_ctx"]) defaultParams.n_ctx = [params[@"n_ctx"] intValue];
> 21 |     if (params[@"use_mlock"]) defaultParams.use_mlock = [params[@"use_mlock"]boolValue];
     |                                             ^ no member named 'use_mlock' in 'cpu_params'
  22 | 
  23 |     BOOL isMetalEnabled = false;
  24 |     NSString *reasonNoMetal = @"";

❌  (node_modules/cui-llama.rn/ios/RNLlamaContext.mm:25:19)

  23 |     BOOL isMetalEnabled = false;
  24 |     NSString *reasonNoMetal = @"";
> 25 |     defaultParams.n_gpu_layers = 0;
     |                   ^ no member named 'n_gpu_layers' in 'cpu_params'
  26 |     if (params[@"n_gpu_layers"] && [params[@"n_gpu_layers"] intValue] > 0) {
  27 | #ifdef LM_GGML_USE_METAL
  28 |         // Check ggml-metal availability

❌  (node_modules/cui-llama.rn/ios/RNLlamaContext.mm:46:31)

  44 |                 reasonNoMetal = [error localizedDescription];
  45 |             } else {
> 46 |                 defaultParams.n_gpu_layers = [params[@"n_gpu_layers"] intValue];
     |                               ^ no member named 'n_gpu_layers' in 'cpu_params'
  47 |                 isMetalEnabled = true;
  48 |             }
  49 |         }

❌  (node_modules/cui-llama.rn/ios/RNLlamaContext.mm:56:43)

  54 | #endif
  55 |     }
> 56 |     if (params[@"n_batch"]) defaultParams.n_batch = [params[@"n_batch"] intValue];
     |                                           ^ no member named 'n_batch' in 'cpu_params'
  57 |     if (params[@"use_mmap"]) defaultParams.use_mmap = [params[@"use_mmap"] boolValue];
  58 | 
  59 |     if (params[@"lora"]) {

❌  (node_modules/cui-llama.rn/ios/RNLlamaContext.mm:57:44)

  55 |     }
  56 |     if (params[@"n_batch"]) defaultParams.n_batch = [params[@"n_batch"] intValue];
> 57 |     if (params[@"use_mmap"]) defaultParams.use_mmap = [params[@"use_mmap"] boolValue];
     |                                            ^ no member named 'use_mmap' in 'cpu_params'
  58 | 
  59 |     if (params[@"lora"]) {
  60 |         float lora_scaled = 1.0f;

❌  (node_modules/cui-llama.rn/ios/RNLlamaContext.mm:62:23)

  60 |         float lora_scaled = 1.0f;
  61 |         if (params[@"lora_scaled"]) lora_scaled = [params[@"lora_scaled"] floatValue];
> 62 |         defaultParams.lora_adapter.push_back({[params[@"lora"] UTF8String], lora_scaled});
     |                       ^ no member named 'lora_adapter' in 'cpu_params'
  63 |         defaultParams.use_mmap = false;
  64 |     }
  65 | 

❌  (node_modules/cui-llama.rn/ios/RNLlamaContext.mm:63:23)

  61 |         if (params[@"lora_scaled"]) lora_scaled = [params[@"lora_scaled"] floatValue];
  62 |         defaultParams.lora_adapter.push_back({[params[@"lora"] UTF8String], lora_scaled});
> 63 |         defaultParams.use_mmap = false;
     |                       ^ no member named 'use_mmap' in 'cpu_params'
  64 |     }
  65 | 
  66 |     if (params[@"rope_freq_base"]) defaultParams.rope_freq_base = [params[@"rope_freq_base"] floatValue];

❌  (node_modules/cui-llama.rn/ios/RNLlamaContext.mm:66:50)

  64 |     }
  65 | 
> 66 |     if (params[@"rope_freq_base"]) defaultParams.rope_freq_base = [params[@"rope_freq_base"] floatValue];
     |                                                  ^ no member named 'rope_freq_base' in 'cpu_params'
  67 |     if (params[@"rope_freq_scale"]) defaultParams.rope_freq_scale = [params[@"rope_freq_scale"] floatValue];
  68 | 
  69 |     if (params[@"seed"]) defaultParams.seed = [params[@"seed"] intValue];

❌  (node_modules/cui-llama.rn/ios/RNLlamaContext.mm:67:51)

  65 | 
  66 |     if (params[@"rope_freq_base"]) defaultParams.rope_freq_base = [params[@"rope_freq_base"] floatValue];
> 67 |     if (params[@"rope_freq_scale"]) defaultParams.rope_freq_scale = [params[@"rope_freq_scale"] floatValue];
     |                                                   ^ no member named 'rope_freq_scale' in 'cpu_params'
  68 | 
  69 |     if (params[@"seed"]) defaultParams.seed = [params[@"seed"] intValue];
  70 | 

❌  (node_modules/cui-llama.rn/ios/RNLlamaContext.mm:69:40)

  67 |     if (params[@"rope_freq_scale"]) defaultParams.rope_freq_scale = [params[@"rope_freq_scale"] floatValue];
  68 | 
> 69 |     if (params[@"seed"]) defaultParams.seed = [params[@"seed"] intValue];
     |                                        ^ no member named 'seed' in 'cpu_params'
  70 | 
  71 |     int nThreads = params[@"n_threads"] ? [params[@"n_threads"] intValue] : 0;
  72 |     const int maxThreads = (int) [[NSProcessInfo processInfo] processorCount];

❌  (node_modules/cui-llama.rn/ios/RNLlamaContext.mm:81:58)

  79 |         context->llama = new rnllama::llama_rn_context();
  80 |     }
> 81 |     context->is_model_loaded = context->llama->loadModel(defaultParams);
     |                                                          ^ non-const lvalue reference to type 'common_params' cannot bind to a value of unrelated type 'cpu_params'
  82 |     context->is_metal_enabled = isMetalEnabled;
  83 |     context->reason_no_metal = reasonNoMetal;
  84 | 

❌  (node_modules/cui-llama.rn/ios/RNLlamaContext.mm:131:15)

  129 | 
  130 | - (NSString *)getFormattedChat:(NSArray *)messages withTemplate:(NSString *)chatTemplate {
> 131 |   std::vector<llama_chat_msg> chat;
      |               ^ unknown type name 'llama_chat_msg'; did you mean 'llama_chat_message'?
  132 | 
  133 |   for (NSDictionary *msg in messages) {
  134 |     std::string role = [[msg objectForKey:@"role"] UTF8String];

❌  (node_modules/cui-llama.rn/ios/RNLlamaContext.mm:136:10)

  134 |     std::string role = [[msg objectForKey:@"role"] UTF8String];
  135 |     std::string content = [[msg objectForKey:@"content"] UTF8String];
> 136 |     chat.push_back({ role, content });
      |          ^ no matching member function for call to 'push_back'
  137 |   }
  138 | 
  139 |   auto tmpl = chatTemplate == nil ? "" : [chatTemplate UTF8String];

❌  (node_modules/cui-llama.rn/ios/RNLlamaContext.mm:140:25)

  138 | 
  139 |   auto tmpl = chatTemplate == nil ? "" : [chatTemplate UTF8String];
> 140 |   auto formatted_chat = llama_chat_apply_template(llama->model, tmpl, chat, true);
      |                         ^ no matching function for call to 'llama_chat_apply_template'
  141 |   return [NSString stringWithUTF8String:formatted_chat.c_str()];
  142 | }
  143 | 

❌  (node_modules/cui-llama.rn/ios/RNLlamaContext.mm:171:5)

  169 |     llama->rewind();
  170 | 
> 171 |     llama_reset_timings(llama->ctx);
      |     ^ use of undeclared identifier 'llama_reset_timings'
  172 | 
  173 |     NSString *prompt = [params objectForKey:@"prompt"];
  174 | 

❌  fatal error: too many errors emitted, stopping now [-ferror-limit=]

› Compiling cui-llama.rn Pods/llama-rn » RNLlama.mm
› Compiling expo-keep-awake Pods/ExpoKeepAwake » ExpoKeepAwake-dummy.m

› 20 error(s), and 0 warning(s)

CommandError: Failed to build iOS project. "xcodebuild" exited with error code 65.

[Vali-98]

Unfortunately I don't have the means to properly update and test cui-llama.rn on IOS, as I lack any apple devices.

There is a PR on the original llama.rn with IOS sync'd with the latest llama.cpp.

https://github.com/mybigday/llama.rn/pull/79

However, it lacks the new features I added to llama.rn like querying cpu features, sync tokenize and the progress callback.

[Strider-Alex]

I'd like to bring new features from cui-llama.rn to iOS. Could you give me some ideas on where to start?

[Vali-98]

I'd like to bring new features from cui-llama.rn to iOS. Could you give me some ideas on where to start?

First it might be wise to familiarize yourself with native modules in react-native:

https://reactnative.dev/docs/0.72/native-modules-ios

You probably want to look at the c++ -> swift adapter such as this: https://github.com/mybigday/llama.rn/blob/3ee23e4f69c6ce67249ceb3f5e8551730799214c/ios/RNLlamaContext.mm

I think if that .mm file was updated with the new fields it would work.

Also, you will need to re-run pod install based on the llama.rn readme.

I am not an IOS dev, so I can't really help any further I will be closing the issue, but feel free to continue asking questions.