vrr/prima.cpp
2
0
Fork 0
mirror of https://github.com/Lizonghang/prima.cpp.git synced 2025-09-10 03:44:34 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

improve the computing buffer estimate

Browse source
This commit is contained in:
Zonghang Li 2025-06-19 08:02:43 +00:00
parent 0b4ffdfde5
commit dd589561b4
8 changed files with 87 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

5
include/llama.h
View file

@ -327,6 +327,7 @@ extern "C" {
uint32_t n_cycles; // number of cycles to output one token
bool prefetch; // whether to prefetch layer weights
bool force; // force to start prefetching after computation
float master_priority; // priority to assign workload to the master (set 1.01 to use master first, and 0.99 to offload to other devices)
bool keep_out_in_metal; // whether to keep output weights in metal memory
char * master_ip; // ip address of the master node
char * next_node_ip; // ip address of the next node
@ -575,8 +576,8 @@ extern "C" {
const struct llama_context_params cparams,
bool use_gpu,
bool is_master,
int n_layers,
int n_gpu_layers);
struct model_bytes n_bytes,
bool offload);
// Return the size of KV cache in the model
LLAMA_API void llama_total_kv_size(