vrr/prima.cpp
2
0
Fork 0
mirror of https://github.com/Lizonghang/prima.cpp.git synced 2025-09-16 01:19:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add arg --cuda-mem

Browse source
This commit is contained in:
Zonghang Li 2025-01-16 09:15:34 +04:00
parent dab6b2e1c2
commit 46e99218b4
5 changed files with 19 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

1
common/common.h
View file

@ -148,6 +148,7 @@ struct gpt_params {
std::string master_ip = "localhost"; // ip address of the master node
std::string next_node_ip = "localhost"; // ip address of my next node
bool unload = false; // unload layer weights after use or not
int32_t cuda_mem = 999.0; // cuda memory to use, in GiB
int32_t n_predict = -1; // new tokens to predict
int32_t n_ctx = 0; // context size
int32_t n_batch = 2048; // logical batch size for prompt processing (must be >=32 to use BLAS)