vrr/prima.cpp
2
0
Fork 0
mirror of https://github.com/Lizonghang/prima.cpp.git synced 2025-09-10 11:24:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add f32, f16, q8, q4k speed test for cuda

Browse source
This commit is contained in:
Lizonghang 2024-11-10 23:41:13 +04:00
parent f4260bb346
commit 10f6f92c7e
3 changed files with 59 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/llama.cpp
View file

@ -3582,7 +3582,10 @@ void llama_profile_device(device_info * dev_info, struct llama_model * model, co
dev_info->gpu_props.memory_free = round(gpu_props.memory_free / (double)(1 << 30) * 100) / 100;
dev_info->gpu_props.memory_total = round(gpu_props.memory_total / (double)(1 << 30) * 100) / 100;
dev_info->gpu_props.metal_flops = device_metal_flops(model, GGML_TYPE_F32);
dev_info->gpu_props.cuda_flops = device_cuda_flops(model, GGML_TYPE_F32);
dev_info->gpu_props.cuda_flops_f32 = device_cuda_flops(model, GGML_TYPE_F32);
dev_info->gpu_props.cuda_flops_f16 = device_cuda_flops(model, GGML_TYPE_F16);
dev_info->gpu_props.cuda_flops_q8 = device_cuda_flops(model, GGML_TYPE_Q8_0);
dev_info->gpu_props.cuda_flops_q4k = device_cuda_flops(model, GGML_TYPE_Q4_K);
}
ggml_backend_buffer_type_t llama_dev_buffer_type(struct llama_model * model, int device) {