vrr/prima.cpp
2
0
Fork 0
mirror of https://github.com/Lizonghang/prima.cpp.git synced 2025-09-05 20:09:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix n_embd cannot be divided by quantized block size

Browse source
This commit is contained in:
Li, Zonghang 2025-06-03 14:06:31 +04:00
parent 421b3deca5
commit b30f749e5e
2 changed files with 32 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
common/common.cpp
View file

@ -1588,6 +1588,7 @@ struct llama_init_result llama_init_from_gpt_params(gpt_params & params) {
if (n_world == 1) {
uint32_t n_layers = llama_model_n_layers(model);
// assign all layers to this device
params.n_layer_window[0] = n_layers;
cparams.n_layer_window[0] = n_layers;
@ -1596,6 +1597,8 @@ struct llama_init_result llama_init_from_gpt_params(gpt_params & params) {
#if defined(GGML_USE_METAL) || defined(GGML_USE_CUDA)
params.n_gpu_layers = std::min((int32_t)n_layers, params.n_gpu_layers);
cparams.n_gpu_layers = params.n_gpu_layers;
mparams.n_gpu_layers = params.n_gpu_layers;
#endif
} else {

34
common/profiler.cpp
View file

@ -48,6 +48,16 @@
#include <dirent.h>
static int gcd_int(int a, int b) {
while (b != 0) {
int t = b;
b = a % b;
a = t;
}
return a;
}
static size_t get_page_size() {
size_t page_size = 0;
@ -154,8 +164,25 @@ uint32_t device_cpu_cores() {
static float device_flops(struct llama_model * model, enum ggml_type src0t, enum ggml_type src1t, enum profiler_backend_type btype, int n_threads) {
int n_repeat = 1;
int n_embd = std::min(llama_n_embd(model), 4096);
if (btype == PROFILER_BACKEND_TYPE_CPU) n_embd /= 8; // simulate small tensor calculation on cpu
int n_embd = std::min(llama_n_embd(model), 4096);
// simulate small tensor calculation on cpu
if (btype == PROFILER_BACKEND_TYPE_CPU) n_embd /= 8;
// ensure that the block sizes of the tensors are compatible
int bs0 = ggml_blck_size(src0t);
int bs1 = ggml_blck_size(src1t);
int gcd = gcd_int(bs0, bs1);
int lcm = bs0 / gcd * bs1;
if (n_embd % bs0 != 0 || n_embd % bs1 != 0) {
if (n_embd < lcm) {
n_embd = 2 * lcm;
} else {
n_embd = 2 * (n_embd / lcm) * lcm;
}
}
std::vector<float> matrix_A(n_embd * n_embd, 1.0f);
std::vector<float> matrix_B(n_embd * n_embd, 1.0f / n_embd);
@ -188,9 +215,6 @@ static float device_flops(struct llama_model * model, enum ggml_type src0t, enum
};
struct ggml_context * ctx = ggml_init(params);
if(n_embd < ggml_blck_size(src0t)){
n_embd = 2 * ggml_blck_size(src0t);
}
struct ggml_tensor * tensor_a = ggml_new_tensor_2d(ctx, src0t, n_embd, n_embd);
struct ggml_tensor * tensor_b = ggml_new_tensor_2d(ctx, src1t, n_embd, n_embd);