vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-11 01:24:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

merge new rope param nonsense

Browse source
This commit is contained in:
Concedo 2023-09-30 11:33:30 +08:00
commit b84e210f0d
38 changed files with 2811 additions and 764 deletions
Download patch file Download diff file Expand all files Collapse all files

12
otherarch/neox_v3.cpp
View file

@ -513,9 +513,17 @@ bool gpt_neox_eval(
struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, n_embd/n_head, n_head, N, cur->nb[1]/n_head, cur->nb[1], 1*sizeof(float)*n_embd/n_head));
struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, n_embd/n_head, n_head, N, cur->nb[1]/n_head, cur->nb[1], 2*sizeof(float)*n_embd/n_head));
struct ggml_tensor * KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
{
int * data = (int *) KQ_pos->data;
for (int i = 0; i < N; ++i) {
data[i] = n_past + i;
}
}
// using mode = 2 for GPT-NeoX mode
Qcur = ggml_rope_custom_inplace(ctx0, Qcur, n_past, n_rot, 2, n_ctx, freq_base, freq_scale);
Kcur = ggml_rope_custom_inplace(ctx0, Kcur, n_past, n_rot, 2, n_ctx, freq_base, freq_scale);
Qcur = ggml_rope_custom_inplace(ctx0, Qcur, KQ_pos, n_rot, 2, n_ctx, freq_base, freq_scale);
Kcur = ggml_rope_custom_inplace(ctx0, Kcur, KQ_pos, n_rot, 2, n_ctx, freq_base, freq_scale);
// store key and value to memory
{