vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-11 09:34:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

embeddings change to encode

Browse source
This commit is contained in:
Concedo 2025-05-28 23:24:33 +08:00
parent 26bf5b446d
commit fcc1b43c06
1 changed files with 8 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

20
otherarch/embeddings_adapter.cpp
View file

@ -34,7 +34,7 @@ static void batch_add_seq(llama_batch & batch, const std::vector<int32_t> & toke
} }
} }
static void batch_decode(llama_context * ctx, llama_batch & batch, float * output, int n_seq, int n_embd, int embd_norm) { static void batch_encode(llama_context * ctx, llama_batch & batch, float * output, int n_seq, int n_embd, int embd_norm) {
const enum llama_pooling_type pooling_type = llama_pooling_type(ctx); const enum llama_pooling_type pooling_type = llama_pooling_type(ctx);
const struct llama_model * model = llama_get_model(ctx); const struct llama_model * model = llama_get_model(ctx);
@ -46,16 +46,10 @@ static void batch_decode(llama_context * ctx, llama_batch & batch, float * outpu
{ {
printf("\n%s: n_tokens = %d, n_seq = %d\n", __func__, batch.n_tokens, n_seq); printf("\n%s: n_tokens = %d, n_seq = %d\n", __func__, batch.n_tokens, n_seq);
} }
if (llama_model_has_encoder(model) && !llama_model_has_decoder(model)) {
// encoder-only model // run model
if (llama_encode(ctx, batch) < 0) { if (llama_encode(ctx, batch) < 0) {
printf("\n%s : failed to encode\n", __func__); printf("%s : failed to process\n", __func__);
}
} else if (!llama_model_has_encoder(model) && llama_model_has_decoder(model)) {
// decoder-only model
if (llama_decode(ctx, batch) < 0) {
printf("\n%s : failed to decode\n", __func__);
}
} }
for (int i = 0; i < batch.n_tokens; i++) { for (int i = 0; i < batch.n_tokens; i++) {
@ -150,7 +144,7 @@ bool embeddingstype_load_model(const embeddings_load_model_inputs inputs)
std::vector<int> tmp = {1, 2, 3, 4}; std::vector<int> tmp = {1, 2, 3, 4};
llama_kv_self_clear(embeddings_ctx); llama_kv_self_clear(embeddings_ctx);
auto er = llama_decode(embeddings_ctx, llama_batch_get_one(tmp.data(), tmp.size())); auto er = llama_encode(embeddings_ctx, llama_batch_get_one(tmp.data(), tmp.size()));
if(er!=0) if(er!=0)
{ {
printf("\nEmbeddings Model Eval returned nonzero: %d\n",er); printf("\nEmbeddings Model Eval returned nonzero: %d\n",er);
@ -265,7 +259,7 @@ embeddings_generation_outputs embeddingstype_generate(const embeddings_generatio
// encode if at capacity // encode if at capacity
if (batch.n_tokens + n_toks > n_batch) { if (batch.n_tokens + n_toks > n_batch) {
float * out = emb + e * n_embd; float * out = emb + e * n_embd;
batch_decode(embeddings_ctx, batch, out, s, n_embd, embd_normalize); batch_encode(embeddings_ctx, batch, out, s, n_embd, embd_normalize);
e += pooling_type == LLAMA_POOLING_TYPE_NONE ? batch.n_tokens : s; e += pooling_type == LLAMA_POOLING_TYPE_NONE ? batch.n_tokens : s;
s = 0; s = 0;
common_batch_clear(batch); common_batch_clear(batch);
@ -277,7 +271,7 @@ embeddings_generation_outputs embeddingstype_generate(const embeddings_generatio
// final batch // final batch
float * out = emb + e * n_embd; float * out = emb + e * n_embd;
batch_decode(embeddings_ctx, batch, out, s, n_embd, embd_normalize); batch_encode(embeddings_ctx, batch, out, s, n_embd, embd_normalize);
std::string outputarray = "["; std::string outputarray = "[";
for (int i = 0; i < n_embd; i++) { for (int i = 0; i < n_embd; i++) {