vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-13 18:39:48 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

ubatch : new splitting logic (#14217)

Browse source 
ggml-ci
This commit is contained in:
Georgi Gerganov 2025-06-20 10:14:14 +03:00 • committed by GitHub
parent 9eaa51e7f0
commit 4c9fdfbe15
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 992 additions and 915 deletions
Download patch file Download diff file Expand all files Collapse all files

9
src/llama-memory-hybrid.h
View file

@ -50,9 +50,9 @@ public:
//
llama_memory_state_ptr init_batch(
const llama_batch & batch,
llama_batch_allocr & balloc,
uint32_t n_ubatch,
bool embd_pooled) override;
bool embd_all) override;
llama_memory_state_ptr init_full() override;
@ -107,7 +107,6 @@ public:
// init success
llama_memory_hybrid_state(
llama_memory_hybrid * mem,
llama_sbatch sbatch,
std::vector<uint32_t> heads_attn,
std::vector<llama_ubatch> ubatches);
@ -116,8 +115,6 @@ public:
bool next() override;
bool apply() override;
std::vector<int64_t> & out_ids() override;
llama_memory_status get_status() const override;
const llama_ubatch & get_ubatch() const override;
@ -129,8 +126,6 @@ public:
const llama_memory_recurrent_state * get_state_recr() const;
private:
llama_sbatch sbatch;
// the index of the next ubatch to process
size_t i_next = 0;