batch : add n_used count (#14512)

ggml-ci
This commit is contained in:
Georgi Gerganov 2025-07-04 09:04:59 +03:00 committed by GitHub
parent 499a8f5a78
commit c79184d2d1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 34 additions and 1 deletions

View file

@ -80,6 +80,11 @@ llama_memory_context_ptr llama_memory_hybrid::init_batch(llama_batch_allocr & ba
ubatches.push_back(std::move(ubatch)); // NOLINT
}
if (balloc.get_n_used() < balloc.get_n_tokens()) {
// failed to find a suitable split
break;
}
// prepare the recurrent batches first
if (!mem_recr->prepare(ubatches)) {
// TODO: will the recurrent cache be in an undefined context at this point?