vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-10 17:14:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

smoosh the tfs logic for performance (#1393)

Browse source
This commit is contained in:
Reithan 2025-02-28 07:55:06 -08:00 • committed by GitHub
parent dd6e4038ea
commit 202c029924
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 12 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

19
gpttype_adapter.cpp
View file

@ -1321,25 +1321,17 @@ void sample_tail_free(llama_token_data_array * cur_p, float z, size_t min_keep)
sample_softmax(cur_p);
// Compute the first and second derivatives
std::vector<float> first_derivatives(cur_p->size - 1);
std::vector<float> second_derivatives(cur_p->size - 2);
float second_derivatives_sum = 0.0f;
for (size_t i = 0; i < first_derivatives.size(); ++i) {
first_derivatives[i] = cur_p->data[i].p - cur_p->data[i + 1].p;
}
for (size_t i = 0; i < second_derivatives.size(); ++i) {
second_derivatives[i] = first_derivatives[i] - first_derivatives[i + 1];
}
// Calculate absolute value of second derivatives
for (size_t i = 0; i < second_derivatives.size(); ++i) {
second_derivatives[i] = std::abs(second_derivatives[i]);
float first_derivatives_1 = cur_p->data[i].p - cur_p->data[i + 1].p;
float first_derivatives_2 = cur_p->data[i + 1].p - cur_p->data[i + 2].p;
second_derivatives[i] = std::abs(first_derivatives_1 - first_derivatives_2);
second_derivatives_sum += second_derivatives[i];
}
// Normalize the second derivatives
{
const float second_derivatives_sum = std::accumulate(second_derivatives.begin(), second_derivatives.end(), 0.0f);
if (second_derivatives_sum > 1e-6f) {
for (float & value : second_derivatives) {
value /= second_derivatives_sum;
@ -1349,7 +1341,6 @@ void sample_tail_free(llama_token_data_array * cur_p, float z, size_t min_keep)
value = 1.0f / second_derivatives.size();
}
}
}
float cum_sum = 0.0f;
size_t last_idx = cur_p->size;