diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 5ea5b14df..1f1f68b81 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -4780,11 +4780,71 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int } } +static const int smartcache_snapshot_min_spacing = 150; + +static bool smartcache_prefix_compatible(const std::vector & a, const std::vector & b) +{ + const size_t min_size = std::min(a.size(), b.size()); + for(size_t i=0;i currctxsize ? slot_size - currctxsize : currctxsize - slot_size; + if(distance > smartcache_snapshot_min_spacing) + { + continue; + } + if(!smartcache_prefix_compatible(slot_tokens,current_context_tokens)) + { + continue; + } + if(slot_size < best_size) + { + best_size = slot_size; + best_slot = i; + } + } + return best_slot; +} + int smartcache_quick_snapshot(int specific_slot = -1) { int identical_slot = get_identical_existing_slot(); if(identical_slot==-1) { + if(specific_slot==-1) + { + int nearby_slot = get_nearby_compatible_smartcache_slot(); + if(nearby_slot!=-1) + { + if(savestates[nearby_slot].savestate_context_tokens.size() <= current_context_tokens.size()) + { + touch_slot(nearby_slot); + return nearby_slot; + } + gpttype_save_state_kv(nearby_slot); + return nearby_slot; + } + } if(specific_slot!=-1) { gpttype_save_state_kv(specific_slot); @@ -4796,7 +4856,6 @@ int smartcache_quick_snapshot(int specific_slot = -1) gpttype_save_state_kv(oldest_slot); return oldest_slot; } - } else {