diff --git a/common/speculative.cpp b/common/speculative.cpp index 1d2977e42..580728a20 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -2246,7 +2246,7 @@ common_params common_base_params_to_speculative(const common_params & params) { return result; } -struct common_init_speculative_result::impl { +struct common_speculative_init_result::impl { impl() = default; ~impl() = default; @@ -2255,7 +2255,7 @@ struct common_init_speculative_result::impl { llama_context_ptr context; }; -common_init_speculative_result::common_init_speculative_result( +common_speculative_init_result::common_speculative_init_result( common_params & params, llama_model * model_tgt, llama_context * ctx_tgt) : @@ -2301,8 +2301,7 @@ common_init_speculative_result::common_init_speculative_result( } else if (spec_mtp) { model_path = params.model.path; - LOG_TRC("%s: creating MTP draft context against the target model '%s'\n", - __func__, model_path.c_str()); + LOG_TRC("%s: creating MTP draft context against the target model '%s'\n", __func__, model_path.c_str()); llama_context * ctx_dft = llama_init_from_model(model_tgt, cparams); if (ctx_dft == nullptr) { @@ -2314,18 +2313,18 @@ common_init_speculative_result::common_init_speculative_result( } } -common_init_speculative_result::~common_init_speculative_result() = default; +common_speculative_init_result::~common_speculative_init_result() = default; -llama_model * common_init_speculative_result::model() { +llama_model * common_speculative_init_result::model() { return pimpl->model.get(); } -llama_context * common_init_speculative_result::context() { +llama_context * common_speculative_init_result::context() { return pimpl->context.get(); } -common_init_speculative_result_ptr common_init_speculative_from_params(common_params & params, llama_model * model_tgt, llama_context * ctx_tgt) { - return std::make_unique(params, model_tgt, ctx_tgt); +common_speculative_init_result_ptr common_speculative_init_from_params(common_params & params, llama_model * model_tgt, llama_context * ctx_tgt) { + return std::make_unique(params, model_tgt, ctx_tgt); } // initialization of the speculative decoding system diff --git a/common/speculative.h b/common/speculative.h index 2f6d225b6..062bf2093 100644 --- a/common/speculative.h +++ b/common/speculative.h @@ -83,9 +83,9 @@ struct common_speculative_deleter { typedef std::unique_ptr common_speculative_ptr; -struct common_init_speculative_result { - common_init_speculative_result(common_params & params, llama_model * model_tgt, llama_context * ctx_tgt); - ~common_init_speculative_result(); +struct common_speculative_init_result { + common_speculative_init_result(common_params & params, llama_model * model_tgt, llama_context * ctx_tgt); + ~common_speculative_init_result(); llama_model * model(); llama_context * context(); @@ -95,6 +95,6 @@ private: std::unique_ptr pimpl; }; -using common_init_speculative_result_ptr = std::unique_ptr; +using common_speculative_init_result_ptr = std::unique_ptr; -common_init_speculative_result_ptr common_init_speculative_from_params(common_params & params, llama_model * model_tgt, llama_context * ctx_tgt); +common_speculative_init_result_ptr common_speculative_init_from_params(common_params & params, llama_model * model_tgt, llama_context * ctx_tgt); diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 3890fe5f0..df5666581 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -900,7 +900,7 @@ private: llama_model * model_dft = nullptr; llama_context * ctx_dft = nullptr; - common_init_speculative_result_ptr spec_init; + common_speculative_init_result_ptr spec_init; common_context_seq_rm_type ctx_tgt_seq_rm_type = COMMON_CONTEXT_SEQ_RM_TYPE_NO; common_context_seq_rm_type ctx_dft_seq_rm_type = COMMON_CONTEXT_SEQ_RM_TYPE_NO; @@ -1171,11 +1171,12 @@ private: { common_params params_dft = common_base_params_to_speculative(params_base); + // progress callback params_dft.load_progress_callback = load_progress_callback; params_dft.load_progress_callback_user_data = &load_progress_spec; - spec_init = common_init_speculative_from_params(params_dft, model_tgt, ctx_tgt); + spec_init = common_speculative_init_from_params(params_dft, model_tgt, ctx_tgt); model_dft = spec_init->model(); ctx_dft = spec_init->context(); @@ -2306,8 +2307,8 @@ private: // this is not true for SWA models: https://github.com/ggml-org/llama.cpp/pull/24411#issuecomment-4677983225 cur.update_pos(slot.prompt.n_tokens() - n_tokens_cur, pos_min, pos_max); - cur.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); - cur.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + cur.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + cur.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); // stash the draft's speculative state with the checkpoint common_speculative_get_state(spec.get(), slot.id, cur.data_spec); @@ -3264,8 +3265,8 @@ private: if (!do_reset) { // restore the context checkpoint - it->load_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); - it->load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + it->load_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + it->load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); // restore the draft's speculative state common_speculative_set_state(spec.get(), slot.id, it->data_spec);