mirror of
https://github.com/kvcache-ai/ktransformers.git
synced 2026-04-28 03:39:48 +00:00
57d14d22bc
* refactor: move legacy code to archive/ directory - Moved ktransformers, csrc, third_party, merge_tensors to archive/ - Moved build scripts and configurations to archive/ - Kept kt-kernel, KT-SFT, doc, and README files in root - Preserved complete git history for all moved files * refactor: restructure repository to focus on kt-kernel and KT-SFT modules * fix README * fix README * fix README * fix README * docs: add performance benchmarks to kt-kernel section Add comprehensive performance data for kt-kernel to match KT-SFT's presentation: - AMX kernel optimization: 21.3 TFLOPS (3.9× faster than PyTorch) - Prefill phase: up to 20× speedup vs baseline - Decode phase: up to 4× speedup - NUMA optimization: up to 63% throughput improvement - Multi-GPU (8×L20): 227.85 tokens/s total throughput with DeepSeek-R1 FP8 Source: https://lmsys.org/blog/2025-10-22-KTransformers/ This provides users with concrete performance metrics for both core modules, making it easier to understand the capabilities of each component. * refactor: improve kt-kernel performance data with specific hardware and models Replace generic performance descriptions with concrete benchmarks: - Specify exact hardware: 8×L20 GPU + Xeon Gold 6454S, Single/Dual-socket Xeon + AMX - Include specific models: DeepSeek-R1-0528 (FP8), DeepSeek-V3 (671B) - Show detailed metrics: total throughput, output throughput, concurrency details - Match KT-SFT presentation style for consistency This provides users with actionable performance data they can use to evaluate hardware requirements and expected performance for their use cases. * fix README * docs: clean up performance table and improve formatting * add pic for README * refactor: simplify .gitmodules and backup legacy submodules - Remove 7 legacy submodules from root .gitmodules (archive/third_party/*) - Keep only 2 active submodules for kt-kernel (llama.cpp, pybind11) - Backup complete .gitmodules to archive/.gitmodules - Add documentation in archive/README.md for researchers who need legacy submodules This reduces initial clone size by ~500MB and avoids downloading unused dependencies. * refactor: move doc/ back to root directory Keep documentation in root for easier access and maintenance. * refactor: consolidate all images to doc/assets/ - Move kt-kernel/assets/heterogeneous_computing.png to doc/assets/ - Remove KT-SFT/assets/ (images already in doc/assets/) - Update KT-SFT/README.md image references to ../doc/assets/ - Eliminates ~7.9MB image duplication - Centralizes all documentation assets in one location * fix pic path for README
61 lines
1.9 KiB
C++
61 lines
1.9 KiB
C++
/**
|
|
* @Description :
|
|
* @Author : Xie Weiyu
|
|
* @Date : 2024-11-22 09:52:48
|
|
* @Version : 1.0.0
|
|
* @LastEditors : Xie Weiyu
|
|
* @LastEditTime : 2024-11-25 07:51:09
|
|
* @Copyright (c) 2024 by KVCache.AI, All Rights Reserved.
|
|
**/
|
|
|
|
#include "common.hpp"
|
|
|
|
int main(int argc, char* argv[]) {
|
|
qw25_7B_gpu_config.v_cache_on = false;
|
|
config.gpu_cache_config = qw25_7B_gpu_config;
|
|
config.v_cache_on = false;
|
|
|
|
init(argc, argv);
|
|
spdlog::set_level(spdlog::level::debug);
|
|
auto kvc2 = kvc2::create_kvc2(config);
|
|
|
|
std::mt19937 gen(123);
|
|
auto ids1 = random_ids(10 * config.num_token_per_page, gen);
|
|
auto k1 = random_kvcache(10, gen);
|
|
|
|
kvc2->raw_insert(test_model_name, test_quant_type, ids1.data(), ids1.size(), k1, {});
|
|
|
|
// complete same
|
|
#pragma omp parallel for
|
|
for (size_t ti = 0; ti < 3; ti++) {
|
|
auto h = kvc2->lookup_to_gpu(test_model_name, test_quant_type, ids1.data(), ids1.size(),
|
|
ids1.size() + 2 * config.num_token_per_page);
|
|
auto k = h->handle_data(true);
|
|
cmp_handle_data(k1, k, 10);
|
|
|
|
auto block_idx = h->get_gpu_block_idx();
|
|
auto [kcache, vcache] = kvc2->get_kvcache();
|
|
|
|
auto k_from_gpu = empty_kvcache(15);
|
|
|
|
size_t gpu_count = config.gpu_cache_config->gpu_devices_id.size();
|
|
size_t element_size_per_gpu = test_cache_info.element_size(config.num_token_per_page) / gpu_count;
|
|
for (size_t i = 0; i < k_from_gpu.size(); i++) {
|
|
for (size_t j = 0; j < block_idx.size(); j++) {
|
|
size_t b_idx = block_idx[j];
|
|
for (size_t gpu_idx = 0; gpu_idx < gpu_count; gpu_idx++) {
|
|
{
|
|
auto kt = kcache[gpu_idx][i][b_idx].to(torch::kCPU);
|
|
void* src = kt.data_ptr();
|
|
void* dst = offset_by_bytes(k_from_gpu[i][j], gpu_idx * element_size_per_gpu);
|
|
memcpy(dst, src, element_size_per_gpu);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
cmp_handle_data(k1, k_from_gpu, 10);
|
|
}
|
|
|
|
SPDLOG_CRITICAL("All Test Passed: {}", argv[0]);
|
|
return 0;
|
|
}
|