From e69c67713fba0cb90e7613b8ac6f3a9e0cded7e3 Mon Sep 17 00:00:00 2001 From: Jiaqi Liao <30439460+SkqLiao@users.noreply.github.com> Date: Thu, 20 Nov 2025 13:55:55 +0800 Subject: [PATCH] [refactor] fix third_party issue (#1632) * [refactor]: relocate third_party directory * [fix]: fix custom_flashinfer for kt-sft --- .gitmodules | 12 ++- kt-kernel/CMakeLists.txt | 8 +- kt-kernel/README.md | 2 +- kt-kernel/install.sh | 82 ++----------------- kt-kernel/operators/amx/test/mmq-test.cpp | 4 +- kt-kernel/operators/amx/test/mmq.cpp | 4 +- kt-sft/install.sh | 0 third_party/custom_flashinfer | 1 + .../third_party => third_party}/llama.cpp | 0 .../llamafile/README.md | 0 .../llamafile/bench.h | 0 .../llamafile/flags.cpp | 0 .../llamafile/flags.h | 0 .../llamafile/iqk_mul_mat.inc | 0 .../llamafile/iqk_mul_mat_amd_avx2.cpp | 0 .../llamafile/iqk_mul_mat_amd_zen4.cpp | 0 .../llamafile/iqk_mul_mat_arm.inc | 0 .../llamafile/iqk_mul_mat_arm82.cpp | 0 .../llamafile/macros.h | 0 .../llamafile/micros.h | 0 .../llamafile/numba.h | 0 .../llamafile/sgemm.cpp | 0 .../llamafile/sgemm.h | 0 .../llamafile/tinyblas_cpu.h | 0 .../llamafile/tinyblas_cpu_mixmul.inc | 0 .../llamafile/tinyblas_cpu_mixmul_amd_avx.cpp | 0 .../tinyblas_cpu_mixmul_amd_avx2.cpp | 0 .../tinyblas_cpu_mixmul_amd_avx512f.cpp | 0 .../tinyblas_cpu_mixmul_amd_avxvnni.cpp | 0 .../llamafile/tinyblas_cpu_mixmul_amd_fma.cpp | 0 .../tinyblas_cpu_mixmul_amd_zen4.cpp | 0 .../llamafile/tinyblas_cpu_mixmul_arm80.cpp | 0 .../llamafile/tinyblas_cpu_mixmul_arm82.cpp | 0 .../llamafile/tinyblas_cpu_sgemm.inc | 0 .../llamafile/tinyblas_cpu_sgemm_amd_avx.cpp | 0 .../llamafile/tinyblas_cpu_sgemm_amd_avx2.cpp | 0 .../tinyblas_cpu_sgemm_amd_avx512f.cpp | 0 .../tinyblas_cpu_sgemm_amd_avxvnni.cpp | 0 .../llamafile/tinyblas_cpu_sgemm_amd_fma.cpp | 0 .../llamafile/tinyblas_cpu_sgemm_amd_zen4.cpp | 0 .../llamafile/tinyblas_cpu_sgemm_arm80.cpp | 0 .../llamafile/tinyblas_cpu_sgemm_arm82.cpp | 0 .../llamafile/tinyblas_cpu_unsupported.cpp | 0 .../third_party => third_party}/pybind11 | 0 44 files changed, 27 insertions(+), 86 deletions(-) mode change 100644 => 100755 kt-sft/install.sh create mode 160000 third_party/custom_flashinfer rename {kt-kernel/third_party => third_party}/llama.cpp (100%) rename {kt-kernel/third_party => third_party}/llamafile/README.md (100%) rename {kt-kernel/third_party => third_party}/llamafile/bench.h (100%) rename {kt-kernel/third_party => third_party}/llamafile/flags.cpp (100%) rename {kt-kernel/third_party => third_party}/llamafile/flags.h (100%) rename {kt-kernel/third_party => third_party}/llamafile/iqk_mul_mat.inc (100%) rename {kt-kernel/third_party => third_party}/llamafile/iqk_mul_mat_amd_avx2.cpp (100%) rename {kt-kernel/third_party => third_party}/llamafile/iqk_mul_mat_amd_zen4.cpp (100%) rename {kt-kernel/third_party => third_party}/llamafile/iqk_mul_mat_arm.inc (100%) rename {kt-kernel/third_party => third_party}/llamafile/iqk_mul_mat_arm82.cpp (100%) rename {kt-kernel/third_party => third_party}/llamafile/macros.h (100%) rename {kt-kernel/third_party => third_party}/llamafile/micros.h (100%) rename {kt-kernel/third_party => third_party}/llamafile/numba.h (100%) rename {kt-kernel/third_party => third_party}/llamafile/sgemm.cpp (100%) rename {kt-kernel/third_party => third_party}/llamafile/sgemm.h (100%) rename {kt-kernel/third_party => third_party}/llamafile/tinyblas_cpu.h (100%) rename {kt-kernel/third_party => third_party}/llamafile/tinyblas_cpu_mixmul.inc (100%) rename {kt-kernel/third_party => third_party}/llamafile/tinyblas_cpu_mixmul_amd_avx.cpp (100%) rename {kt-kernel/third_party => third_party}/llamafile/tinyblas_cpu_mixmul_amd_avx2.cpp (100%) rename {kt-kernel/third_party => third_party}/llamafile/tinyblas_cpu_mixmul_amd_avx512f.cpp (100%) rename {kt-kernel/third_party => third_party}/llamafile/tinyblas_cpu_mixmul_amd_avxvnni.cpp (100%) rename {kt-kernel/third_party => third_party}/llamafile/tinyblas_cpu_mixmul_amd_fma.cpp (100%) rename {kt-kernel/third_party => third_party}/llamafile/tinyblas_cpu_mixmul_amd_zen4.cpp (100%) rename {kt-kernel/third_party => third_party}/llamafile/tinyblas_cpu_mixmul_arm80.cpp (100%) rename {kt-kernel/third_party => third_party}/llamafile/tinyblas_cpu_mixmul_arm82.cpp (100%) rename {kt-kernel/third_party => third_party}/llamafile/tinyblas_cpu_sgemm.inc (100%) rename {kt-kernel/third_party => third_party}/llamafile/tinyblas_cpu_sgemm_amd_avx.cpp (100%) rename {kt-kernel/third_party => third_party}/llamafile/tinyblas_cpu_sgemm_amd_avx2.cpp (100%) rename {kt-kernel/third_party => third_party}/llamafile/tinyblas_cpu_sgemm_amd_avx512f.cpp (100%) rename {kt-kernel/third_party => third_party}/llamafile/tinyblas_cpu_sgemm_amd_avxvnni.cpp (100%) rename {kt-kernel/third_party => third_party}/llamafile/tinyblas_cpu_sgemm_amd_fma.cpp (100%) rename {kt-kernel/third_party => third_party}/llamafile/tinyblas_cpu_sgemm_amd_zen4.cpp (100%) rename {kt-kernel/third_party => third_party}/llamafile/tinyblas_cpu_sgemm_arm80.cpp (100%) rename {kt-kernel/third_party => third_party}/llamafile/tinyblas_cpu_sgemm_arm82.cpp (100%) rename {kt-kernel/third_party => third_party}/llamafile/tinyblas_cpu_unsupported.cpp (100%) rename {kt-kernel/third_party => third_party}/pybind11 (100%) diff --git a/.gitmodules b/.gitmodules index 7f6e818a..23b49a53 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,10 @@ -[submodule "kt-kernel/third_party/llama.cpp"] - path = kt-kernel/third_party/llama.cpp +[submodule "third_party/llama.cpp"] + path = third_party/llama.cpp url = https://github.com/ggerganov/llama.cpp.git -[submodule "kt-kernel/third_party/pybind11"] - path = kt-kernel/third_party/pybind11 +[submodule "third_party/pybind11"] + path = third_party/pybind11 url = https://github.com/pybind/pybind11.git +[submodule "third_party/custom_flashinfer"] + path = third_party/custom_flashinfer + url = https://github.com/kvcache-ai/custom_flashinfer.git + branch = fix-precision-mla-merge-main diff --git a/kt-kernel/CMakeLists.txt b/kt-kernel/CMakeLists.txt index 6429b7b6..438c43d3 100644 --- a/kt-kernel/CMakeLists.txt +++ b/kt-kernel/CMakeLists.txt @@ -387,10 +387,10 @@ message(STATUS "ARCH_FLAGS: ${ARCH_FLAGS}") add_compile_options("$<$:${ARCH_FLAGS}>") add_compile_options("$<$:${ARCH_FLAGS}>") -add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/third_party/pybind11 ${CMAKE_CURRENT_BINARY_DIR}/third_party/pybind11) -add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/third_party/llama.cpp ${CMAKE_CURRENT_BINARY_DIR}/third_party/llama.cpp) +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../third_party/pybind11 ${CMAKE_CURRENT_BINARY_DIR}/third_party/pybind11) +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../third_party/llama.cpp ${CMAKE_CURRENT_BINARY_DIR}/third_party/llama.cpp) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/third_party) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../third_party) if(KTRANSFORMERS_USE_CUDA) include(CheckLanguage) check_language(CUDA) @@ -438,7 +438,7 @@ endif() aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_DIR1) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/cpu_backend SOURCE_DIR2) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/operators/llamafile SOURCE_DIR3) -aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/third_party/llamafile SOURCE_DIR4) +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/../third_party/llamafile SOURCE_DIR4) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/operators/kvcache SOURCE_DIR5) # message(STATUS "SOURCE_DIR3: ${SOURCE_DIR3}") diff --git a/kt-kernel/README.md b/kt-kernel/README.md index 2c0b48dd..2f2551ec 100644 --- a/kt-kernel/README.md +++ b/kt-kernel/README.md @@ -96,7 +96,7 @@ If you need specific build options (e.g., for LLAMAFILE backend, compatibility, ```bash # Example for LLAMAFILE backend on AMX CPU with AVX512 -export CPUINFER_CPU_INSTRUCT=AVX512 # Options: NATIVE, AVX512, AVX2 +export CPUINFER_CPU_INSTRUCT=AVX512 # Options: NATIVE, AVX512, AVX2, FANCY export CPUINFER_ENABLE_AMX=OFF # Options: ON, OFF # Run with manual mode (build only) diff --git a/kt-kernel/install.sh b/kt-kernel/install.sh index 2ee30bff..99c2a382 100755 --- a/kt-kernel/install.sh +++ b/kt-kernel/install.sh @@ -1,68 +1,6 @@ #!/usr/bin/env bash set -euo pipefail -install_dependencies() { - echo "Checking and installing system dependencies..." - - # Determine if we need to use sudo - SUDO="" - if [ "$EUID" -ne 0 ]; then - if command -v sudo &> /dev/null; then - SUDO="sudo" - else - echo "Warning: Not running as root and sudo not found. Package installation may fail." - echo "Please run as root or install sudo." - fi - fi - - if command -v conda &> /dev/null; then - echo "Installing cmake via conda..." - conda install -y cmake - else - echo "Warning: conda not found. Skipping cmake installation via conda." - echo "Please install conda or manually install cmake." - fi - - # Detect OS type - if [ -f /etc/os-release ]; then - . /etc/os-release - OS=$ID - elif [ -f /etc/debian_version ]; then - OS="debian" - elif [ -f /etc/redhat-release ]; then - OS="rhel" - else - echo "Warning: Unable to detect OS type. Skipping dependency installation." - return 0 - fi - - # Install dependencies based on OS - case "$OS" in - debian|ubuntu|linuxmint|pop) - echo "Detected Debian-based system. Installing libhwloc-dev and pkg-config..." - $SUDO apt update - $SUDO apt install -y libhwloc-dev pkg-config - ;; - fedora|rhel|centos|rocky|almalinux) - echo "Detected Red Hat-based system. Installing hwloc-devel and pkgconfig..." - $SUDO dnf install -y hwloc-devel pkgconfig || $SUDO yum install -y hwloc-devel pkgconfig - ;; - arch|manjaro) - echo "Detected Arch-based system. Installing hwloc and pkgconf..." - $SUDO pacman -S --noconfirm hwloc pkgconf - ;; - opensuse*|sles) - echo "Detected openSUSE-based system. Installing hwloc-devel and pkg-config..." - $SUDO zypper install -y hwloc-devel pkg-config - ;; - *) - echo "Warning: Unsupported OS '$OS'. Please manually install libhwloc-dev and pkg-config." - ;; - esac -} - -install_dependencies - usage() { cat <