vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-07-09 17:08:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'upstream' into concedo_experimental

Browse source 
# Conflicts:
#	.github/actions/windows-setup-cuda/action.yml
#	.github/workflows/build-3rd-party.yml
#	.github/workflows/build-android.yml
#	.github/workflows/build-apple.yml
#	.github/workflows/build-cann.yml
#	.github/workflows/build-cuda-ubuntu.yml
#	.github/workflows/build-ibm.yml
#	.github/workflows/build-msys.yml
#	.github/workflows/build-opencl.yml
#	.github/workflows/build-openvino.yml
#	.github/workflows/build-riscv.yml
#	.github/workflows/build-rpc.yml
#	.github/workflows/build-sanitize.yml
#	.github/workflows/build-self-hosted.yml
#	.github/workflows/build-sycl.yml
#	.github/workflows/build-vulkan.yml
#	.github/workflows/build-webgpu.yml
#	.github/workflows/hip-quality-check.yml
#	.github/workflows/release.yml
#	.github/workflows/server-sanitize.yml
#	.github/workflows/server-self-hosted.yml
#	.github/workflows/server.yml
#	.github/workflows/ui-self-hosted.yml
#	.github/workflows/ui.yml
#	CONTRIBUTING.md
#	ci/run.sh
#	docs/autoparser.md
#	docs/multimodal/granitevision.md
#	examples/model-conversion/README.md
#	ggml/src/ggml-hexagon/ggml-hexagon.cpp
#	ggml/src/ggml-hexagon/htp/CMakeLists.txt
#	ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c
#	ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c
#	ggml/src/ggml-hexagon/htp/htp-ops.h
#	ggml/src/ggml-hexagon/htp/main.c
#	ggml/src/ggml-hexagon/htp/matmul-ops.c
#	ggml/src/ggml-opencl/CMakeLists.txt
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	ggml/src/ggml-vulkan/CMakeLists.txt
#	ggml/src/ggml-vulkan/ggml-vulkan.cpp
#	ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt
#	ggml/src/ggml-webgpu/ggml-webgpu.cpp
#	ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl
#	ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl
#	ggml/src/ggml-zendnn/ggml-zendnn.cpp
#	pyproject.toml
#	scripts/snapdragon/adb/run-completion.sh
#	scripts/snapdragon/adb/run-tool.sh
#	scripts/sync_vendor.py
#	tests/gguf-model-data.cpp
#	tools/cli/README.md
#	tools/completion/README.md
#	tools/server/README.md
This commit is contained in:
Concedo 2026-05-28 15:28:29 +08:00
commit ca942eb172
20 changed files with 1618 additions and 120 deletions
Download patch file Download diff file Expand all files Collapse all files

10
common/arg.cpp
View file

@ -3028,7 +3028,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.default_template_kwargs[item.key()] = item.value().dump();
}
}
).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_CHAT_TEMPLATE_KWARGS"));
).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_CHAT_TEMPLATE_KWARGS"));
add_opt(common_arg(
{"-to", "--timeout"}, "N",
string_format("server read/write timeout in seconds (default: %d)", params.timeout_read),
@ -3329,7 +3329,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params &, const std::string & value) {
common_log_set_file(common_log_main(), value.c_str());
}
).set_env("LLAMA_LOG_FILE"));
).set_env("LLAMA_ARG_LOG_FILE"));
add_opt(common_arg(
{"--log-colors"}, "[on|off|auto]",
"Set colored logging ('on', 'off', or 'auto', default: 'auto')\n"
@ -3346,7 +3346,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
string_format("error: unknown value for --log-colors: '%s'\n", value.c_str()));
}
}
).set_env("LLAMA_LOG_COLORS"));
).set_env("LLAMA_ARG_LOG_COLORS"));
add_opt(common_arg(
{"-v", "--verbose", "--log-verbose"},
"Set verbosity level to infinity (i.e. log all messages, useful for debugging)",
@ -3361,7 +3361,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params) {
params.offline = true;
}
).set_env("LLAMA_OFFLINE"));
).set_env("LLAMA_ARG_OFFLINE"));
add_opt(common_arg(
{"-lv", "--verbosity", "--log-verbosity"}, "N",
string_format("Set the verbosity threshold. Messages with a higher verbosity will be ignored. Values:\n"
@ -3376,7 +3376,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.verbosity = value;
common_log_set_verbosity_thold(value);
}
).set_env("LLAMA_LOG_VERBOSITY"));
).set_env("LLAMA_ARG_LOG_VERBOSITY"));
add_opt(common_arg(
{"--log-prefix"},
{"--no-log-prefix"},

3
conversion/base.py
View file

@ -1625,6 +1625,9 @@ class TextModel(ModelBase):
if chkhsh == "f728162c1315c26e40249849799b4ba3fe584c32084b4795b03eb295e63cb5af":
# ref: https://huggingface.co/lewtun/talkie-1930-13b-it-hf
res = "talkie"
if chkhsh == "36f3066e97b7f3994b379aaacde306c1444c6ae84e81a5ae3cd2b7ed3b8c42d4":
# ref: https://huggingface.co/openbmb/MiniCPM5-1B
res = "minicpm5"
if res is None:
logger.warning("\n")

1
convert_hf_to_gguf_update.py
View file

@ -157,6 +157,7 @@ models = [
{"name": "f2llmv2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/codefuse-ai/F2LLM-v2-4B", },
{"name": "sarvam-moe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/sarvamai/sarvam-30b", },
{"name": "talkie", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/lewtun/talkie-1930-13b-it-hf", },
{"name": "minicpm5", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openbmb/MiniCPM5-1B"},
]
# some models are known to be broken upstream, so we will skip them as exceptions

332
ggml/src/ggml-vulkan/ggml-vulkan.cpp
View file

@ -32,6 +32,20 @@ DispatchLoaderDynamic & ggml_vk_default_dispatcher();
#define VULKAN_HPP_DEFAULT_DISPATCHER ggml_vk_default_dispatcher()
#include <vulkan/vulkan.hpp>
// Fallback definitions for VK_NV_cooperative_matrix_decode_vector in case the
// installed Vulkan headers predate the extension.
#ifndef VK_NV_cooperative_matrix_decode_vector
#define VK_NV_cooperative_matrix_decode_vector 1
#define VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME "VK_NV_cooperative_matrix_decode_vector"
#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_DECODE_VECTOR_FEATURES_NV ((VkStructureType)1000689000)
typedef struct VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV {
VkStructureType sType;
void* pNext;
VkBool32 cooperativeMatrixDecodeVector;
} VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV;
#endif
// SPIRV-Headers: LunarG Windows SDK uses Include/spirv-headers/spirv.hpp (not spirv/unified1/). MinGW/MSYS2 and
// Linux packages use Khronos layout spirv/unified1/spirv.hpp. See docs/build.md#vulkan.
#include <spirv-headers/spirv.hpp> //kcpp: use bundled spirv.hpp to ensure same version
@ -403,6 +417,7 @@ enum vk_conv_shapes {
CONV_SHAPE_128x128,
CONV_SHAPE_64x32,
CONV_SHAPE_32x256,
CONV_SHAPE_64x128,
CONV_SHAPE_COUNT,
};
@ -417,6 +432,7 @@ vk_conv_block_size vk_conv_block_sizes[CONV_SHAPE_COUNT] = {
{ 128, 128, 16 }, // CONV_SHAPE_128x128
{ 64, 32, 32 }, // CONV_SHAPE_64x32
{ 32, 256, 16 }, // CONV_SHAPE_32x256
{ 64, 128, 16 }, // CONV_SHAPE_64x128
};
enum dmmv_wg_sizes {
@ -452,14 +468,16 @@ struct vk_fa_pipeline_state {
};
struct vk_conv2d_pipeline_state {
vk_conv2d_pipeline_state(uint32_t s0, uint32_t s1, uint32_t p0, uint32_t p1, uint32_t d0, uint32_t d1, uint32_t KW, uint32_t KH)
: s0(s0), s1(s1), p0(p0), p1(p1), d0(d0), d1(d1), KW(KW), KH(KH) {}
vk_conv2d_pipeline_state(uint32_t s0, uint32_t s1, uint32_t p0, uint32_t p1, uint32_t d0, uint32_t d1, uint32_t KW, uint32_t KH, uint32_t aligned)
: s0(s0), s1(s1), p0(p0), p1(p1), d0(d0), d1(d1), KW(KW), KH(KH), aligned(aligned) {}
uint32_t s0, s1, p0, p1, d0, d1, KW, KH;
// when set, shader can skip K/CRS/NPQ bounds checks and address clamps
uint32_t aligned;
bool operator<(const vk_conv2d_pipeline_state &b) const {
return std::tie(s0, s1, p0, p1, d0, d1, KW, KH) <
std::tie(b.s0, b.s1, b.p0, b.p1, b.d0, b.d1, b.KW, b.KH);
return std::tie(s0, s1, p0, p1, d0, d1, KW, KH, aligned) <
std::tie(b.s0, b.s1, b.p0, b.p1, b.d0, b.d1, b.KW, b.KH, b.aligned);
}
};
@ -679,6 +697,7 @@ struct vk_device_struct {
uint32_t coopmat_int_k;
bool coopmat2;
bool coopmat2_decode_vector;
bool pipeline_executable_properties_support {};
@ -769,7 +788,8 @@ struct vk_device_struct {
vk_pipeline pipeline_clamp_f32;
vk_pipeline pipeline_pad_f32;
vk_pipeline pipeline_roll_f32;
vk_pipeline pipeline_repeat_f32, pipeline_repeat_back_f32;
vk_pipeline pipeline_repeat_i32, pipeline_repeat_back_f32;
vk_pipeline pipeline_repeat_i16;
vk_pipeline pipeline_cpy_f32_f32, pipeline_cpy_f32_f16, pipeline_cpy_f16_f16, pipeline_cpy_f16_f32, pipeline_cpy_f32_bf16, pipeline_cpy_bf16_f32, pipeline_cpy_f32_i32, pipeline_cpy_i32_f32;
vk_pipeline pipeline_contig_cpy_f32_f32, pipeline_contig_cpy_f32_f16, pipeline_contig_cpy_f16_f16, pipeline_contig_cpy_f16_f32, pipeline_contig_cpy_f32_bf16, pipeline_contig_cpy_bf16_f32, pipeline_contig_cpy_f32_i32, pipeline_contig_cpy_i32_f32;
vk_pipeline pipeline_cpy_f32_quant[GGML_TYPE_COUNT];
@ -2167,6 +2187,136 @@ static uint32_t compile_count = 0;
static std::mutex compile_count_mutex;
static std::condition_variable compile_count_cond;
static constexpr uint32_t kSpvOpCooperativeMatrixLoadTensorNV = 5367;
static constexpr uint32_t kSpvCapabilityCooperativeMatrixDecodeVectorNV = 5447;
static constexpr uint32_t kSpvTensorAddressingDecodeVectorFuncBit = 0x4;
// Remove SPV_NV_cooperative_matrix_decode_vector usage from a SPIR-V module so it
// can be loaded on drivers that only support SPV_NV_cooperative_matrix2. Drops the
// OpExtension declaration, the CooperativeMatrixDecodeVectorNV OpCapability, and the
// DecodeVectorFunc operand from any OpCooperativeMatrixLoadTensorNV instruction.
// Returns true when the input used the extension (and `out` was populated with a
// stripped copy); returns false otherwise without touching `out`.
static bool ggml_vk_strip_decode_vector(const uint32_t * code, size_t word_count, std::vector<uint32_t> & out) {
static const char kDecodeVectorExt[] = "SPV_NV_cooperative_matrix_decode_vector";
if (word_count < 5) {
return false;
}
bool uses_decode_vector = false;
for (size_t pos = 5; pos < word_count; ) {
uint32_t word = code[pos];
uint32_t wc = word >> spv::WordCountShift;
uint32_t op = word & spv::OpCodeMask;
GGML_ASSERT(wc > 0 && pos + wc <= word_count);
if (op == spv::OpExtension && wc >= 2) {
const char * s = reinterpret_cast<const char *>(&code[pos + 1]);
if (strcmp(s, kDecodeVectorExt) == 0) {
uses_decode_vector = true;
break;
}
}
pos += wc;
}
if (!uses_decode_vector) {
return false;
}
VK_LOG_DEBUG("ggml_vk_strip_decode_vector: stripping SPV_NV_cooperative_matrix_decode_vector");
// Bulk-copy unchanged runs and only break the run when an instruction needs to
// be dropped or patched. Use reserve + insert/push_back so the destination buffer
// is touched exactly once (no zero-initialization pass from resize()).
out.clear();
out.reserve(word_count);
size_t run_start = 0;
auto flush_run = [&](size_t up_to) {
if (up_to > run_start) {
out.insert(out.end(), code + run_start, code + up_to);
}
};
for (size_t pos = 5; pos < word_count; ) {
uint32_t word = code[pos];
uint32_t wc = word >> spv::WordCountShift;
uint32_t op = word & spv::OpCodeMask;
GGML_ASSERT(wc > 0 && pos + wc <= word_count);
if (op == spv::OpExtension && wc >= 2) {
const char * s = reinterpret_cast<const char *>(&code[pos + 1]);
if (strcmp(s, kDecodeVectorExt) == 0) {
flush_run(pos);
pos += wc;
run_start = pos;
continue;
}
}
if (op == spv::OpCapability && wc == 2 && code[pos + 1] == kSpvCapabilityCooperativeMatrixDecodeVectorNV) {
flush_run(pos);
pos += wc;
run_start = pos;
continue;
}
if (op == kSpvOpCooperativeMatrixLoadTensorNV) {
// [opcode/wc][ResultType][Result][Pointer][Object][TensorLayout][MemOperand mask][mem extras...][TA mask][ta extras...]
GGML_ASSERT(wc >= 8);
uint32_t mem_mask = code[pos + 6];
size_t cur = pos + 7;
// Each of these MemoryAccess bits (when set) carries one trailing operand.
cur += (mem_mask & 0x2) ? 1 : 0; // Aligned
cur += (mem_mask & 0x8) ? 1 : 0; // MakePointerAvailable
cur += (mem_mask & 0x10) ? 1 : 0; // MakePointerVisible
cur += (mem_mask & 0x10000) ? 1 : 0; // AliasScopeINTELMask
cur += (mem_mask & 0x20000) ? 1 : 0; // NoAliasINTELMask
GGML_ASSERT(cur < pos + wc);
uint32_t ta_mask = code[cur];
if ((ta_mask & kSpvTensorAddressingDecodeVectorFuncBit) == 0) {
pos += wc;
continue; // leave instruction inside the current unchanged run
}
flush_run(pos);
// Append unchanged prefix of the instruction (header through the mem-extras).
size_t inst_start = out.size();
size_t pre_n = cur - pos;
out.insert(out.end(), code + pos, code + pos + pre_n);
// Emit TA mask with the DecodeVectorFunc bit cleared.
out.push_back(ta_mask & ~kSpvTensorAddressingDecodeVectorFuncBit);
// TA extras: TensorView (0x1) and DecodeFunc (0x2) are kept verbatim;
// DecodeVectorFunc (0x4) is dropped along with its trailing id operand.
size_t keep_ta_extras = ((ta_mask & 0x1) ? 1 : 0) + ((ta_mask & 0x2) ? 1 : 0);
if (keep_ta_extras) {
out.insert(out.end(), code + cur + 1, code + cur + 1 + keep_ta_extras);
}
GGML_ASSERT(wc == pre_n + 1 + keep_ta_extras + 1);
// Patch the instruction header with the new (one-shorter) word count.
uint32_t new_wc = wc - 1;
out[inst_start] = (new_wc << spv::WordCountShift) | op;
pos += wc;
run_start = pos;
continue;
}
pos += wc;
}
flush_run(word_count);
return true;
}
static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipeline, size_t spv_size, const void* spv_data, const std::string entrypoint,
uint32_t parameter_count, std::array<uint32_t, 3> wg_denoms, std::vector<uint32_t> specialization_constants,
bool disable_robustness, bool require_full_subgroups, uint32_t required_subgroup_size) {
@ -2238,6 +2388,18 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin
shader_module_create_info = vk::ShaderModuleCreateInfo({}, spirv.size() * sizeof(uint32_t), spirv.data());
}
#if defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT)
if (device->coopmat2 && !device->coopmat2_decode_vector) {
const uint32_t * src = spirv.empty() ? reinterpret_cast<const uint32_t *>(spv_data) : spirv.data();
size_t src_n = spirv.empty() ? spv_size / sizeof(uint32_t) : spirv.size();
std::vector<uint32_t> stripped;
if (ggml_vk_strip_decode_vector(src, src_n, stripped)) {
spirv = std::move(stripped);
shader_module_create_info = vk::ShaderModuleCreateInfo({}, spirv.size() * sizeof(uint32_t), spirv.data());
}
}
#endif
pipeline->shader_module = device->device.createShaderModule(shader_module_create_info);
vk::PushConstantRange pcr(
@ -4709,9 +4871,11 @@ static void ggml_vk_load_shaders(vk_device& device) {
ggml_vk_create_pipeline(device, device->pipeline_roll_f32, "roll_f32", roll_f32_len, roll_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_repeat_f32, "repeat_f32", repeat_f32_len, repeat_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_repeat_i32, "repeat_i32", repeat_i32_len, repeat_i32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_repeat_back_f32, "repeat_back_f32", repeat_back_f32_len, repeat_back_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_repeat_i16, "repeat_i16", repeat_i16_len, repeat_i16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
#define CREATE_UNARY(name) \
ggml_vk_create_pipeline(device, device->pipeline_ ## name [0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); \
ggml_vk_create_pipeline(device, device->pipeline_ ## name [1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);
@ -4939,7 +5103,8 @@ static void ggml_vk_load_shaders(vk_device& device) {
// conv2d, conv_transpose_2d
for (uint32_t s = 0; s < CONV_SHAPE_COUNT; ++s) {
uint32_t conv2d_WG_SIZE = 256;
// smaller WG for the small-tile fallback gives more concurrent WGs per SM
uint32_t conv2d_WG_SIZE = (s == CONV_SHAPE_64x32) ? 128 : 256;
uint32_t use_collectives = 0; // Enables subgroup ops for preventing the re-calculation of indices.
uint32_t conv2d_TS_K = (s == CONV_SHAPE_64x32) ? 4 : 8;
uint32_t conv2d_SHMEM_PAD = 4;
@ -4978,18 +5143,77 @@ static void ggml_vk_load_shaders(vk_device& device) {
conv2d_BS.CRS); // CRS block size should be capped at subgroup size for correctness when shuffle is used.
}
uint32_t conv2d_shmem_req =
(conv2d_BS.K * (conv2d_BS.CRS + conv2d_SHMEM_PAD) + conv2d_BS.CRS * (conv2d_BS.NPQ + conv2d_SHMEM_PAD)) * sizeof(float);
if (device->properties.limits.maxComputeSharedMemorySize < conv2d_shmem_req) {
// cm1 is used only when cm2 is unavailable; capped at 64x128 (due to shared memory size).
// Requires 16x16x16 f16-acc since that's the fragment shape hard-coded in the shader.
// Subgroup size must be 32 or 64 (to keep WG_SIZE sane) and we need
// subgroup_size_control to force the driver to actually use it.
bool conv2d_use_cm1 = false;
#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT)
conv2d_use_cm1 = !device->coopmat2 &&
device->coopmat_support && device->coopmat_support_16x16x16_f16acc &&
device->subgroup_size_control &&
(device->subgroup_size == 32 || device->subgroup_size == 64) &&
s != CONV_SHAPE_128x128;
#endif
const uint32_t conv2d_cm1_shmem_pad = 8;
auto shmem_req = [&](uint32_t pad, bool csh_store, bool fp16_shmem) {
const uint32_t elem_size = fp16_shmem ? (uint32_t)sizeof(uint16_t) : (uint32_t)sizeof(float);
const uint32_t csh_elems = csh_store ? conv2d_BS.K * conv2d_BS.NPQ : 0u;
return (conv2d_BS.K * (conv2d_BS.CRS + pad) + conv2d_BS.CRS * (conv2d_BS.NPQ + pad) + csh_elems) * elem_size;
};
// coopmat1 needs to store the output through shared memory, so check up front
// whether it'll fit and disable it before applying coopmat1 parameters.
if (conv2d_use_cm1 && device->properties.limits.maxComputeSharedMemorySize < shmem_req(conv2d_cm1_shmem_pad, true, true)) {
conv2d_use_cm1 = false;
}
uint32_t conv2d_WM = 16, conv2d_WN = 16; // cm1 subgroup tile, ignored otherwise
if (conv2d_use_cm1) {
conv2d_SHMEM_PAD = conv2d_cm1_shmem_pad;
// 16x16x16 fragments; pick WM/WN to keep WG_SIZE at 256
// (i.e. 8 subgroups for sg=32, 4 subgroups for sg=64).
const bool sg64 = (device->subgroup_size == 64);
switch (s) {
case CONV_SHAPE_64x32: conv2d_WM = sg64 ? 32 : 16; conv2d_WN = 16; break;
case CONV_SHAPE_64x128: conv2d_WM = 32; conv2d_WN = sg64 ? 64 : 32; break;
case CONV_SHAPE_32x256: conv2d_WM = sg64 ? 16 : 32; conv2d_WN = sg64 ? 128 : 32; break;
default: break;
}
const uint32_t warps_M = conv2d_BS.K / conv2d_WM;
const uint32_t warps_N = conv2d_BS.NPQ / conv2d_WN;
conv2d_WG_SIZE = warps_M * warps_N * device->subgroup_size;
}
// stage cm2 accumulator through shmem for coalesced global stores;
// skipped on 128x128 where the extra Csh footprint hurts occupancy.
// cm1 always uses the staged path.
uint32_t conv2d_csh_store = (device->coopmat2 && s != CONV_SHAPE_128x128) ? 1u : 0u;
if (conv2d_use_cm1) {
conv2d_csh_store = 1;
}
// shmem is fp16 on cm2/cm1 (matches Csh), fp32 on scalar
const bool conv2d_use_fp16_shmem = device->coopmat2 || conv2d_use_cm1;
// shrink CRS if the non-cm1 config still doesn't fit
if (device->properties.limits.maxComputeSharedMemorySize < shmem_req(conv2d_SHMEM_PAD, conv2d_csh_store, conv2d_use_fp16_shmem)) {
GGML_ASSERT(!conv2d_use_cm1);
conv2d_BS.CRS = 8;
if (use_collectives) {
conv2d_BS.CRS = std::min(device->subgroup_size, conv2d_BS.CRS);
}
conv2d_csh_store = 0;
}
std::array<uint32_t, 3> wg_denoms = { conv2d_BS.K, 1, 1 };
std::vector<uint32_t> spec_constants = { conv2d_WG_SIZE, conv2d_BS.K, conv2d_BS.CRS, conv2d_BS.NPQ, conv2d_TS_K, use_collectives, conv2d_SHMEM_PAD };
// cm1 needs a fixed subgroup width to match the WG_SIZE we computed
const uint32_t conv2d_required_subgroup_size = conv2d_use_cm1 ? device->subgroup_size : 0;
#define CREATE_CONV(name, type_suffix, spv_suffix) \
for (auto &c : device->pipeline_##name##type_suffix[s]) { \
const vk_conv2d_pipeline_state &state = c.first; \
@ -5002,10 +5226,14 @@ static void ggml_vk_load_shaders(vk_device& device) {
spec_constants_cpy.push_back(state.d1); \
spec_constants_cpy.push_back(state.KW); \
spec_constants_cpy.push_back(state.KH); \
spec_constants_cpy.push_back(state.aligned); \
spec_constants_cpy.push_back(conv2d_csh_store); \
spec_constants_cpy.push_back(conv2d_WM); \
spec_constants_cpy.push_back(conv2d_WN); \
ggml_vk_create_pipeline( \
device, c.second, #name #type_suffix, \
name##type_suffix##spv_suffix##_len, name##type_suffix##spv_suffix##_data, "main", 3, \
sizeof(vk_op_conv2d_push_constants), wg_denoms, spec_constants_cpy, 1, true, use_collectives); \
sizeof(vk_op_conv2d_push_constants), wg_denoms, spec_constants_cpy, 1, true, use_collectives || conv2d_required_subgroup_size, conv2d_required_subgroup_size); \
}
#define CREATE_CONVS(spv_suffix) \
CREATE_CONV(conv2d, _f32, spv_suffix) \
@ -5016,6 +5244,11 @@ static void ggml_vk_load_shaders(vk_device& device) {
if (device->coopmat2) {
CREATE_CONVS(_cm2)
} else
#endif
#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT)
if (conv2d_use_cm1) {
CREATE_CONVS(_cm1)
} else
#endif
if (conv2d_UNROLL) {
CREATE_CONVS(_unroll)
@ -5094,6 +5327,7 @@ static vk_device ggml_vk_get_device(size_t idx) {
bool amd_shader_core_properties2 = false;
bool pipeline_robustness = false;
bool coopmat2_support = false;
bool coopmat2_decode_vector_support = false;
bool pipeline_executable_properties_support = false;
device->coopmat_support = false;
device->integer_dot_product = false;
@ -5128,6 +5362,9 @@ static vk_device ggml_vk_get_device(size_t idx) {
!getenv("GGML_VK_DISABLE_COOPMAT2")) {
coopmat2_support = true;
#endif
} else if (strcmp(VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME, properties.extensionName) == 0 &&
!getenv("GGML_VK_DISABLE_COOPMAT2_DECODE_VECTOR")) {
coopmat2_decode_vector_support = true;
#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)
} else if (strcmp("VK_KHR_shader_integer_dot_product", properties.extensionName) == 0 &&
!getenv("GGML_VK_DISABLE_INTEGER_DOT_PRODUCT")) {
@ -5413,6 +5650,14 @@ static vk_device ggml_vk_get_device(size_t idx) {
}
#endif
VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV coopmat2_decode_vector_features {};
coopmat2_decode_vector_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_DECODE_VECTOR_FEATURES_NV;
if (coopmat2_decode_vector_support) {
last_struct->pNext = (VkBaseOutStructure *)&coopmat2_decode_vector_features;
last_struct = (VkBaseOutStructure *)&coopmat2_decode_vector_features;
device_extensions.push_back(VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME);
}
#if defined(VK_KHR_shader_bfloat16)
VkPhysicalDeviceShaderBfloat16FeaturesKHR bfloat16_features {};
bfloat16_features.pNext = nullptr;
@ -5572,6 +5817,7 @@ static vk_device ggml_vk_get_device(size_t idx) {
found_fp32_128 && found_fp32_256 &&
coopmat2_props.cooperativeMatrixFlexibleDimensionsMaxDimension >= 512) {
device->coopmat2 = true;
device->coopmat2_decode_vector = coopmat2_decode_vector_support && coopmat2_decode_vector_features.cooperativeMatrixDecodeVector;
}
}
#endif
@ -5787,8 +6033,12 @@ static vk_device ggml_vk_get_device(size_t idx) {
ggml_vk_load_shaders(device);
// Only use transfer queue on AMD non-GCN, when the graphics queue is not enabled
const bool prefers_transfer_queue = device->vendor_id == VK_VENDOR_ID_AMD && device->architecture != AMD_GCN && !allow_graphics_queue;
// Prefer a dedicated transfer queue on AMD dGPUs (non-GCN) when graphics queue use is disabled.
const bool prefers_transfer_queue =
device->vendor_id == VK_VENDOR_ID_AMD &&
device->architecture != AMD_GCN &&
!device->uma &&
!allow_graphics_queue;
if (!device->single_queue) {
const uint32_t transfer_queue_index = compute_queue_family_index == transfer_queue_family_index ? 1 : 0;
@ -5854,6 +6104,7 @@ static void ggml_vk_print_gpu_info(size_t idx) {
bool fp16_compute = false;
bool coopmat_support = false;
bool coopmat2_support = false;
bool coopmat2_decode_vector_support = false;
bool integer_dot_product = false;
bool bfloat16_support = false;
@ -5872,6 +6123,9 @@ static void ggml_vk_print_gpu_info(size_t idx) {
!getenv("GGML_VK_DISABLE_COOPMAT2")) {
coopmat2_support = true;
#endif
} else if (strcmp(VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME, properties.extensionName) == 0 &&
!getenv("GGML_VK_DISABLE_COOPMAT2_DECODE_VECTOR")) {
coopmat2_decode_vector_support = true;
#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)
} else if (strcmp("VK_KHR_shader_integer_dot_product", properties.extensionName) == 0 &&
!getenv("GGML_VK_DISABLE_INTEGER_DOT_PRODUCT")) {
@ -5956,6 +6210,13 @@ static void ggml_vk_print_gpu_info(size_t idx) {
}
#endif
VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV coopmat2_decode_vector_features {};
coopmat2_decode_vector_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_DECODE_VECTOR_FEATURES_NV;
if (coopmat2_decode_vector_support) {
last_struct->pNext = (VkBaseOutStructure *)&coopmat2_decode_vector_features;
last_struct = (VkBaseOutStructure *)&coopmat2_decode_vector_features;
}
vkGetPhysicalDeviceFeatures2(physical_device, &device_features2);
fp16 = fp16 && vk12_features.shaderFloat16;
@ -5980,7 +6241,14 @@ static void ggml_vk_print_gpu_info(size_t idx) {
#endif
&& ggml_vk_khr_cooperative_matrix_support(props2.properties, driver_props, device_architecture);
std::string matrix_cores = coopmat2_support ? "NV_coopmat2" : coopmat_support ? "KHR_coopmat" : "none";
coopmat2_decode_vector_support = coopmat2_decode_vector_support && coopmat2_decode_vector_features.cooperativeMatrixDecodeVector;
#if !defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT)
coopmat2_decode_vector_support = false;
#endif
std::string matrix_cores = coopmat2_support ? (coopmat2_decode_vector_support ? "NV_coopmat2v" : "NV_coopmat2")
: coopmat_support ? "KHR_coopmat"
: "none";
std::string device_name = props2.properties.deviceName.data();
GGML_LOG_DEBUG("ggml_vulkan: %zu = %s (%s) | uma: %d | fp16: %d | bf16: %d | warp size: %zu | shared memory: %d | int dot: %d | matrix cores: %s\n",
@ -9500,10 +9768,23 @@ static vk_conv_shapes ggml_vk_conv_select_shape(ggml_backend_vk_context * ctx, u
// so small convolutions will still choose a smaller tile.
const uint32_t shader_core_count = ctx->device->shader_core_count > 0 ? ctx->device->shader_core_count : 32;
if (K > 64 && n_tiles(CONV_SHAPE_128x128) >= shader_core_count * 2) {
// 128x128 isn't used with cm1 due to shared memory size; fall through to a smaller tile.
bool allow_128x128 = true;
#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT)
if (!ctx->device->coopmat2 && ctx->device->coopmat_support && ctx->device->coopmat_support_16x16x16_f16acc) {
allow_128x128 = false;
}
#endif
if (allow_128x128 && K > 64 && n_tiles(CONV_SHAPE_128x128) >= shader_core_count * 2) {
return CONV_SHAPE_128x128;
} else if (K <= 32 && n_tiles(CONV_SHAPE_32x256) >= shader_core_count * 2) {
return CONV_SHAPE_32x256;
} else if (K <= 64 && n_tiles(CONV_SHAPE_64x128) >= shader_core_count * 2) {
return CONV_SHAPE_64x128;
} else if (!allow_128x128 && K > 64 && n_tiles(CONV_SHAPE_64x128) >= shader_core_count * 2) {
// cm1 fallback for large K when 128x128 isn't available
return CONV_SHAPE_64x128;
} else {
return CONV_SHAPE_64x32;
}
@ -9675,7 +9956,10 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
return nullptr;
case GGML_OP_REPEAT:
if (ggml_type_size(src0->type) == sizeof(float) && ggml_type_size(dst->type) == sizeof(float)) {
return ctx->device->pipeline_repeat_f32;
return ctx->device->pipeline_repeat_i32;
}
if (ggml_type_size(src0->type) == 2 && ggml_type_size(dst->type) == 2) {
return ctx->device->pipeline_repeat_i16;
}
return nullptr;
case GGML_OP_REPEAT_BACK:
@ -10035,7 +10319,18 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
uint32_t p1 = !transpose ? (uint32_t)ggml_get_op_params_i32(dst, 3) : 0;
uint32_t d0 = !transpose ? (uint32_t)ggml_get_op_params_i32(dst, 4) : 1;
uint32_t d1 = !transpose ? (uint32_t)ggml_get_op_params_i32(dst, 5) : 1;
vk_conv2d_pipeline_state conv2d_pipeline_state(s0, s1, p0, p1, d0, d1, KW, KH);
// tile-aligned shapes let the shader skip bounds checks
const uint32_t Cin = (uint32_t)src1->ne[2];
const uint32_t CRS = Cin * KW * KH;
const uint32_t BS_K = vk_conv_block_sizes[shape].K;
const uint32_t BS_CRS = vk_conv_block_sizes[shape].CRS;
const uint32_t BS_NPQ = vk_conv_block_sizes[shape].NPQ;
const uint32_t aligned = ((K % BS_K == 0) &&
(CRS % BS_CRS == 0) &&
(NPQ % BS_NPQ == 0)) ? 1u : 0u;
vk_conv2d_pipeline_state conv2d_pipeline_state(s0, s1, p0, p1, d0, d1, KW, KH, aligned);
std::map<vk_conv2d_pipeline_state, vk_pipeline> *pipelines = nullptr;
if (op == GGML_OP_CONV_2D) {
@ -16179,7 +16474,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
return false;
}
case GGML_OP_REPEAT:
return ggml_type_size(op->type) == sizeof(float) && ggml_type_size(op->src[0]->type) == sizeof(float);
return ggml_type_size(op->type) == ggml_type_size(op->src[0]->type) &&
(ggml_type_size(op->type) == sizeof(float) || ggml_type_size(op->type) == 2);
case GGML_OP_REPEAT_BACK:
return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32;
case GGML_OP_ROPE:

159
ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp
View file

@ -7,6 +7,13 @@
#extension GL_KHR_memory_scope_semantics : enable
#endif
#ifdef COOPMAT
#extension GL_KHR_cooperative_matrix : enable
#extension GL_KHR_shader_subgroup_basic : enable
#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require
#extension GL_KHR_memory_scope_semantics : enable
#endif
#ifdef USE_COLLECTIVES
# extension GL_KHR_shader_subgroup_shuffle : enable
#endif
@ -77,6 +84,39 @@ layout(constant_id = 12) const uint d1 = 1;
// Kernel spatial sizes
layout(constant_id = 13) const uint KW = 1;
layout(constant_id = 14) const uint KH = 1;
// when set, skip bounds checks and address clamps (K/CRS/NPQ are tile-aligned)
layout(constant_id = 15) const uint aligned = 0;
// stage cm2 result through shmem (Csh) for coalesced stores. cm1 always does this.
layout(constant_id = 16) const uint csh_store = 0;
#ifdef COOPMAT
// cm1 subgroup tile: each subgroup computes a WM x WN region as a grid of
// TM x TN x TK fragments. Requires WM%TM == WN%TN == BS_K%WM == BS_NPQ%WN ==
// BS_CRS%TK == 0, and WG_SIZE == (BS_K/WM) * (BS_NPQ/WN) * subgroup_size.
layout(constant_id = 17) const uint WM = 32;
layout(constant_id = 18) const uint WN = 32;
const uint TM = 16;
const uint TN = 16;
const uint TK = 16;
const uint cms_per_row = WM / TM;
const uint cms_per_col = WN / TN;
const uint warps_M = BS_K / WM;
const uint warps_N = BS_NPQ / WN;
#endif
// without padding, H_idx/W_idx are in bounds by construction (non-TRANSPOSE only)
#ifdef TRANSPOSE
const bool hw_in_bounds = false;
#else
const bool hw_in_bounds = (p0 == 0) && (p1 == 0);
#endif
// TRANSPOSE stride alignment is trivially satisfied for stride 1
#ifdef TRANSPOSE
const bool stride_in_bounds = (s0 == 1) && (s1 == 1);
#else
const bool stride_in_bounds = true;
#endif
uint32_t tid = gl_LocalInvocationID.x;
const uint32_t WG_SIZE = gl_WorkGroupSize.x;
@ -94,7 +134,7 @@ uint32_t n_elems_out = K * NPQ;
// Number of blocktiles per input
uint32_t NB_CRS = splitWork(CRS, BS_CRS);
#ifdef COOPMAT2
#if defined(COOPMAT2) || defined(COOPMAT)
#define SHMEM_TYPE float16_t
#else
#define SHMEM_TYPE float
@ -112,6 +152,17 @@ const uint32_t Bsh_len = BS_CRS * Bsh_stride;
shared SHMEM_TYPE Ash[Ash_len]; // K x CRS
shared SHMEM_TYPE Bsh[Bsh_len]; // CRS x NPQ
#if defined(COOPMAT2) || defined(COOPMAT)
// stage matC through shmem so global stores are row-major (NPQ-contiguous)
const uint32_t Csh_stride = BS_NPQ;
#ifdef COOPMAT
const uint32_t Csh_len = BS_K * Csh_stride;
#else
const uint32_t Csh_len = csh_store != 0 ? BS_K * Csh_stride : 1;
#endif
shared SHMEM_TYPE Csh[Csh_len]; // K x NPQ
#endif
// Threadtile sizes
const uint32_t TS_NPQ = BS_K * BS_NPQ / WG_SIZE / TS_K;
@ -161,7 +212,7 @@ ACC_TYPE perElemOpStore(const in uint32_t r, const in uint32_t c, const in ACC_T
uint32_t OH_idx = fastdiv(NPQ_idx - N_idx * p.OH * p.OW, p.OWmp, p.OWL); // divide by p.OW;
uint32_t OW_idx = NPQ_idx - N_idx * p.OH * p.OW - OH_idx * p.OW;
uint32_t dst_idx = OW_idx + OH_idx * p.nb1 + K_idx * p.nb2 + N_idx * p.nb3;
if (K_idx < K && NPQ_idx < NPQ) {
if (aligned != 0 || (K_idx < K && NPQ_idx < NPQ)) {
dst_data[dst_idx] = D_TYPE(elem);
}
return elem;
@ -176,6 +227,13 @@ void main() {
#ifdef COOPMAT2
coopmat<ACC_TYPE, gl_ScopeWorkgroup, BS_K, BS_NPQ, gl_MatrixUseAccumulator> matC;
matC = coopmat<ACC_TYPE, gl_ScopeWorkgroup, BS_K, BS_NPQ, gl_MatrixUseAccumulator>(0.0);
#elif defined(COOPMAT)
coopmat<float16_t, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator> sums[cms_per_row * cms_per_col];
[[unroll]] for (uint i = 0; i < cms_per_row * cms_per_col; i++) {
sums[i] = coopmat<float16_t, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator>(0.0);
}
const uint warp_r = gl_SubgroupID / warps_N;
const uint warp_c = gl_SubgroupID % warps_N;
#else
float regC[TS_K][TS_NPQ];
for (uint32_t T_ly = 0; T_ly < TS_K; T_ly++) {
@ -228,12 +286,15 @@ void main() {
uint32_t B_lx = Ac;
uint32_t K_idx = B_idx_K * BS_K + B_ly; /* Global K_idx (row index of A)*/
#ifdef TRANSPOSE
uint32_t knl_idx = min(KW_idx_a + KH_idx_a * p.nb01 + K_idx * p.nb02 + Cin_idx_a * p.nb03, K * CRS - 1);
uint32_t knl_idx = KW_idx_a + KH_idx_a * p.nb01 + K_idx * p.nb02 + Cin_idx_a * p.nb03;
#else
uint32_t knl_idx = min(KW_idx_a + KH_idx_a * p.nb01 + Cin_idx_a * p.nb02 + K_idx * p.nb03, K * CRS - 1);
uint32_t knl_idx = KW_idx_a + KH_idx_a * p.nb01 + Cin_idx_a * p.nb02 + K_idx * p.nb03;
#endif
if (aligned == 0) {
knl_idx = min(knl_idx, K * CRS - 1);
}
float val = knl_data[knl_idx];
if (K_idx >= K || CRS_idx_a >= CRS) {
if (aligned == 0 && (K_idx >= K || CRS_idx_a >= CRS)) {
val = 0.0;
}
Ash[B_ly * Ash_stride + B_lx] = SHMEM_TYPE(val);
@ -282,15 +343,27 @@ void main() {
uint32_t H_idx = OH_idx * s1 + KH_idx_b * d1 - p1;
uint32_t W_idx = OW_idx * s0 + KW_idx_b * d0 - p0;
#endif
uint32_t src_idx =
min(max(W_idx + H_idx * p.nb11 + Cin_idx_b * p.nb12 + N_idx * p.nb13, 0), p.Cin * p.N * p.W * p.H - 1);
uint32_t src_idx = W_idx + H_idx * p.nb11 + Cin_idx_b * p.nb12 + N_idx * p.nb13;
// skip clamp when address can't go OOB
if (aligned == 0 || !hw_in_bounds || !stride_in_bounds) {
src_idx = min(max(src_idx, 0), p.Cin * p.N * p.W * p.H - 1);
}
float val = src_data[src_idx];
if (CRS_idx_b >= CRS || NPQ_idx >= NPQ
|| H_idx >= p.H || W_idx >= p.W // Lower bound checks aren't necessary. (idx >= 0x80000000 for such case)
bool oob = false;
if (aligned == 0 && (CRS_idx_b >= CRS || NPQ_idx >= NPQ)) {
oob = true;
}
// also catches lower-bound underflow (idx wraps to 0x80000000+)
if (!hw_in_bounds && (H_idx >= p.H || W_idx >= p.W)) {
oob = true;
}
#ifdef TRANSPOSE
|| (H_idx_x_s1 - H_idx * s1 != 0) || (W_idx_x_s0 - W_idx * s0 != 0)
if (!stride_in_bounds &&
((H_idx_x_s1 - H_idx * s1 != 0) || (W_idx_x_s0 - W_idx * s0 != 0))) {
oob = true;
}
#endif
) {
if (oob) {
val = 0.0;
}
Bsh[B_ly * Bsh_stride + B_lx] = SHMEM_TYPE(val);
@ -303,6 +376,23 @@ void main() {
coopMatLoad(matA, Ash, 0, Ash_stride, gl_CooperativeMatrixLayoutRowMajor);
coopMatLoad(matB, Bsh, 0, Bsh_stride, gl_CooperativeMatrixLayoutRowMajor);
matC = coopMatMulAdd(matA, matB, matC);
#elif defined(COOPMAT)
// each subgroup multiplies its grid of fragments per TK-sized CRS chunk
[[unroll]] for (uint k_step = 0; k_step < BS_CRS / TK; k_step++) {
coopmat<float16_t, gl_ScopeSubgroup, TM, TK, gl_MatrixUseA> cache_a[cms_per_row];
[[unroll]] for (uint cm_row = 0; cm_row < cms_per_row; cm_row++) {
const uint a_off = (warp_r * WM + cm_row * TM) * Ash_stride + k_step * TK;
coopMatLoad(cache_a[cm_row], Ash, a_off, Ash_stride, gl_CooperativeMatrixLayoutRowMajor);
}
[[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) {
coopmat<float16_t, gl_ScopeSubgroup, TK, TN, gl_MatrixUseB> cache_b;
const uint b_off = k_step * TK * Bsh_stride + warp_c * WN + cm_col * TN;
coopMatLoad(cache_b, Bsh, b_off, Bsh_stride, gl_CooperativeMatrixLayoutRowMajor);
[[unroll]] for (uint cm_row = 0; cm_row < cms_per_row; cm_row++) {
sums[cm_col * cms_per_row + cm_row] = coopMatMulAdd(cache_a[cm_row], cache_b, sums[cm_col * cms_per_row + cm_row]);
}
}
}
#else
if (T_y * TS_K < K) {
UNROLL for (uint32_t CRS_lidx = 0; CRS_lidx < BS_CRS; CRS_lidx++) {
@ -325,8 +415,51 @@ void main() {
barrier();
}
/* Save C* */
#if defined(COOPMAT2) || defined(COOPMAT)
// stage matC into Csh, then write to dst with coalesced NPQ-contiguous stores
#ifdef COOPMAT
const bool use_staged_store = true;
#else
const bool use_staged_store = (csh_store != 0);
#endif
if (use_staged_store) {
#ifdef COOPMAT
// cm1: each subgroup stores its fragment grid into its Csh slot
[[unroll]] for (uint cm_row = 0; cm_row < cms_per_row; cm_row++) {
[[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) {
const uint csh_off = (warp_r * WM + cm_row * TM) * Csh_stride + warp_c * WN + cm_col * TN;
coopMatStore(sums[cm_col * cms_per_row + cm_row], Csh, csh_off, Csh_stride, gl_CooperativeMatrixLayoutRowMajor);
}
}
#else
coopMatStore(matC, Csh, 0, Csh_stride, gl_CooperativeMatrixLayoutRowMajor);
#endif
barrier();
// cooperative shmem->global: WG threads spread across BS_NPQ (the
// contiguous direction of dst), each iter covers store_rows_per_iter K-rows
const uint32_t store_rows_per_iter = WG_SIZE / BS_NPQ;
const uint32_t store_iters = BS_K / store_rows_per_iter;
const uint32_t k_thread_offset = tid / BS_NPQ;
const uint32_t npq_thread = tid % BS_NPQ;
[[unroll]] for (uint32_t i = 0; i < store_iters; i++) {
uint32_t k_local = i * store_rows_per_iter + k_thread_offset;
uint32_t K_idx = B_idx_K * BS_K + k_local;
uint32_t NPQ_idx = B_idx_NPQ * BS_NPQ + npq_thread;
uint32_t N_idx = fastdiv(NPQ_idx, p.OWOHmp, p.OWOHL);
uint32_t OH_idx = fastdiv(NPQ_idx - N_idx * p.OH * p.OW, p.OWmp, p.OWL);
uint32_t OW_idx = NPQ_idx - N_idx * p.OH * p.OW - OH_idx * p.OW;
uint32_t dst_idx = OW_idx + OH_idx * p.nb1 + K_idx * p.nb2 + N_idx * p.nb3;
if (aligned != 0 || (K_idx < K && NPQ_idx < NPQ)) {
dst_data[dst_idx] = D_TYPE(Csh[k_local * Csh_stride + npq_thread]);
}
}
}
#ifdef COOPMAT2
coopMatPerElementNV(matC, matC, perElemOpStore);
else {
coopMatPerElementNV(matC, matC, perElemOpStore);
}
#endif
#else
if (T_y * TS_K < K) {
for (uint32_t T_ly = 0; T_ly < TS_K; T_ly++) {
@ -337,7 +470,7 @@ void main() {
uint32_t OH_idx = fastdiv(NPQ_idx - N_idx * p.OH * p.OW, p.OWmp, p.OWL); // divide by p.OW;
uint32_t OW_idx = NPQ_idx - N_idx * p.OH * p.OW - OH_idx * p.OW;
uint32_t dst_idx = OW_idx + OH_idx * p.nb1 + K_idx * p.nb2 + N_idx * p.nb3;
if (K_idx < K && NPQ_idx < NPQ) {
if (aligned != 0 || (K_idx < K && NPQ_idx < NPQ)) {
dst_data[dst_idx] = regC[T_ly][T_lx];
}
}

39
ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl
View file

@ -5,21 +5,60 @@
#include "types.glsl"
#if defined(DATA_A_F32)
FLOAT_TYPE dequantize1(uint ib, uint iqs, uint a_offset) {
return data_a[a_offset + ib];
}
vec2 dequantize(uint ib, uint iqs, uint a_offset) {
return vec2(data_a[a_offset + ib], data_a[a_offset + ib + 1]);
}
vec4 dequantize4(uint ib, uint iqs, uint a_offset) {
return vec4(data_a[a_offset + ib ], data_a[a_offset + ib + 1],
data_a[a_offset + ib + 2], data_a[a_offset + ib + 3]);
}
vec4 dequantize4_2aligned(uint ib, uint iqs, uint a_offset) {
return vec4(data_a[a_offset + ib ], data_a[a_offset + ib + 1],
data_a[a_offset + ib + 2], data_a[a_offset + ib + 3]);
}
#endif
#if defined(DATA_A_F16)
FLOAT_TYPE dequantize1(uint ib, uint iqs, uint a_offset) {
return data_a[a_offset + ib];
}
vec2 dequantize(uint ib, uint iqs, uint a_offset) {
return vec2(data_a[a_offset + ib], data_a[a_offset + ib + 1]);
}
vec4 dequantize4(uint ib, uint iqs, uint a_offset) {
return vec4(data_a[a_offset + ib ], data_a[a_offset + ib + 1],
data_a[a_offset + ib + 2], data_a[a_offset + ib + 3]);
}
vec4 dequantize4_2aligned(uint ib, uint iqs, uint a_offset) {
const vec2 a = data_a_packed32[(a_offset + ib)/2];
const vec2 b = data_a_packed32[(a_offset + ib)/2 + 1];
return vec4(a, b);
}
#endif
#if defined(DATA_A_BF16)
FLOAT_TYPE dequantize1(uint ib, uint iqs, uint a_offset) {
return bf16_to_fp32(data_a[a_offset + ib]);
}
vec2 dequantize(uint ib, uint iqs, uint a_offset) {
return vec2(bf16_to_fp32(data_a[a_offset + ib]), bf16_to_fp32(data_a[a_offset + ib + 1]));
}
vec4 dequantize4(uint ib, uint iqs, uint a_offset) {
return vec4(bf16_to_fp32(data_a[a_offset + ib ]), bf16_to_fp32(data_a[a_offset + ib + 1]),
bf16_to_fp32(data_a[a_offset + ib + 2]), bf16_to_fp32(data_a[a_offset + ib + 3]));
}
vec4 dequantize4_2aligned(uint ib, uint iqs, uint a_offset) {
const uint a = data_a_packed32[(a_offset + ib)/2];
const uint b = data_a_packed32[(a_offset + ib)/2 + 1];
return vec4(uintBitsToFloat((a & 0x0000ffff) << 16),
uintBitsToFloat( a & 0xffff0000),
uintBitsToFloat((b & 0x0000ffff) << 16),
uintBitsToFloat( b & 0xffff0000));
}
#endif
#if defined(DATA_A_Q4_0)

608
ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl
View file

@ -1,4 +1,12 @@
// Each format defines a scalar dequantFunc<T> plus a V=4 dequantFunc<T>_v
// passed as the optional vector decoder to coopMatLoadTensorNV via
// GL_NV_cooperative_matrix_decode_vector. When the driver doesn't support
// the extension, ggml-vulkan.cpp strips it from the compiled SPIR-V.
#ifdef GL_NV_cooperative_matrix_decode_vector
#extension GL_NV_cooperative_matrix_decode_vector : enable
#endif
#include "types.glsl"
layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufF32 {
@ -25,6 +33,19 @@ float16_t dequantFuncQ1_0(const in decodeBufQ1_0 bl, const in uint blockCoords[2
return bit != 0u ? d : -d;
}
f16vec4 dequantFuncQ1_0_v(const in decodeBufQ1_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
const float16_t d = bl.block.d;
const float16_t md = -d;
const uint idx = coordInBlock[1];
const uint qs_nib = uint(bl.block.qs[idx >> 3]) >> (idx & 0x4u);
return f16vec4(
(qs_nib & 1u) != 0u ? d : md,
(qs_nib & 2u) != 0u ? d : md,
(qs_nib & 4u) != 0u ? d : md,
(qs_nib & 8u) != 0u ? d : md);
}
layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ4_0 {
block_q4_0_packed16 block;
};
@ -42,10 +63,28 @@ float16_t dequantFuncQ4_0(const in decodeBufQ4_0 bl, const in uint blockCoords[2
return ret;
}
f16vec4 dequantFuncQ4_0_v(const in decodeBufQ4_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
const float16_t d = bl.block.d;
const uint idx = coordInBlock[1];
const uint shift = (idx & 0x10) >> 2; // 0 or 4
const uint qs_i = (idx & 0xE) >> 1; // even, in {0,2,4,6}
const uint qsw = uint32_t(bl.block.qs[qs_i ])
| (uint32_t(bl.block.qs[qs_i + 1u]) << 16);
// shift in {0,4}: per-byte mask 0x0F isolates the wanted nibble in each byte.
const uint q4 = (qsw >> shift) & 0x0F0F0F0Fu;
const u8vec4 q = unpack8(q4);
return f16vec4((vec4(q) - vec4(8.0)) * vec4(float(d)));
}
layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufQ4_1 {
block_q4_1 block;
};
layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufQ4_1_packed32 {
block_q4_1_packed32 block;
};
float16_t dequantFuncQ4_1(const in decodeBufQ4_1 bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
const float16_t d = bl.block.d;
@ -60,10 +99,27 @@ float16_t dequantFuncQ4_1(const in decodeBufQ4_1 bl, const in uint blockCoords[2
return ret;
}
f16vec4 dequantFuncQ4_1_v(const in decodeBufQ4_1 bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
decodeBufQ4_1_packed32 bl32 = decodeBufQ4_1_packed32(bl);
const float16_t d = bl.block.d;
const float16_t m = bl.block.m;
const uint idx = coordInBlock[1];
const uint shift = (idx & 0x10) >> 2; // 0 or 4
const uint qs_w = (idx & 0xC) >> 2; // iqs / 4 in [0,4)
const uint qsw = uint32_t(bl32.block.qs[qs_w]);
const u8vec4 q = unpack8((qsw >> shift) & 0x0F0F0F0Fu);
return f16vec4(vec4(q) * vec4(float(d)) + vec4(float(m)));
}
layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ5_0 {
block_q5_0 block;
};
layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ5_0_packed16 {
block_q5_0_packed16 block;
};
float16_t dequantFuncQ5_0(const in decodeBufQ5_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
const float16_t d = bl.block.d;
@ -82,10 +138,32 @@ float16_t dequantFuncQ5_0(const in decodeBufQ5_0 bl, const in uint blockCoords[2
return ret;
}
f16vec4 dequantFuncQ5_0_v(const in decodeBufQ5_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
decodeBufQ5_0_packed16 bl16 = decodeBufQ5_0_packed16(bl);
const float16_t d = bl.block.d;
const uint idx = coordInBlock[1];
const uint shift = (idx & 0x10) >> 2; // 0 or 4
const uint qs_i = (idx & 0xC) >> 1; // packed16 word index, in {0,2,4,6}
const uint qsw = uint32_t(bl16.block.qs[qs_i ])
| (uint32_t(bl16.block.qs[qs_i + 1u]) << 16);
const u8vec4 ql = unpack8((qsw >> shift) & 0x0F0F0F0Fu);
const uint uint_qh = uint(bl16.block.qh[1]) << 16 | uint(bl16.block.qh[0]);
const uint qh_pack = uint_qh >> idx; // bits 0..3 = element idx..idx+3 high bits
const uvec4 qh_high = (uvec4(qh_pack, qh_pack >> 1u, qh_pack >> 2u, qh_pack >> 3u) & uvec4(0x01u)) << 4u;
return f16vec4((vec4(ql) + vec4(qh_high) - vec4(16.0)) * vec4(float(d)));
}
layout(buffer_reference, std430, buffer_reference_align = 8) buffer decodeBufQ5_1 {
block_q5_1 block;
};
layout(buffer_reference, std430, buffer_reference_align = 8) buffer decodeBufQ5_1_packed32 {
block_q5_1_packed32 block;
};
float16_t dequantFuncQ5_1(const in decodeBufQ5_1 bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
const float16_t d = bl.block.d;
@ -105,6 +183,23 @@ float16_t dequantFuncQ5_1(const in decodeBufQ5_1 bl, const in uint blockCoords[2
return ret;
}
f16vec4 dequantFuncQ5_1_v(const in decodeBufQ5_1 bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
decodeBufQ5_1_packed32 bl32 = decodeBufQ5_1_packed32(bl);
const float16_t d = bl.block.d;
const float16_t m = bl.block.m;
const uint idx = coordInBlock[1];
const uint shift = (idx & 0x10) >> 2; // 0 or 4
const uint qs_w = (idx & 0xC) >> 2; // iqs / 4 in [0,4)
const uint qsw = uint32_t(bl32.block.qs[qs_w]);
const u8vec4 ql = unpack8((qsw >> shift) & 0x0F0F0F0Fu);
const uint qh_pack = bl.block.qh >> idx; // bits 0..3 = element idx..idx+3 high bits
const uvec4 qh_high = (uvec4(qh_pack, qh_pack >> 1u, qh_pack >> 2u, qh_pack >> 3u) & uvec4(0x01u)) << 4u;
return f16vec4((vec4(ql) + vec4(qh_high)) * vec4(float(d)) + vec4(float(m)));
}
layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ8_0 {
block_q8_0_packed16 block;
};
@ -121,6 +216,17 @@ float16_t dequantFuncQ8_0(const in decodeBufQ8_0 bl, const in uint blockCoords[2
return ret;
}
f16vec4 dequantFuncQ8_0_v(const in decodeBufQ8_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
const float16_t d = bl.block.d;
const uint idx = coordInBlock[1];
const uint base = idx >> 1u;
const uint w = uint(uint16_t(bl.block.qs[base]))
| (uint(uint16_t(bl.block.qs[base + 1u])) << 16u);
const i8vec4 qi = unpack8(int32_t(w));
return f16vec4(vec4(qi) * vec4(float(d)));
}
layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufQ2_K {
block_q2_K block;
};
@ -129,6 +235,10 @@ layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ2
block_q2_K_packed16 block;
};
layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufQ2_K_packed32 {
block_q2_K_packed32 block;
};
float16_t dequantFuncQ2_K(const in decodeBufQ2_K bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
decodeBufQ2_K_packed16 bl16 = decodeBufQ2_K_packed16(bl);
@ -147,10 +257,36 @@ float16_t dequantFuncQ2_K(const in decodeBufQ2_K bl, const in uint blockCoords[2
return ret;
}
f16vec4 dequantFuncQ2_K_v(const in decodeBufQ2_K bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
decodeBufQ2_K_packed32 bl32 = decodeBufQ2_K_packed32(bl);
const f16vec2 dm = bl.block.dm;
const uint idx = coordInBlock[1];
const uint scalesi = idx >> 4; // 0..15
const uint qsshift = (idx & 0x60) >> 4; // 0,2,4,6
// qs_i (packed16) = ((idx & 0x80) >> 3) + ((idx & 0x1E) >> 1) is even for idx % 4 == 0,
// so qs_w (packed32) = qs_i / 2 = ((idx & 0x80) >> 4) + ((idx & 0x1Cu) >> 2).
const uint qs_w = ((idx & 0x80) >> 4) + ((idx & 0x1Cu) >> 2);
const uint qsw = uint32_t(bl32.block.qs[qs_w]);
const uint qs4 = (qsw >> qsshift) & 0x03030303u;
const u8vec4 qi = unpack8(qs4);
const uint scales = bl.block.scales[scalesi];
const float16_t d_sub = dm.x * float16_t(scales & 0xF);
const float16_t m_sub = dm.y * float16_t(scales >> 4);
return f16vec4(vec4(qi) * vec4(float(d_sub)) - vec4(float(m_sub)));
}
layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ3_K {
block_q3_K block;
};
layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ3_K_packed16 {
block_q3_K_packed16 block;
};
float16_t dequantFuncQ3_K(const in decodeBufQ3_K bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
const uint idx = coordInBlock[1];
@ -179,6 +315,47 @@ float16_t dequantFuncQ3_K(const in decodeBufQ3_K bl, const in uint blockCoords[2
return ret;
}
f16vec4 dequantFuncQ3_K_v(const in decodeBufQ3_K bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
decodeBufQ3_K_packed16 bl16 = decodeBufQ3_K_packed16(bl);
const uint idx = coordInBlock[1];
const uint n = idx >> 7; // 0,1
const uint is = idx >> 4; // 0..15
const uint halfsplit = (idx & 0x60) >> 5; // 0,1,2,3
const uint qsshift = halfsplit << 1; // 0,2,4,6
const uint hbit = (n << 2) + halfsplit; // 0..7 (bit position in hmask byte)
uint32_t scaleidx0 = (is < 8) ? is : (is - 8);
uint32_t scaleidx0shift = (is < 8) ? 0u : 4u;
uint32_t scaleidx1 = is + 8 - (is / 4) * 4;
uint32_t scaleidx1shift = (is / 4) * 2;
const int8_t us = int8_t(
((bl.block.scales[scaleidx0] >> scaleidx0shift) & 0xF) |
(((bl.block.scales[scaleidx1] >> scaleidx1shift) & 3) << 4));
const float16_t dl = bl.block.d * float16_t(int(us) - 32);
// For idx % 4 == 0: (idx & 0x1F) == (idx & 0x1C) is a multiple of 4.
const uint qsi = (n << 5) + (idx & 0x1Cu);
const uint hmi = (idx & 0x1Cu);
// Two adjacent uint16 packed16 reads, combined into a uint32 in registers.
// After this: byte j of qsw / hmw holds the data for element idx+j.
const uint qsw = uint32_t(bl16.block.qs[qsi >> 1])
| (uint32_t(bl16.block.qs[(qsi >> 1) + 1u]) << 16);
const uint hmw = uint32_t(bl16.block.hmask[hmi >> 1])
| (uint32_t(bl16.block.hmask[(hmi >> 1) + 1u]) << 16);
// qsshift in {0,2,4,6} and hbit in {0..7}: per-byte masks isolate the wanted bits
// with no inter-byte leakage.
const uint ql4 = (qsw >> qsshift) & 0x03030303u;
const uint qh4 = (hmw >> hbit) & 0x01010101u;
const ivec4 q = ivec4(unpack8(ql4 | (qh4 << 2))) - ivec4(4);
return f16vec4(vec4(q) * vec4(float(dl)));
}
layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ4_K {
block_q4_K block;
};
@ -187,6 +364,10 @@ layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ4
block_q4_K_packed16 block;
};
layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ4_K_packed32 {
block_q4_K_packed32 block;
};
layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ4_K_packed128 {
block_q4_K_packed128 block;
};
@ -334,6 +515,55 @@ float16_t dequantFuncQ4_K(const in decodeBufQ4_K bl, const in uint blockCoords[2
return float16_t(ret);
}
f16vec4 dequantFuncQ4_K_v(const in decodeBufQ4_K bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
decodeBufQ4_K_packed32 bl32 = decodeBufQ4_K_packed32(bl);
decodeBufQ4_K_packed128 bl128 = decodeBufQ4_K_packed128(bl);
const uint idx = coordInBlock[1];
const uint is = idx >> 5; // 0..7
#if defined(IS_MUL_MM2) && defined(DATA_A_Q4_K)
vec2 v = shAscales[is * shAscales_stride + (blockCoords[0] % BM)];
float d = v.x;
float m = v.y;
#else
uvec4 v = bl128.block.q4k[0];
const vec2 loadd = vec2(unpackFloat2x16(v.x));
uint32_t sc;
uint32_t mbyte;
uint32_t scale0 = v.y;
uint32_t scale4 = v.z;
uint32_t scale8 = v.w;
uint32_t sc_lo = scale0;
uint32_t mb_lo = scale4;
uint32_t sc_hi = (scale8 & 0x0F0F0F0F) | ((scale0 & 0xC0C0C0C0) >> 2);
uint32_t mb_hi = ((scale8 & 0xF0F0F0F0) >> 4) | ((scale4 & 0xC0C0C0C0) >> 2);
sc = is < 4 ? sc_lo : sc_hi;
mbyte = is < 4 ? mb_lo : mb_hi;
sc = sc >> (8 * (is & 3));
mbyte = mbyte >> (8 * (is & 3));
sc &= 0x3F;
mbyte &= 0x3F;
const float d = loadd.x * float(sc);
const float m = loadd.y * float(mbyte);
#endif
// idx in [0,256); vector decode uses idx a multiple of 4. packed32 word index:
// (qs_i >> 1) == (idx >> 6) * 8 + ((idx & 0x1E) >> 2). sh is 0 or 4 only, so a
// single (w >> sh) & 0x0F0F0F0F isolates all four nibbles without inter-byte leakage.
const uint sh = (idx & 0x20u) >> 3u;
const uint w = uint32_t(bl32.block.qs[(idx >> 6) * 8u + ((idx & 0x1Eu) >> 2)]);
const u8vec4 q = unpack8((w >> sh) & 0x0F0F0F0Fu);
return f16vec4(vec4(d) * vec4(q) - vec4(m));
}
layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ5_K {
block_q5_K block;
};
@ -346,6 +576,10 @@ layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ5
block_q5_K_packed128 block;
};
layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ5_K_packed32 {
block_q5_K_packed32 block;
};
float16_t dequantFuncQ5_K(const in decodeBufQ5_K bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
decodeBufQ5_K_packed16 bl16 = decodeBufQ5_K_packed16(bl);
@ -399,6 +633,58 @@ float16_t dequantFuncQ5_K(const in decodeBufQ5_K bl, const in uint blockCoords[2
return float16_t(ret);
}
f16vec4 dequantFuncQ5_K_v(const in decodeBufQ5_K bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
decodeBufQ5_K_packed32 bl32 = decodeBufQ5_K_packed32(bl);
decodeBufQ5_K_packed128 bl128 = decodeBufQ5_K_packed128(bl);
const uint idx = coordInBlock[1];
const uint is = idx >> 5;
#if defined(IS_MUL_MM2) && defined(DATA_A_Q5_K)
vec2 v = shAscales[is * shAscales_stride + (blockCoords[0] % BM)];
float d = v.x;
float m = v.y;
#else
uvec4 v = bl128.block.q5k[0];
const f16vec2 loadd = unpackFloat2x16(v.x);
uint32_t sc;
uint32_t mbyte;
uint32_t scale0 = v.y;
uint32_t scale4 = v.z;
uint32_t scale8 = v.w;
uint32_t sc_lo = scale0;
uint32_t mb_lo = scale4;
uint32_t sc_hi = (scale8 & 0x0F0F0F0F) | ((scale0 & 0xC0C0C0C0) >> 2);
uint32_t mb_hi = ((scale8 & 0xF0F0F0F0) >> 4) | ((scale4 & 0xC0C0C0C0) >> 2);
sc = is < 4 ? sc_lo : sc_hi;
mbyte = is < 4 ? mb_lo : mb_hi;
sc = sc >> (8 * (is & 3));
mbyte = mbyte >> (8 * (is & 3));
sc &= 0x3F;
mbyte &= 0x3F;
const float16_t d = loadd.x * float16_t(sc);
const float16_t m = loadd.y * float16_t(mbyte);
#endif
// sh is 0 or 4; mask 0x0F0F0F0F covers the four nibbles regardless (no inter-byte leakage).
const uint sh = (idx & 0x20u) >> 3u;
const uint qs_w = (idx >> 6) * 8u + ((idx & 0x1Eu) >> 2);
const uint qh_w = (idx & 0x1Eu) >> 2;
const uint ql4 = (uint32_t(bl32.block.qs[qs_w]) >> sh) & 0x0F0F0F0Fu;
// qh stores bit `is` per element across 4 consecutive bytes; one shift+mask handles all 4.
const uint qh4 = ((uint32_t(bl32.block.qh[qh_w]) >> is) & 0x01010101u) << 4u;
const u8vec4 qi = unpack8(ql4 | qh4);
return f16vec4(vec4(qi) * vec4(d) - vec4(m));
}
layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ6_K {
block_q6_K block;
};
@ -431,6 +717,35 @@ float16_t dequantFuncQ6_K(const in decodeBufQ6_K bl, const in uint blockCoords[2
return ret;
}
f16vec4 dequantFuncQ6_K_v(const in decodeBufQ6_K bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
decodeBufQ6_K_packed16 bl16 = decodeBufQ6_K_packed16(bl);
const uint idx = coordInBlock[1];
const uint b = (idx & 0x40) >> 6;
const uint qhshift = (idx & 0x60) >> 4; // 0,2,4,6
const uint is = idx >> 4;
const uint sh = b * 4; // 0 or 4
const float16_t dscale = bl.block.d * float16_t(bl.block.scales[is]);
const uint ql_i = ((idx & 0x80) >> 2) + ((idx & 0x3E) >> 1);
const uint qh_i = ((idx & 0x80) >> 3) + ((idx & 0x1E) >> 1);
// Two adjacent uint16 packed16 reads, combined into a uint32 in registers.
// After this: byte j of qlw / qhw holds the data for element idx+j.
const uint qlw = uint32_t(bl16.block.ql[ql_i ]) | (uint32_t(bl16.block.ql[ql_i + 1]) << 16);
const uint qhw = uint32_t(bl16.block.qh[qh_i ]) | (uint32_t(bl16.block.qh[qh_i + 1]) << 16);
// sh in {0,4} and qhshift in {0,2,4,6}: per-byte masks 0x0F / 0x03 keep only the
// wanted bits with no inter-byte leakage; place qh's 2 bits at nibble high position.
const uint ql4 = (qlw >> sh) & 0x0F0F0F0Fu;
const uint qh4 = ((qhw >> qhshift) & 0x03030303u) << 4u;
const ivec4 qi = ivec4(unpack8(ql4 | qh4));
return f16vec4((vec4(qi) - vec4(32.0f)) * vec4(float(dscale)));
}
#if defined(DATA_A_IQ1_S)
layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ1_S {
block_iq1_s block;
@ -453,6 +768,29 @@ float16_t dequantFuncIQ1_S(const in decodeBufIQ1_S bl, const in uint blockCoords
float16_t ret = float16_t(dl) * (float16_t(bitfieldExtract(int(grid), 2 * int(idx % 8), 2)) + float16_t(delta));
return ret;
}
f16vec4 dequantFuncIQ1_S_v(const in decodeBufIQ1_S bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
const float16_t d = bl.block.d;
const uint idx = coordInBlock[1];
const uint ib32 = idx >> 5;
const uint ib8 = idx >> 3;
const int i8b = int(idx & 4); // 0 or 4
const uint qh = bl.block.qh[ib32];
const uint qs = bl.block.qs[ib8];
const float dl = float(d) * float(2 * bitfieldExtract(qh, 12, 3) + 1);
const float delta = ((qh & 0x8000u) != 0u) ? -IQ1S_DELTA : IQ1S_DELTA;
const uint grid = iq1s_grid[qs | (bitfieldExtract(qh, 3 * int(ib8 & 3), 3) << 8)];
const ivec4 q = ivec4(
bitfieldExtract(int(grid), 2 * (i8b + 0), 2),
bitfieldExtract(int(grid), 2 * (i8b + 1), 2),
bitfieldExtract(int(grid), 2 * (i8b + 2), 2),
bitfieldExtract(int(grid), 2 * (i8b + 3), 2));
return f16vec4((vec4(q) + vec4(delta)) * dl);
}
#endif
#if defined(DATA_A_IQ1_M)
@ -485,6 +823,33 @@ float16_t dequantFuncIQ1_M(const in decodeBufIQ1_M bl, const in uint blockCoords
float16_t ret = d * float16_t(dl) * (float16_t(bitfieldExtract(int(grid), 2 * i8, 2)) + float16_t(delta));
return ret;
}
f16vec4 dequantFuncIQ1_M_v(const in decodeBufIQ1_M bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
decodeBufIQ1_M_packed64 bl64 = decodeBufIQ1_M_packed64(bl);
const uint idx = coordInBlock[1];
uvec2 scales = unpack32(bl64.block.scales);
const float16_t d = uint16BitsToHalf(uint16_t(((scales.x & 0xF000) >> 12) | ((scales.x & 0xF0000000) >> 24) | ((scales.y & 0xF000) >> 4) | ((scales.y & 0xF0000000) >> 16)));
const uint ib8 = idx >> 3;
const uint ib16 = idx >> 4;
const int i8b = int(idx & 4); // 0 or 4 -- i8 base for the V=4 group
const uint sc = bl.block.scales[ib8 / 8];
const uint qs = bl.block.qs[ib8];
const uint qh = bl.block.qh[ib16] >> (4 * (ib8 & 1));
const float dl = 2.0 * float(bitfieldExtract(sc, 3 * int(ib16 & 3), 3)) + 1.0;
const float delta = ((qh & 8u) != 0u) ? -IQ1S_DELTA : IQ1S_DELTA;
const uint grid = iq1s_grid[qs | ((qh & 7u) << 8)];
const ivec4 q = ivec4(
bitfieldExtract(int(grid), 2 * (i8b + 0), 2),
bitfieldExtract(int(grid), 2 * (i8b + 1), 2),
bitfieldExtract(int(grid), 2 * (i8b + 2), 2),
bitfieldExtract(int(grid), 2 * (i8b + 3), 2));
return f16vec4((vec4(q) + vec4(delta)) * (float(d) * dl));
}
#endif
#if defined(DATA_A_IQ2_XXS)
@ -520,6 +885,33 @@ float16_t dequantFuncIQ2_XXS(const in decodeBufIQ2_XXS bl, const in uint blockCo
vec2 ret = dscale * g * ((sign & (1 << (idx & 7))) != 0 ? -1.0hf : 1.0hf);
return float16_t(ret[idx & 1]);
}
f16vec4 dequantFuncIQ2_XXS_v(const in decodeBufIQ2_XXS bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
decodeBufIQ2_XXS_packed16 bl16 = decodeBufIQ2_XXS_packed16(bl);
const uint idx = coordInBlock[1];
const uint ib32 = idx >> 5;
const uint ib8 = (idx & 0x18) >> 3;
const uint iqs = 8 * ib32 + ib8;
const uint qs = bl.block.qs[iqs];
const uint signscale = pack32(u16vec2(bl16.block.qs[4*ib32+2], bl16.block.qs[4*ib32+3]));
const float dscale = float(bl.block.d) * 0.25 * (0.5 + float(signscale >> 28));
uint sign = bitfieldExtract(signscale, 7 * int(ib8), 7);
sign |= bitCount(sign) << 7;
const uint sb = sign >> (idx & 7u);
const uint g2 = iq2xxs_grid[qs][(idx & 4) >> 2];
const u8vec4 g = unpack8(g2);
return f16vec4(
dscale * float(g.x) * ((sb & 1u) != 0u ? -1.0 : 1.0),
dscale * float(g.y) * ((sb & 2u) != 0u ? -1.0 : 1.0),
dscale * float(g.z) * ((sb & 4u) != 0u ? -1.0 : 1.0),
dscale * float(g.w) * ((sb & 8u) != 0u ? -1.0 : 1.0));
}
#endif
#if defined(DATA_A_IQ2_XS)
@ -548,6 +940,31 @@ float16_t dequantFuncIQ2_XS(const in decodeBufIQ2_XS bl, const in uint blockCoor
vec2 ret = dscale * g * ((sign & (1 << (idx & 7))) != 0 ? -1.0hf : 1.0hf);
return float16_t(ret[idx & 1]);
}
f16vec4 dequantFuncIQ2_XS_v(const in decodeBufIQ2_XS bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
const uint idx = coordInBlock[1];
const uint is = idx >> 5;
const uint sshift = (idx & 0x10) >> 2;
const uint iqs = idx >> 3;
const uint16_t qs = bl.block.qs[iqs];
const float dscale = float(bl.block.d) * 0.25 * (0.5 + float((bl.block.scales[is] >> sshift) & 0xF));
uint sign = uint(qs >> 9);
sign |= bitCount(sign) << 7;
const uint sb = sign >> (idx & 7u);
const uint g2 = iq2xs_grid[qs & 0x1FF][(idx & 4) >> 2];
const u8vec4 g = unpack8(g2);
return f16vec4(
dscale * float(g.x) * ((sb & 1u) != 0u ? -1.0 : 1.0),
dscale * float(g.y) * ((sb & 2u) != 0u ? -1.0 : 1.0),
dscale * float(g.z) * ((sb & 4u) != 0u ? -1.0 : 1.0),
dscale * float(g.w) * ((sb & 8u) != 0u ? -1.0 : 1.0));
}
#endif
#if defined(DATA_A_IQ2_S)
@ -576,6 +993,32 @@ float16_t dequantFuncIQ2_S(const in decodeBufIQ2_S bl, const in uint blockCoords
const vec2 v = db * vec2(sign01) * vec2(unpack8(g2));
return float16_t(v[idx & 1]);
}
f16vec4 dequantFuncIQ2_S_v(const in decodeBufIQ2_S bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
const uint idx = coordInBlock[1];
const uint ib32 = idx >> 5;
const uint ib8 = idx >> 3;
const uint qhshift = 2 * (ib8 % 4);
const uint scale = (bl.block.scales[ib32] >> ((idx & 0x10) >> 2)) & 0xf;
const uint qs = bl.block.qs[ib8];
const uint qh = bl.block.qh[ib32];
const uint sb = uint(bl.block.qs[QUANT_K / 8 + ib8]) >> (idx & 0x6u);
const float d = float(bl.block.d);
const float db = d * 0.25 * (0.5 + scale);
const uint g2 = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(idx & 4) >> 2];
const u8vec4 g = unpack8(g2);
return f16vec4(
db * float(g.x) * ((sb & 1u) != 0u ? -1.0 : 1.0),
db * float(g.y) * ((sb & 2u) != 0u ? -1.0 : 1.0),
db * float(g.z) * ((sb & 4u) != 0u ? -1.0 : 1.0),
db * float(g.w) * ((sb & 8u) != 0u ? -1.0 : 1.0));
}
#endif
#if defined(DATA_A_IQ3_XXS)
@ -609,6 +1052,32 @@ float16_t dequantFuncIQ3_XXS(const in decodeBufIQ3_XXS bl, const in uint blockCo
const vec2 v = db * vec2(sign01) * vec2(unpack8(grid).xy);
return float16_t(v[idx & 1]);
}
f16vec4 dequantFuncIQ3_XXS_v(const in decodeBufIQ3_XXS bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
decodeBufIQ3_XXS_packed16 bl16 = decodeBufIQ3_XXS_packed16(bl);
const uint idx = coordInBlock[1];
const uint iqs = idx >> 2;
const uint is = QUANT_K / 4 + ((idx & 0xE0) >> 3);
const float d = float(bl.block.d);
const uint qs = bl.block.qs[iqs];
const uint signs = pack32(u16vec2(bl16.block.qs[is/2+0], bl16.block.qs[is/2+1]));
const float db = d * 0.5 * (0.5 + (signs >> 28));
const uint sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7);
const uint sb = (sign7 | (bitCount(sign7) << 7)) >> (idx & 0x6u);
const uint grid = iq3xxs_grid[qs];
const u8vec4 g = unpack8(grid);
return f16vec4(
db * float(g.x) * ((sb & 1u) != 0u ? -1.0 : 1.0),
db * float(g.y) * ((sb & 2u) != 0u ? -1.0 : 1.0),
db * float(g.z) * ((sb & 4u) != 0u ? -1.0 : 1.0),
db * float(g.w) * ((sb & 8u) != 0u ? -1.0 : 1.0));
}
#endif
#if defined(DATA_A_IQ3_S)
@ -635,6 +1104,30 @@ float16_t dequantFuncIQ3_S(const in decodeBufIQ3_S bl, const in uint blockCoords
return float16_t(v[idx & 1]);
}
f16vec4 dequantFuncIQ3_S_v(const in decodeBufIQ3_S bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
const uint idx = coordInBlock[1];
const uint iqs = idx >> 2;
const uint iqh = idx >> 5;
const float d = float(bl.block.d);
const uint qs = bl.block.qs[iqs];
const uint qh = bl.block.qh[iqh];
const uint sb = uint(bl.block.signs[iqs / 2]) >> (idx & 0x6u);
const uint scale = bl.block.scales[iqs / 16];
const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf));
const uint grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)];
const u8vec4 g = unpack8(grid);
return f16vec4(
db * float(g.x) * ((sb & 1u) != 0u ? -1.0 : 1.0),
db * float(g.y) * ((sb & 2u) != 0u ? -1.0 : 1.0),
db * float(g.z) * ((sb & 4u) != 0u ? -1.0 : 1.0),
db * float(g.w) * ((sb & 8u) != 0u ? -1.0 : 1.0));
}
#endif
#if defined(DATA_A_IQ4_XS)
@ -642,6 +1135,10 @@ layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ4
block_iq4_xs block;
};
layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufIQ4_XS_packed32 {
block_iq4_xs_packed32 block;
};
float16_t dequantFuncIQ4_XS(const in decodeBufIQ4_XS bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
const float16_t d = bl.block.d;
@ -657,6 +1154,30 @@ float16_t dequantFuncIQ4_XS(const in decodeBufIQ4_XS bl, const in uint blockCoor
float16_t ret = d * float16_t(int(sl | (sh << 4)) - 32) * float16_t(kvalues_iq4nl[q]);
return ret;
}
f16vec4 dequantFuncIQ4_XS_v(const in decodeBufIQ4_XS bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
decodeBufIQ4_XS_packed32 bl32 = decodeBufIQ4_XS_packed32(bl);
const float16_t d = bl.block.d;
const uint idx = coordInBlock[1];
const uint ib32 = idx >> 5; // 0..7
const uint sl = (bl32.block.scales_l >> (4 * ib32)) & 0xF;
const uint sh = (uint(bl32.block.scales_h) >> (2 * ib32)) & 0x3;
const uint qshift = (idx & 0x10) >> 2; // {0, 4}
const uint qs_w = 4 * ib32 + ((idx & 0xC) >> 2); // iqs / 4, in [0,32)
const float16_t dl = d * float16_t(int(sl | (sh << 4)) - 32);
const uint qsw = bl32.block.qs[qs_w];
const u8vec4 qv = unpack8((qsw >> qshift) & 0x0F0F0F0Fu);
const vec4 ret = vec4(
float(kvalues_iq4nl[qv.x]),
float(kvalues_iq4nl[qv.y]),
float(kvalues_iq4nl[qv.z]),
float(kvalues_iq4nl[qv.w])) * float(dl);
return f16vec4(ret);
}
#endif
#if defined(DATA_A_IQ4_NL)
@ -664,6 +1185,10 @@ layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ4
block_iq4_nl block;
};
layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ4_NL_packed16 {
block_iq4_nl_packed16 block;
};
float16_t dequantFuncIQ4_NL(const in decodeBufIQ4_NL bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
const float16_t d = bl.block.d;
@ -676,6 +1201,24 @@ float16_t dequantFuncIQ4_NL(const in decodeBufIQ4_NL bl, const in uint blockCoor
float16_t ret = float16_t(kvalues_iq4nl[qs]) * d;
return ret;
}
f16vec4 dequantFuncIQ4_NL_v(const in decodeBufIQ4_NL bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
decodeBufIQ4_NL_packed16 bl16 = decodeBufIQ4_NL_packed16(bl);
const float16_t d = bl.block.d;
const uint idx = coordInBlock[1];
const uint shift = (idx & 0x10) >> 2; // 0 or 4
const uint qs_i = (idx & 0xC) >> 1; // packed16 word index, in {0,2,4,6}
const uint qsw = uint32_t(bl16.block.qs[qs_i ])
| (uint32_t(bl16.block.qs[qs_i + 1u]) << 16);
// shift in {0,4}: per-byte mask 0x0F isolates the wanted nibble in each byte.
const u8vec4 q = unpack8((qsw >> shift) & 0x0F0F0F0Fu);
return f16vec4(
float(d) * float(kvalues_iq4nl[q.x]),
float(d) * float(kvalues_iq4nl[q.y]),
float(d) * float(kvalues_iq4nl[q.z]),
float(d) * float(kvalues_iq4nl[q.w]));
}
#endif
#if defined(DATA_A_MXFP4)
@ -695,6 +1238,26 @@ float16_t dequantFuncMXFP4(const in decodeBufMXFP4 bl, const in uint blockCoords
float16_t ret = float16_t(kvalues_mxfp4[qs] * d * 0.5);
return ret;
}
f16vec4 dequantFuncMXFP4_v(const in decodeBufMXFP4 bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
const float d = e8m0_to_fp32(bl.block.e);
const uint idx = coordInBlock[1];
const uint iqs = idx & 0xF;
const uint shift = (idx & 0x10) >> 2;
uvec4 qv = uvec4(
uint(bl.block.qs[iqs]),
uint(bl.block.qs[iqs + 1u]),
uint(bl.block.qs[iqs + 2u]),
uint(bl.block.qs[iqs + 3u]));
qv = (qv >> shift) & 0xFu;
const vec4 ret = vec4(
float(kvalues_mxfp4[qv.x]),
float(kvalues_mxfp4[qv.y]),
float(kvalues_mxfp4[qv.z]),
float(kvalues_mxfp4[qv.w])) * d * 0.5f;
return f16vec4(ret);
}
#endif
#if defined(DATA_A_NVFP4)
@ -702,6 +1265,10 @@ layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufNVF
block_nvfp4 block;
};
layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufNVFP4_packed32 {
block_nvfp4_packed32 block;
};
float16_t dequantFuncNVFP4(const in decodeBufNVFP4 bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
const uint idx = coordInBlock[1];
@ -713,56 +1280,97 @@ float16_t dequantFuncNVFP4(const in decodeBufNVFP4 bl, const in uint blockCoords
qs = (qs >> shift) & 0xF;
return float16_t(kvalues_mxfp4[qs] * d * 0.5);
}
f16vec4 dequantFuncNVFP4_v(const in decodeBufNVFP4 bl, const in uint blockCoords[2], const in uint coordInBlock[2])
{
decodeBufNVFP4_packed32 bl32 = decodeBufNVFP4_packed32(bl);
const uint idx = coordInBlock[1];
const uint sub = idx >> 4;
const uint qs_w = ((idx & 0x30) >> 3) + ((idx & 0x4u) >> 2); // iqs / 4, in [0,8)
const uint shift = (idx & 0x8) >> 1;
const float d = ue4m3_to_fp32(bl.block.d[sub]);
const uint qsw = uint32_t(bl32.block.qs[qs_w]);
const u8vec4 qv = unpack8((qsw >> shift) & 0x0F0F0F0Fu);
const vec4 ret = vec4(
float(kvalues_mxfp4[qv.x]),
float(kvalues_mxfp4[qv.y]),
float(kvalues_mxfp4[qv.z]),
float(kvalues_mxfp4[qv.w])) * d * 0.5f;
return f16vec4(ret);
}
#endif
#if defined(DATA_A_Q1_0)
#define dequantFuncA dequantFuncQ1_0
#define dequantFuncA_v dequantFuncQ1_0_v
#elif defined(DATA_A_Q4_0)
#define dequantFuncA dequantFuncQ4_0
#define dequantFuncA_v dequantFuncQ4_0_v
#elif defined(DATA_A_Q4_1)
#define dequantFuncA dequantFuncQ4_1
#define dequantFuncA_v dequantFuncQ4_1_v
#elif defined(DATA_A_Q5_0)
#define dequantFuncA dequantFuncQ5_0
#define dequantFuncA_v dequantFuncQ5_0_v
#elif defined(DATA_A_Q5_1)
#define dequantFuncA dequantFuncQ5_1
#define dequantFuncA_v dequantFuncQ5_1_v
#elif defined(DATA_A_Q8_0)
#define dequantFuncA dequantFuncQ8_0
#define dequantFuncA_v dequantFuncQ8_0_v
#elif defined(DATA_A_Q2_K)
#define dequantFuncA dequantFuncQ2_K
#define dequantFuncA_v dequantFuncQ2_K_v
#elif defined(DATA_A_Q3_K)
#define dequantFuncA dequantFuncQ3_K
#define dequantFuncA_v dequantFuncQ3_K_v
#elif defined(DATA_A_Q4_K)
#define dequantFuncA dequantFuncQ4_K
#define dequantFuncA_v dequantFuncQ4_K_v
#define fetch_scales fetch_scalesQ4_K
#define store_scales store_scalesQ4_K
#elif defined(DATA_A_Q5_K)
#define dequantFuncA dequantFuncQ5_K
#define dequantFuncA_v dequantFuncQ5_K_v
#define fetch_scales fetch_scalesQ5_K
#define store_scales store_scalesQ4_K
#elif defined(DATA_A_Q6_K)
#define dequantFuncA dequantFuncQ6_K
#define dequantFuncA_v dequantFuncQ6_K_v
#elif defined(DATA_A_IQ1_S)
#define dequantFuncA dequantFuncIQ1_S
#define dequantFuncA_v dequantFuncIQ1_S_v
#elif defined(DATA_A_IQ1_M)
#define dequantFuncA dequantFuncIQ1_M
#define dequantFuncA_v dequantFuncIQ1_M_v
#elif defined(DATA_A_IQ2_XXS)
#define dequantFuncA dequantFuncIQ2_XXS
#define dequantFuncA_v dequantFuncIQ2_XXS_v
#elif defined(DATA_A_IQ2_XS)
#define dequantFuncA dequantFuncIQ2_XS
#define dequantFuncA_v dequantFuncIQ2_XS_v
#elif defined(DATA_A_IQ2_S)
#define dequantFuncA dequantFuncIQ2_S
#define dequantFuncA_v dequantFuncIQ2_S_v
#elif defined(DATA_A_IQ3_XXS)
#define dequantFuncA dequantFuncIQ3_XXS
#define dequantFuncA_v dequantFuncIQ3_XXS_v
#elif defined(DATA_A_IQ3_S)
#define dequantFuncA dequantFuncIQ3_S
#define dequantFuncA_v dequantFuncIQ3_S_v
#elif defined(DATA_A_IQ4_XS)
#define dequantFuncA dequantFuncIQ4_XS
#define dequantFuncA_v dequantFuncIQ4_XS_v
#elif defined(DATA_A_IQ4_NL)
#define dequantFuncA dequantFuncIQ4_NL
#define dequantFuncA_v dequantFuncIQ4_NL_v
#elif defined(DATA_A_MXFP4)
#define dequantFuncA dequantFuncMXFP4
#define dequantFuncA_v dequantFuncMXFP4_v
#elif defined(DATA_A_NVFP4)
#define dequantFuncA dequantFuncNVFP4
#define dequantFuncA_v dequantFuncNVFP4_v
#elif defined(DATA_A_F32)
#define dequantFuncA dequantFuncF32
#endif

7
ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp Normal file
View file

@ -0,0 +1,7 @@
#version 460
#extension GL_NV_cooperative_matrix_decode_vector : require
void main()
{
}

42
ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp
View file

@ -11,6 +11,9 @@
#extension GL_KHR_memory_scope_semantics : enable
#extension GL_KHR_cooperative_matrix : enable
#extension GL_NV_cooperative_matrix2 : enable
#ifdef GL_NV_cooperative_matrix_decode_vector
#extension GL_NV_cooperative_matrix_decode_vector : enable
#endif
#extension GL_EXT_buffer_reference : enable
#extension GL_KHR_shader_subgroup_ballot : enable
#extension GL_KHR_shader_subgroup_vote : enable
@ -54,6 +57,41 @@ float16_t faDecodeV(const decodeBufFA_V bl_in, const uint blockCoords[2], const
}
}
// V=4 vector decode for K/V; dispatches to per-format _v decoders.
f16vec4 faDecodeKVector(const decodeBufFA_K bl_in, const uint blockCoords[2], const uint coordInBlock[2]) {
switch (FaTypeK) {
case 0u: return f16vec4(decodeBufF32(bl_in).block);
case 2u: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock);
case 3u: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock);
case 6u: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock);
case 7u: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock);
case 8u: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock);
case 41u: return dequantFuncQ1_0_v(decodeBufQ1_0(bl_in), blockCoords, coordInBlock);
default: return f16vec4(0);
}
}
f16vec4 faDecodeVVector(const decodeBufFA_V bl_in, const uint blockCoords[2], const uint coordInBlock[2]) {
switch (FaTypeV) {
case 0u: return f16vec4(decodeBufF32(bl_in).block);
case 2u: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock);
case 3u: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock);
case 6u: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock);
case 7u: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock);
case 8u: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock);
case 41u: return dequantFuncQ1_0_v(decodeBufQ1_0(bl_in), blockCoords, coordInBlock);
default: return f16vec4(0);
}
}
#ifdef GL_NV_cooperative_matrix_decode_vector
#define FADECODEK , faDecodeK, faDecodeKVector
#define FADECODEV , faDecodeV, faDecodeVVector
#else
#define FADECODEK , faDecodeK
#define FADECODEV , faDecodeV
#endif
layout (binding = 0) readonly buffer Q {uint8_t data_q[];};
layout (binding = 1) readonly buffer K {uint8_t data_k[];};
layout (binding = 2) readonly buffer V {uint8_t data_v[];};
@ -259,7 +297,7 @@ void main() {
// F16: bs_k==1 (direct load). F32: bs_k==4 (vec4 / dequantFuncF32). Q4/Q8 family: bs_k==32. Q1_0: bs_k==128.
const bool k_use_decode = (bs_k > 1u);
if (k_use_decode) {
coopMatLoadTensorNV(K_T, data_k, k_offset, sliceTensorLayoutNV(tensorLayoutK, j * Bc, Bc, 0, HSK_pad), tensorViewTranspose, faDecodeK);
coopMatLoadTensorNV(K_T, data_k, k_offset, sliceTensorLayoutNV(tensorLayoutK, j * Bc, Bc, 0, HSK_pad), tensorViewTranspose FADECODEK);
} else {
coopMatLoadTensorNV(K_T, data_k, k_offset, sliceTensorLayoutNV(tensorLayoutK, j * Bc, Bc, 0, HSK_pad), tensorViewTranspose);
}
@ -325,7 +363,7 @@ void main() {
uint32_t v_offset = iv2*p.nb22 + iv3*p.nb23;
const bool v_use_decode = (bs_v > 1u);
if (v_use_decode) {
coopMatLoadTensorNV(V, data_v, v_offset, sliceTensorLayoutNV(tensorLayoutV, j * Bc, Bc, 0, HSV_pad), faDecodeV);
coopMatLoadTensorNV(V, data_v, v_offset, sliceTensorLayoutNV(tensorLayoutV, j * Bc, Bc, 0, HSV_pad) FADECODEV);
} else {
coopMatLoadTensorNV(V, data_v, v_offset, sliceTensorLayoutNV(tensorLayoutV, j * Bc, Bc, 0, HSV_pad));
}

147
ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp
View file

@ -10,12 +10,38 @@ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
#if !defined(DATA_A_F32) && !defined(DATA_A_F16) && !defined(DATA_A_BF16)
#define K_PER_ITER 8
#else
#define K_PER_ITER 2
#define K_PER_ITER 4
#endif
uint a_offset, b_offset, d_offset, y_offset;
vec4 load_b(const uint j, const uint iybs, const uint iqs, const bool lastiter, out bool OOB_y, out bool OOB_z, out bool OOB_w) {
// Check if the latter elements are OOB, and don't fetch B or accumulate it.
OOB_y = lastiter && (iybs + iqs + y_offset >= p.ncols);
OOB_z = lastiter && (iybs + iqs + y_offset*2 >= p.ncols);
OOB_w = lastiter && (iybs + iqs + y_offset*3 >= p.ncols);
if (!OOB_w) {
return vec4(FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs]),
FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs + y_offset]),
FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs + y_offset*2]),
FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs + y_offset*3]));
} else if (!OOB_z) {
return vec4(FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs]),
FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs + y_offset]),
FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs + y_offset*2]),
0);
} else if (!OOB_y) {
return vec4(FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs]),
FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs + y_offset]),
0, 0);
} else {
return vec4(FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs]),
0, 0, 0);
}
}
void iter(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint first_row, const uint num_rows, const uint tid, const uint i, bool lastiter)
{
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
@ -25,6 +51,8 @@ void iter(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint first_row, const
#if K_PER_ITER == 8
#if QUANT_R == 2
// Note that we end up fetching bogus elements here, but its fine as they'll be
// within an accessible block.
const vec4 bv02 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + iybs + iqs) / 4]);
const vec4 bv13 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + iybs + iqs + y_offset) / 4]);
const vec4 bv0 = vec4(bv02.x, bv13.x, bv02.y, bv13.y);
@ -34,18 +62,11 @@ void iter(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint first_row, const
const vec4 bv1 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + iybs + iqs) / 4 + 1]);
#endif
#else
// Check if the second of the pair of elements is OOB, and don't fetch B or
// accumulate it. We still fetch a pair of elements for A, which is fine for
// quantized formats since they'll be within the same block. We should
// probably skip fetching the second element for F16/F32, but as of now we
// still do.
const bool OOB = lastiter && (iybs + iqs + y_offset >= p.ncols);
bool OOB_y;
bool OOB_z;
bool OOB_w;
FLOAT_TYPE b0 = 0, b1 = 0;
b0 = FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs]);
if (!OOB) {
b1 = FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs + y_offset]);
}
const vec4 b = load_b(j, iybs, iqs, lastiter, OOB_y, OOB_z, OOB_w);
#endif
uint ibi = first_row*p.ncols;
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
@ -71,22 +92,60 @@ void iter(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint first_row, const
temp[j][n] += rowtmp;
#else
const vec2 v = dequantize(ib, iqs, a_offset);
// matrix multiplication
temp[j][n] = fma(FLOAT_TYPE(v.x), b0, temp[j][n]);
if (!OOB) {
temp[j][n] = fma(FLOAT_TYPE(v.y), b1, temp[j][n]);
if (!OOB_w) {
const vec4 v = dequantize4(ib, iqs, a_offset);
temp[j][n] += dot(v, b);
} else if (!OOB_z) {
const vec2 v0 = dequantize(ib, iqs, a_offset);
const FLOAT_TYPE v1 = dequantize1(ib + 2/QUANT_R, iqs, a_offset);
const vec3 v = vec3(v0.x, v0.y, v1);
const vec3 b0 = vec3(b.x, b.y, b.z);
temp[j][n] += dot(v, b0);
} else if (!OOB_y) {
const vec2 v0 = dequantize(ib, iqs, a_offset);
const vec2 b0 = vec2(b.x, b.y);
temp[j][n] += dot(v0, b0);
} else {
const FLOAT_TYPE v = dequantize1(ib, iqs, a_offset);
temp[j][n] = fma(v, b.x, temp[j][n]);
}
#endif
}
}
}
#if defined(DATA_A_F32) || defined(DATA_A_F16) || defined(DATA_A_BF16)
void iter_aligned_nonquant(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint first_row, const uint num_rows, const uint tid, const uint i)
{
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
const uint col = i*BLOCK_SIZE + K_PER_ITER*tid;
const uint iqs = 0; // quant index
const uint iybs = col; // y block start index
const vec4 b = data_b_v4[(j*p.batch_stride_b + b_offset + iybs + iqs) / 4];
uint ibi = first_row*p.ncols;
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
const uint ib = (ibi + col)/QUANT_K; // block index
ibi += p.ncols;
const vec4 v = dequantize4_2aligned(ib, iqs, a_offset);
// matrix multiplication
temp[j][n] += dot(v, b);
}
}
}
#endif
void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
const uint tid = gl_LocalInvocationID.x;
get_offsets(a_offset, b_offset, d_offset);
const bool is_aligned_nonquant =
p.batch_stride_b % 4 == 0 && b_offset % 4 == 0 &&
p.ncols % 4 == 0 && BLOCK_SIZE % 4 == 0 &&
K_PER_ITER == 4;
y_offset = QUANT_R == 1 ? 1 : QUANT_K/2;
@ -105,17 +164,26 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
int unroll_count = 4;
uint unrolled_iters = num_iters & ~(unroll_count - 1);
#if K_PER_ITER == 2
uint i = 0;
#if K_PER_ITER == 4
// If the K dimension is odd, we need lastiter==true on the last iteration
// so OOB is computed correctly. Skip some unrolling to make that happen.
if ((p.ncols & 1) != 0 &&
if ((p.ncols & 3) != 0 &&
unrolled_iters == num_iters &&
unrolled_iters > 0) {
unrolled_iters -= unroll_count;
}
if (is_aligned_nonquant) {
while (i < unrolled_iters) {
// Manually partially unroll the loop
[[unroll]] for (uint k = 0; k < unroll_count; ++k) {
iter_aligned_nonquant(temp, first_row, num_rows, tid, i*K_PER_ITER);
i++;
}
}
} else {
#endif
uint i = 0;
while (i < unrolled_iters) {
// Manually partially unroll the loop
[[unroll]] for (uint k = 0; k < unroll_count; ++k) {
@ -123,18 +191,30 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
i++;
}
}
#if K_PER_ITER == 4
}
#endif
unroll_count = 2;
unrolled_iters = num_iters & ~(unroll_count - 1);
#if K_PER_ITER == 2
if ((p.ncols & 1) != 0 &&
#if K_PER_ITER == 4
if ((p.ncols & 3) != 0 &&
unrolled_iters == num_iters &&
unrolled_iters > 0) {
unrolled_iters -= unroll_count;
}
#endif
if (is_aligned_nonquant) {
while (i < unrolled_iters && is_aligned_nonquant) {
// Manually partially unroll the loop
[[unroll]] for (uint k = 0; k < unroll_count; ++k) {
iter_aligned_nonquant(temp, first_row, num_rows, tid, i*K_PER_ITER);
i++;
}
}
} else {
#endif
while (i < unrolled_iters) {
// Manually partially unroll the loop
[[unroll]] for (uint k = 0; k < unroll_count; ++k) {
@ -142,10 +222,25 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
i++;
}
}
#if K_PER_ITER == 4
}
#endif
#if K_PER_ITER == 4
if (is_aligned_nonquant) {
while (i < num_iters) {
iter_aligned_nonquant(temp, first_row, num_rows, tid, i*K_PER_ITER);
i++;
}
} else {
#endif
while (i < num_iters) {
iter(temp, first_row, num_rows, tid, i*K_PER_ITER, true);
i++;
}
#if K_PER_ITER == 4
}
#endif
reduce_result(temp, d_offset, first_row, num_rows, tid);
}
@ -164,6 +259,6 @@ void main() {
if (first_row >= p.stride_d) {
return;
}
compute_outputs(first_row, p.stride_d - first_row);
compute_outputs(first_row, min(NUM_ROWS, p.stride_d - first_row));
}
}

8
ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp
View file

@ -71,10 +71,12 @@ layout (binding = 1) readonly buffer B {B_TYPE data_b[];};
layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};
#if QUANT_K > 1
#define DECODEFUNCA , dequantFuncA
#include "dequant_funcs_cm2.glsl"
#if defined(dequantFuncA_v) && defined(GL_NV_cooperative_matrix_decode_vector)
#define DECODEFUNCA , dequantFuncA, dequantFuncA_v
#else
#define DECODEFUNCA , dequantFuncA
#endif
#else
#define DECODEFUNCA
#endif

9
ggml/src/ggml-vulkan/vulkan-shaders/types.glsl
View file

@ -31,6 +31,7 @@
#else
#define A_TYPE float16_t
#endif
#define A_TYPE_PACKED32 f16vec2
#endif
#if defined(DATA_A_BF16)
@ -44,6 +45,7 @@
#else
#define A_TYPE uint16_t
#endif
#define A_TYPE_PACKED32 uint32_t
#endif
#define QUANT_K_Q4_0 32
@ -1722,11 +1724,18 @@ struct block_nvfp4
uint8_t qs[QUANT_K_NVFP4 / 2];
};
struct block_nvfp4_packed32
{
uint32_t d[QUANT_K_NVFP4 / 16 / 4];
uint32_t qs[QUANT_K_NVFP4 / 2 / 4];
};
#if defined(DATA_A_NVFP4)
#define QUANT_K QUANT_K_NVFP4
#define QUANT_R QUANT_R_NVFP4
#define QUANT_AUXF 1
#define A_TYPE block_nvfp4
#define A_TYPE_PACKED32 block_nvfp4_packed32
#endif
#if defined(DATA_A_IQ4_NL) || defined(DATA_A_IQ4_XS)

16
ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
View file

@ -815,9 +815,11 @@ void process_shaders() {
string_to_spv("div_f32", "div.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
string_to_spv("repeat_f32", "repeat.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("repeat_i32", "repeat.comp", {{"A_TYPE", "int32_t"}, {"D_TYPE", "int32_t"}});
string_to_spv("repeat_back_f32", "repeat_back.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("repeat_i16", "repeat.comp", {{"A_TYPE", "int16_t"}, {"D_TYPE", "int16_t"}});
string_to_spv("scale_f32", "scale.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
string_to_spv("sqr_f32", "square.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
@ -1001,8 +1003,16 @@ void process_shaders() {
string_to_spv(name + (unroll ? "_unroll" : ""), "conv2d_mm.comp", defines);
#if defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT)
if (unroll) {
defines["COOPMAT2"] = "1";
string_to_spv(name, "conv2d_mm.comp", defines, true, false, true);
auto cm2_defines = defines;
cm2_defines["COOPMAT2"] = "1";
string_to_spv(name, "conv2d_mm.comp", cm2_defines, true, false, true);
}
#endif
#if defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT)
if (unroll) {
auto cm1_defines = defines;
cm1_defines["COOPMAT"] = "1";
string_to_spv(name, "conv2d_mm.comp", cm1_defines, true, true, false);
}
#endif
}

2
gguf-py/pyproject.toml
View file

@ -37,7 +37,7 @@ packages = [
[tool.poetry.dependencies]
python = ">=3.10"
[tool.poetry.dev-dependencies]
[tool.poetry.group.dev.dependencies]
pytest = "^5.2"
[build-system]

11
src/llama-vocab.cpp
View file

@ -736,6 +736,14 @@ struct llm_tokenizer_bpe : llm_tokenizer {
};
byte_encode = false;
break;
case LLAMA_VOCAB_PRE_TYPE_MINICPM5:
regex_exprs = {
// original regex from tokenizer.json (openbmb/MiniCPM5-1B)
"\\p{N}{1,3}",
// "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}+| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}+| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
};
break;
default:
// default regex for BPE tokenization pre-processing
regex_exprs = {
@ -2275,6 +2283,9 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
pre_type = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
} else if (tokenizer_pre == "default") {
pre_type = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
} else if (tokenizer_pre == "minicpm5") {
pre_type = LLAMA_VOCAB_PRE_TYPE_MINICPM5;
ignore_merges = true;
} else if (
tokenizer_pre == "llama3" ||
tokenizer_pre == "llama-v3" ||

1
src/llama-vocab.h
View file

@ -61,6 +61,7 @@ enum llama_vocab_pre_type {
LLAMA_VOCAB_PRE_TYPE_JAIS2 = 49,
LLAMA_VOCAB_PRE_TYPE_GEMMA4 = 50,
LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE = 51,
LLAMA_VOCAB_PRE_TYPE_MINICPM5 = 52,
};
struct LLM_KV;

5
tools/server/server-http.cpp
View file

@ -99,6 +99,7 @@ bool server_http_context::init(const common_params & params) {
srv.reset(
new httplib::SSLServer(params.ssl_file_cert.c_str(), params.ssl_file_key.c_str())
);
is_ssl = true;
} else {
SRV_INF("%s", "running without SSL\n");
srv.reset(new httplib::Server());
@ -378,8 +379,8 @@ bool server_http_context::start() {
thread = std::thread([this]() { pimpl->srv->listen_after_bind(); });
srv->wait_until_ready();
listening_address = is_sock ? string_format("unix://%s", hostname.c_str())
: string_format("http://%s:%d", hostname.c_str(), port);
listening_address = is_sock ? string_format("unix://%s", hostname.c_str())
: string_format("%s://%s:%d", is_ssl ? "https" : "http", hostname.c_str(), port);
return true;
}

1
tools/server/server-http.h
View file

@ -74,6 +74,7 @@ struct server_http_context {
std::string path_prefix;
std::string hostname;
int port;
bool is_ssl = false;
server_http_context();
~server_http_context();

292
vendor/cpp-httplib/httplib.cpp vendored
View file

@ -6650,6 +6650,176 @@ make_host_and_port_string_always_port(const std::string &host, int port) {
return prepare_host_string(host) + ":" + std::to_string(port);
}
bool parse_no_proxy_entry(const std::string &token, NoProxyEntry &out);
NormalizedTarget normalize_target(const std::string &host);
bool ip_in_cidr(const IPBytes &ip, const IPBytes &net, int prefix_bits);
bool host_matches_no_proxy(const NormalizedTarget &target,
const std::vector<NoProxyEntry> &entries);
bool ip_in_cidr(const IPBytes &ip, const IPBytes &net, int prefix_bits) {
if (prefix_bits < 0 || prefix_bits > 128) { return false; }
if (prefix_bits == 0) { return true; }
int full_bytes = prefix_bits / 8;
int rem_bits = prefix_bits % 8;
if (full_bytes > 0 && std::memcmp(ip.data(), net.data(),
static_cast<size_t>(full_bytes)) != 0) {
return false;
}
if (rem_bits == 0) { return true; }
auto i = static_cast<size_t>(full_bytes);
auto mask = static_cast<uint8_t>(0xFFu << (8 - rem_bits));
return (ip[i] & mask) == (net[i] & mask);
}
bool parse_no_proxy_entry(const std::string &token, NoProxyEntry &out) {
if (token.empty()) { return false; }
if (token == "*") {
out.kind = NoProxyKind::Wildcard;
return true;
}
auto slash = token.find('/');
std::string addr_part =
(slash == std::string::npos) ? token : token.substr(0, slash);
std::string prefix_part =
(slash == std::string::npos) ? std::string() : token.substr(slash + 1);
// A bare slash or trailing-slash CIDR like "10.0.0.0/" is malformed;
// don't silently treat it as a /32 (or /128).
if (slash != std::string::npos && prefix_part.empty()) { return false; }
// Accept the bracketed IPv6 form ("[::1]", "[fe80::]/10") as well as the
// bare form. Brackets have no meaning for IPv4, so skip the IPv4 attempt
// when brackets are present.
bool bracketed = addr_part.size() >= 2 && addr_part.front() == '[' &&
addr_part.back() == ']';
if (bracketed) { addr_part = addr_part.substr(1, addr_part.size() - 2); }
if (!bracketed) {
struct in_addr v4;
if (inet_pton(AF_INET, addr_part.c_str(), &v4) == 1) {
int prefix = 32;
if (!prefix_part.empty()) {
auto r = from_chars(prefix_part.data(),
prefix_part.data() + prefix_part.size(), prefix);
if (r.ec != std::errc{} ||
r.ptr != prefix_part.data() + prefix_part.size()) {
return false;
}
if (prefix < 0 || prefix > 32) { return false; }
}
out.kind = NoProxyKind::IPv4Cidr;
std::memcpy(out.net.data(), &v4, sizeof(v4));
out.prefix_bits = prefix;
return true;
}
}
struct in6_addr v6;
if (inet_pton(AF_INET6, addr_part.c_str(), &v6) == 1) {
int prefix = 128;
if (!prefix_part.empty()) {
auto r = from_chars(prefix_part.data(),
prefix_part.data() + prefix_part.size(), prefix);
if (r.ec != std::errc{} ||
r.ptr != prefix_part.data() + prefix_part.size()) {
return false;
}
if (prefix < 0 || prefix > 128) { return false; }
}
out.kind = NoProxyKind::IPv6Cidr;
std::memcpy(out.net.data(), &v6, sizeof(v6));
out.prefix_bits = prefix;
return true;
}
// Bracketed entries can only be IPv6. If the IPv6 parse above failed,
// the entry is malformed — don't fall through to the hostname branch.
if (bracketed) { return false; }
// A '/' on a non-IP token means a CIDR prefix without an address. Reject.
if (slash != std::string::npos) { return false; }
// Port-specific entries (host:port) are not supported.
if (token.find(':') != std::string::npos) { return false; }
std::string hostname = case_ignore::to_lower(token);
while (!hostname.empty() && hostname.front() == '.') {
hostname.erase(hostname.begin());
}
while (!hostname.empty() && hostname.back() == '.') {
hostname.pop_back();
}
if (hostname.empty()) { return false; }
out.kind = NoProxyKind::HostnameSuffix;
out.hostname_pattern = std::move(hostname);
return true;
}
NormalizedTarget normalize_target(const std::string &host) {
NormalizedTarget t;
std::string h = host;
if (h.size() >= 2 && h.front() == '[' && h.back() == ']') {
h = h.substr(1, h.size() - 2);
}
// Strip a single trailing dot so "example.com." canonicalizes to
// "example.com".
if (!h.empty() && h.back() == '.') { h.pop_back(); }
t.hostname = case_ignore::to_lower(h);
if (!t.hostname.empty()) {
struct in_addr v4;
struct in6_addr v6;
if (inet_pton(AF_INET, t.hostname.c_str(), &v4) == 1) {
t.is_ipv4 = true;
std::memcpy(t.ip.data(), &v4, sizeof(v4));
} else if (inet_pton(AF_INET6, t.hostname.c_str(), &v6) == 1) {
t.is_ipv6 = true;
std::memcpy(t.ip.data(), &v6, sizeof(v6));
}
}
return t;
}
bool host_matches_no_proxy(const NormalizedTarget &target,
const std::vector<NoProxyEntry> &entries) {
if (target.hostname.empty()) { return false; }
for (const auto &e : entries) {
switch (e.kind) {
case NoProxyKind::Wildcard: return true;
case NoProxyKind::IPv4Cidr:
if (target.is_ipv4 && ip_in_cidr(target.ip, e.net, e.prefix_bits)) {
return true;
}
break;
case NoProxyKind::IPv6Cidr:
if (target.is_ipv6 && ip_in_cidr(target.ip, e.net, e.prefix_bits)) {
return true;
}
break;
case NoProxyKind::HostnameSuffix:
if (target.is_ipv4 || target.is_ipv6) { break; }
if (target.hostname == e.hostname_pattern) { return true; }
// Dot-boundary suffix match: prevents "evilexample.com" from matching
// an entry of "example.com".
if (target.hostname.size() > e.hostname_pattern.size() + 1) {
auto offset = target.hostname.size() - e.hostname_pattern.size();
if (target.hostname[offset - 1] == '.' &&
target.hostname.compare(offset, e.hostname_pattern.size(),
e.hostname_pattern) == 0) {
return true;
}
}
break;
}
}
return false;
}
template <typename T>
bool check_and_write_headers(Stream &strm, Headers &headers,
T header_writer, Error &error) {
@ -8455,6 +8625,7 @@ void ClientImpl::copy_settings(const ClientImpl &rhs) {
proxy_basic_auth_username_ = rhs.proxy_basic_auth_username_;
proxy_basic_auth_password_ = rhs.proxy_basic_auth_password_;
proxy_bearer_token_auth_token_ = rhs.proxy_bearer_token_auth_token_;
no_proxy_entries_ = rhs.no_proxy_entries_;
logger_ = rhs.logger_;
error_logger_ = rhs.error_logger_;
@ -8470,8 +8641,25 @@ void ClientImpl::copy_settings(const ClientImpl &rhs) {
#endif
}
bool
ClientImpl::is_proxy_enabled_for_host(const std::string &host) const {
if (proxy_host_.empty() || proxy_port_ == -1) { return false; }
if (no_proxy_entries_.empty()) { return true; }
// host_ is const so its normalized form is invariant; cache it. The
// cross-host path (setup_redirect_client passing next_host) re-normalizes.
if (host == host_) {
if (!host_normalized_valid_) {
host_normalized_ = detail::normalize_target(host_);
host_normalized_valid_ = true;
}
return !detail::host_matches_no_proxy(host_normalized_, no_proxy_entries_);
}
auto target = detail::normalize_target(host);
return !detail::host_matches_no_proxy(target, no_proxy_entries_);
}
socket_t ClientImpl::create_client_socket(Error &error) const {
if (!proxy_host_.empty() && proxy_port_ != -1) {
if (is_proxy_enabled_for_host(host_)) {
return detail::create_client_socket(
proxy_host_, std::string(), proxy_port_, address_family_, tcp_nodelay_,
ipv6_v6only_, socket_options_, connection_timeout_sec_,
@ -8543,6 +8731,12 @@ void ClientImpl::close_socket(Socket &socket) {
socket.sock = INVALID_SOCKET;
}
void ClientImpl::disconnect(bool gracefully) {
shutdown_ssl(socket_, gracefully);
shutdown_socket(socket_);
close_socket(socket_);
}
bool ClientImpl::read_response_line(Stream &strm, const Request &req,
Response &res,
bool skip_100_continue) const {
@ -8614,14 +8808,8 @@ bool ClientImpl::send_(Request &req, Response &res, Error &error) {
#endif
if (!is_alive) {
// Attempt to avoid sigpipe by shutting down non-gracefully if it
// seems like the other side has already closed the connection Also,
// there cannot be any requests in flight from other threads since we
// locked request_mutex_, so safe to close everything immediately
const bool shutdown_gracefully = false;
shutdown_ssl(socket_, shutdown_gracefully);
shutdown_socket(socket_);
close_socket(socket_);
// Peer seems gone — non-graceful shutdown to avoid SIGPIPE.
disconnect(/*gracefully=*/false);
}
}
@ -8671,9 +8859,7 @@ bool ClientImpl::send_(Request &req, Response &res, Error &error) {
if (socket_should_be_closed_when_request_is_done_ || close_connection ||
!ret) {
shutdown_ssl(socket_, true);
shutdown_socket(socket_);
close_socket(socket_);
disconnect(/*gracefully=*/true);
}
});
@ -8786,11 +8972,7 @@ ClientImpl::open_stream(const std::string &method, const std::string &path,
}
}
#endif
if (!is_alive) {
shutdown_ssl(socket_, false);
shutdown_socket(socket_);
close_socket(socket_);
}
if (!is_alive) { disconnect(/*gracefully=*/false); }
}
if (!is_alive) {
@ -9082,7 +9264,7 @@ bool ClientImpl::handle_request(Stream &strm, Request &req,
bool ret;
if (!is_ssl() && !proxy_host_.empty() && proxy_port_ != -1) {
if (!is_ssl() && is_proxy_enabled_for_host(host_)) {
auto req2 = req;
req2.path = "http://" +
detail::make_host_and_port_string(host_, port_, false) +
@ -9106,9 +9288,7 @@ bool ClientImpl::handle_request(Stream &strm, Request &req,
// to call it from a different thread since it's a thread-safety issue
// to do these things to the socket if another thread is using the socket.
std::lock_guard<std::mutex> guard(socket_mutex_);
shutdown_ssl(socket_, true);
shutdown_socket(socket_);
close_socket(socket_);
disconnect(/*gracefully=*/true);
}
if (300 < res.status && res.status < 400 && follow_location_) {
@ -9121,6 +9301,14 @@ bool ClientImpl::handle_request(Stream &strm, Request &req,
res.status == StatusCode::ProxyAuthenticationRequired_407) &&
req.authorization_count_ < 5) {
auto is_proxy = res.status == StatusCode::ProxyAuthenticationRequired_407;
// Only retry when the 407 actually came from a proxy hop: plain HTTP
// through an enabled proxy. HTTPS via CONNECT tunnels the 407 from the
// origin (#2457); direct/bypassed origins have no proxy hop at all.
if (is_proxy && !(!is_ssl() && is_proxy_enabled_for_host(host_))) {
return ret;
}
const auto &username =
is_proxy ? proxy_digest_auth_username_ : digest_auth_username_;
const auto &password =
@ -9288,13 +9476,13 @@ void ClientImpl::setup_redirect_client(ClientType &client) {
// host. This function is only called for cross-host redirects; same-host
// redirects are handled directly in ClientImpl::redirect().
// Setup proxy configuration (CRITICAL ORDER - proxy must be set
// before proxy auth)
// Copy the proxy configuration unconditionally; the per-target bypass is
// re-evaluated at send time, so a later hop to a non-bypassed host can
// still use the proxy.
client.no_proxy_entries_ = no_proxy_entries_;
if (!proxy_host_.empty() && proxy_port_ != -1) {
// First set proxy host and port
client.set_proxy(proxy_host_, proxy_port_);
// Then set proxy authentication (order matters!)
if (!proxy_basic_auth_username_.empty()) {
client.set_proxy_basic_auth(proxy_basic_auth_username_,
proxy_basic_auth_password_);
@ -9385,14 +9573,6 @@ bool ClientImpl::write_request(Stream &strm, Request &req,
}
}
if (!proxy_basic_auth_username_.empty() &&
!proxy_basic_auth_password_.empty()) {
if (!req.has_header("Proxy-Authorization")) {
req.headers.insert(make_basic_authentication_header(
proxy_basic_auth_username_, proxy_basic_auth_password_, true));
}
}
if (!bearer_token_auth_token_.empty()) {
if (!req.has_header("Authorization")) {
req.headers.insert(make_bearer_token_authentication_header(
@ -9400,8 +9580,18 @@ bool ClientImpl::write_request(Stream &strm, Request &req,
}
}
if (!proxy_bearer_token_auth_token_.empty()) {
if (!req.has_header("Proxy-Authorization")) {
// Proxy-Authorization is only sent when the proxy is actually used for
// this target — otherwise NO_PROXY-matched requests would leak proxy
// credentials directly to the destination server.
if (is_proxy_enabled_for_host(host_)) {
if (!proxy_basic_auth_username_.empty() &&
!proxy_basic_auth_password_.empty() &&
!req.has_header("Proxy-Authorization")) {
req.headers.insert(make_basic_authentication_header(
proxy_basic_auth_username_, proxy_basic_auth_password_, true));
}
if (!proxy_bearer_token_auth_token_.empty() &&
!req.has_header("Proxy-Authorization")) {
req.headers.insert(make_bearer_token_authentication_header(
proxy_bearer_token_auth_token_, true));
}
@ -9711,7 +9901,7 @@ bool ClientImpl::process_request(Stream &strm, Request &req,
#ifdef CPPHTTPLIB_SSL_ENABLED
if (is_ssl() && !expect_100_continue) {
auto is_proxy_enabled = !proxy_host_.empty() && proxy_port_ != -1;
auto is_proxy_enabled = is_proxy_enabled_for_host(host_);
if (!is_proxy_enabled) {
if (tls::is_peer_closed(socket_.ssl, socket_.sock)) {
error = Error::SSLPeerCouldBeClosed_;
@ -10718,10 +10908,7 @@ void ClientImpl::stop() {
return;
}
// Otherwise, still holding the mutex, we can shut everything down ourselves
shutdown_ssl(socket_, true);
shutdown_socket(socket_);
close_socket(socket_);
disconnect(/*gracefully=*/true);
}
std::string ClientImpl::host() const { return host_; }
@ -10812,6 +10999,8 @@ void ClientImpl::set_interface(const std::string &intf) {
void ClientImpl::set_proxy(const std::string &host, int port) {
proxy_host_ = host;
proxy_port_ = port;
std::lock_guard<std::mutex> guard(socket_mutex_);
disconnect(/*gracefully=*/true);
}
void ClientImpl::set_proxy_basic_auth(const std::string &username,
@ -10824,6 +11013,22 @@ void ClientImpl::set_proxy_bearer_token_auth(const std::string &token) {
proxy_bearer_token_auth_token_ = token;
}
void ClientImpl::set_no_proxy(const std::vector<std::string> &patterns) {
std::vector<detail::NoProxyEntry> parsed;
parsed.reserve(patterns.size());
for (const auto &p : patterns) {
auto trimmed = detail::trim_copy(p);
if (trimmed.empty()) { continue; }
detail::NoProxyEntry entry;
if (detail::parse_no_proxy_entry(trimmed, entry)) {
parsed.push_back(std::move(entry));
}
}
no_proxy_entries_ = std::move(parsed);
std::lock_guard<std::mutex> guard(socket_mutex_);
disconnect(/*gracefully=*/true);
}
#ifdef CPPHTTPLIB_SSL_ENABLED
void ClientImpl::set_digest_auth(const std::string &username,
const std::string &password) {
@ -11525,6 +11730,9 @@ void Client::set_proxy_basic_auth(const std::string &username,
void Client::set_proxy_bearer_token_auth(const std::string &token) {
cli_->set_proxy_bearer_token_auth(token);
}
void Client::set_no_proxy(const std::vector<std::string> &patterns) {
cli_->set_no_proxy(patterns);
}
void Client::set_logger(Logger logger) {
cli_->set_logger(std::move(logger));
@ -11754,7 +11962,7 @@ bool SSLClient::setup_proxy_connection(
Socket &socket,
std::chrono::time_point<std::chrono::steady_clock> start_time,
Response &res, bool &success, Error &error) {
if (proxy_host_.empty() || proxy_port_ == -1) { return true; }
if (!is_proxy_enabled_for_host(host_)) { return true; }
if (!connect_with_proxy(socket, start_time, res, success, error)) {
return false;
@ -11867,7 +12075,7 @@ bool SSLClient::connect_with_proxy(
bool SSLClient::ensure_socket_connection(Socket &socket, Error &error) {
if (!ClientImpl::ensure_socket_connection(socket, error)) { return false; }
if (!proxy_host_.empty() && proxy_port_ != -1) { return true; }
if (is_proxy_enabled_for_host(host_)) { return true; }
if (!initialize_ssl(socket, error)) {
shutdown_socket(socket);

45
vendor/cpp-httplib/httplib.h vendored
View file

@ -8,8 +8,8 @@
#ifndef CPPHTTPLIB_HTTPLIB_H
#define CPPHTTPLIB_HTTPLIB_H
#define CPPHTTPLIB_VERSION "0.45.1"
#define CPPHTTPLIB_VERSION_NUM "0x002d01"
#define CPPHTTPLIB_VERSION "0.46.0"
#define CPPHTTPLIB_VERSION_NUM "0x002e00"
#ifdef _WIN32
#if defined(_WIN32_WINNT) && _WIN32_WINNT < 0x0A00
@ -2024,6 +2024,31 @@ inline ssize_t read_body_content(Stream *stream, BodyReader &br, char *buf,
class decompressor;
enum class NoProxyKind {
Wildcard, // "*"
HostnameSuffix, // "example.com" or ".example.com"
IPv4Cidr, // "10.0.0.0/8" (or single IP, treated as /32)
IPv6Cidr, // "fe80::/10" (or single IP, treated as /128)
};
// Unified 16-byte buffer holding either a v4 (first 4 bytes) or v6 address.
// Lets one CIDR matcher cover both families.
using IPBytes = std::array<uint8_t, 16>;
struct NoProxyEntry {
NoProxyKind kind = NoProxyKind::Wildcard;
std::string hostname_pattern; // lowercased, leading/trailing dot stripped
IPBytes net{};
int prefix_bits = 0;
};
struct NormalizedTarget {
std::string hostname; // lowercase; brackets and trailing dot removed
bool is_ipv4 = false;
bool is_ipv6 = false;
IPBytes ip{};
};
} // namespace detail
class ClientImpl {
@ -2240,6 +2265,7 @@ public:
void set_proxy_basic_auth(const std::string &username,
const std::string &password);
void set_proxy_bearer_token_auth(const std::string &token);
void set_no_proxy(const std::vector<std::string> &patterns);
void set_logger(Logger logger);
void set_error_logger(ErrorLogger error_logger);
@ -2265,16 +2291,19 @@ protected:
std::chrono::time_point<std::chrono::steady_clock> start_time,
Response &res, bool &success, Error &error);
bool is_proxy_enabled_for_host(const std::string &host) const;
// All of:
// shutdown_ssl
// shutdown_socket
// close_socket
// should ONLY be called when socket_mutex_ is locked.
// Also, shutdown_ssl and close_socket should also NOT be called concurrently
// with a DIFFERENT thread sending requests using that socket.
// disconnect
// should ONLY be called when socket_mutex_ is locked, and only when
// no other thread is using the socket.
virtual void shutdown_ssl(Socket &socket, bool shutdown_gracefully);
void shutdown_socket(Socket &socket) const;
void close_socket(Socket &socket);
void disconnect(bool gracefully);
bool process_request(Stream &strm, Request &req, Response &res,
bool close_connection, Error &error);
@ -2352,6 +2381,11 @@ protected:
std::string proxy_basic_auth_password_;
std::string proxy_bearer_token_auth_token_;
std::vector<detail::NoProxyEntry> no_proxy_entries_;
mutable detail::NormalizedTarget host_normalized_;
mutable bool host_normalized_valid_ = false;
mutable std::mutex logger_mutex_;
Logger logger_;
ErrorLogger error_logger_;
@ -2612,6 +2646,7 @@ public:
void set_proxy_basic_auth(const std::string &username,
const std::string &password);
void set_proxy_bearer_token_auth(const std::string &token);
void set_no_proxy(const std::vector<std::string> &patterns);
void set_logger(Logger logger);
void set_error_logger(ErrorLogger error_logger);