mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2025-09-14 19:09:45 +00:00
vulkan: scalar flash attention implementation
This commit is contained in:
Jeff Bolz
parent
9070365020
commit
005756a2a9
3 changed files with 536 additions and 77 deletions
|
|
@ -275,6 +275,7 @@ struct vk_device_struct {
|
||||||
bool prefer_host_memory;
|
bool prefer_host_memory;
|
||||||
bool float_controls_rte_fp16;
|
bool float_controls_rte_fp16;
|
||||||
bool subgroup_add;
|
bool subgroup_add;
|
||||||
|
bool subgroup_shuffle;
|
||||||
|
|
||||||
bool integer_dot_product;
|
bool integer_dot_product;
|
||||||
|
|
||||||
|
|
@ -402,12 +403,20 @@ struct vk_device_struct {
|
||||||
vk_pipeline pipeline_conv2d_dw_cwhn_f32;
|
vk_pipeline pipeline_conv2d_dw_cwhn_f32;
|
||||||
|
|
||||||
// [2][2][2] is for {f16acc,f32acc}x{large,small_rows}x{unaligned, aligned}
|
// [2][2][2] is for {f16acc,f32acc}x{large,small_rows}x{unaligned, aligned}
|
||||||
|
vk_pipeline pipeline_flash_attn_f32_f16_D64_cm2[GGML_TYPE_COUNT][2][2][2];
|
||||||
|
vk_pipeline pipeline_flash_attn_f32_f16_D80_cm2[GGML_TYPE_COUNT][2][2][2];
|
||||||
|
vk_pipeline pipeline_flash_attn_f32_f16_D96_cm2[GGML_TYPE_COUNT][2][2][2];
|
||||||
|
vk_pipeline pipeline_flash_attn_f32_f16_D112_cm2[GGML_TYPE_COUNT][2][2][2];
|
||||||
|
vk_pipeline pipeline_flash_attn_f32_f16_D128_cm2[GGML_TYPE_COUNT][2][2][2];
|
||||||
|
vk_pipeline pipeline_flash_attn_f32_f16_D256_cm2[GGML_TYPE_COUNT][2][2][2];
|
||||||
|
|
||||||
vk_pipeline pipeline_flash_attn_f32_f16_D64[GGML_TYPE_COUNT][2][2][2];
|
vk_pipeline pipeline_flash_attn_f32_f16_D64[GGML_TYPE_COUNT][2][2][2];
|
||||||
vk_pipeline pipeline_flash_attn_f32_f16_D80[GGML_TYPE_COUNT][2][2][2];
|
vk_pipeline pipeline_flash_attn_f32_f16_D80[GGML_TYPE_COUNT][2][2][2];
|
||||||
vk_pipeline pipeline_flash_attn_f32_f16_D96[GGML_TYPE_COUNT][2][2][2];
|
vk_pipeline pipeline_flash_attn_f32_f16_D96[GGML_TYPE_COUNT][2][2][2];
|
||||||
vk_pipeline pipeline_flash_attn_f32_f16_D112[GGML_TYPE_COUNT][2][2][2];
|
vk_pipeline pipeline_flash_attn_f32_f16_D112[GGML_TYPE_COUNT][2][2][2];
|
||||||
vk_pipeline pipeline_flash_attn_f32_f16_D128[GGML_TYPE_COUNT][2][2][2];
|
vk_pipeline pipeline_flash_attn_f32_f16_D128[GGML_TYPE_COUNT][2][2][2];
|
||||||
vk_pipeline pipeline_flash_attn_f32_f16_D256[GGML_TYPE_COUNT][2][2][2];
|
vk_pipeline pipeline_flash_attn_f32_f16_D256[GGML_TYPE_COUNT][2][2][2];
|
||||||
|
|
||||||
vk_pipeline pipeline_flash_attn_split_k_reduce;
|
vk_pipeline pipeline_flash_attn_split_k_reduce;
|
||||||
|
|
||||||
std::unordered_map<std::string, vk_pipeline_ref> pipelines;
|
std::unordered_map<std::string, vk_pipeline_ref> pipelines;
|
||||||
|
|
@ -1581,13 +1590,20 @@ static void ggml_vk_wait_events(vk_context& ctx, std::vector<vk::Event>&& events
|
||||||
|
|
||||||
// number of rows/cols for flash attention shader
|
// number of rows/cols for flash attention shader
|
||||||
static constexpr uint32_t flash_attention_num_small_rows = 32;
|
static constexpr uint32_t flash_attention_num_small_rows = 32;
|
||||||
static std::array<uint32_t, 2> fa_rows_cols(uint32_t D, uint32_t clamp, ggml_type type, bool small_rows) {
|
static constexpr uint32_t scalar_flash_attention_num_small_rows = 4;
|
||||||
|
|
||||||
|
static uint32_t get_fa_num_small_rows(bool scalar) {
|
||||||
|
return scalar ? scalar_flash_attention_num_small_rows : flash_attention_num_small_rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
static std::array<uint32_t, 2> fa_rows_cols(bool scalar, uint32_t D, uint32_t clamp, ggml_type type, bool small_rows) {
|
||||||
GGML_UNUSED(clamp);
|
GGML_UNUSED(clamp);
|
||||||
|
|
||||||
// small rows, large cols
|
// small rows, large cols
|
||||||
if (small_rows) {
|
if (small_rows || scalar) {
|
||||||
return {flash_attention_num_small_rows, 64};
|
return {get_fa_num_small_rows(scalar), 64};
|
||||||
}
|
}
|
||||||
|
|
||||||
// small cols to reduce register count
|
// small cols to reduce register count
|
||||||
if (ggml_is_quantized(type) || D == 256) {
|
if (ggml_is_quantized(type) || D == 256) {
|
||||||
return {64, 32};
|
return {64, 32};
|
||||||
|
|
@ -1882,65 +1898,62 @@ static void ggml_vk_load_shaders(vk_device& device) {
|
||||||
parameter_count, wg_denoms, specialization_constants, disable_robustness, require_full_subgroups, required_subgroup_size));
|
parameter_count, wg_denoms, specialization_constants, disable_robustness, require_full_subgroups, required_subgroup_size));
|
||||||
};
|
};
|
||||||
|
|
||||||
|
auto const &fa_wg_denoms = [&](bool scalar, uint32_t D, uint32_t clamp, ggml_type type, bool small_rows) -> std::array<uint32_t, 3> {
|
||||||
|
return {fa_rows_cols(scalar, D, clamp, type, small_rows)[0], 1, 1};
|
||||||
|
};
|
||||||
|
|
||||||
|
auto const &fa_spec_constants = [&](bool scalar, uint32_t D, uint32_t clamp, ggml_type type, bool small_rows) -> std::vector<uint32_t> {
|
||||||
|
// For large number of rows, 128 invocations seems to work best.
|
||||||
|
// For small number of rows (e.g. N==1), 256 works better. But matrix granularity for 256 is 32, so we
|
||||||
|
// can't use 256 for D==80.
|
||||||
|
// For scalar, use 128 (arbitrary)
|
||||||
|
uint32_t wg_size = scalar ? 128 : ((small_rows && (D % 32) == 0) ? 256 : 128);
|
||||||
|
auto rows_cols = fa_rows_cols(scalar, D, clamp, type, small_rows);
|
||||||
|
|
||||||
|
// D_split can't be larger than a subgroup because we use subgroupShuffle to reduce it.
|
||||||
|
const uint32_t D_split = std::min(device->subgroup_size, 16u);
|
||||||
|
|
||||||
|
// mask dim1 is padded to 64, we rely on this to avoid clamping mask loads
|
||||||
|
GGML_ASSERT((GGML_KQ_MASK_PAD % rows_cols[0]) == 0);
|
||||||
|
return {wg_size, rows_cols[0], rows_cols[1], (D), clamp, D_split};
|
||||||
|
};
|
||||||
|
|
||||||
|
#define CREATE_FA2(TYPE, NAMELC, SCALAR, SUFFIX, D) \
|
||||||
|
ggml_vk_create_pipeline(device, device->pipeline_flash_attn_f32_f16_D ## D ## SUFFIX[TYPE][0][0][0], "flash_attn_f32_f16_D" #D "_f16acc" #NAMELC #SUFFIX, flash_attn_f32_f16_ ## NAMELC ## _f16acc ## SUFFIX ## _len, flash_attn_f32_f16_ ## NAMELC ## _f16acc ## SUFFIX ## _data, "main", 5, sizeof(vk_flash_attn_push_constants), fa_wg_denoms(SCALAR, D,1,TYPE,false), fa_spec_constants(SCALAR, D,1,TYPE,false), 1); \
|
||||||
|
ggml_vk_create_pipeline(device, device->pipeline_flash_attn_f32_f16_D ## D ## SUFFIX[TYPE][0][0][1], "flash_attn_f32_f16_D" #D "_aligned_f16acc" #NAMELC #SUFFIX, flash_attn_f32_f16_ ## NAMELC ## _f16acc ## SUFFIX ## _len, flash_attn_f32_f16_ ## NAMELC ## _f16acc ## SUFFIX ## _data, "main", 5, sizeof(vk_flash_attn_push_constants), fa_wg_denoms(SCALAR, D,0,TYPE,false), fa_spec_constants(SCALAR, D,0,TYPE,false), fa_rows_cols(SCALAR,D,0,TYPE,false)[1]); \
|
||||||
|
ggml_vk_create_pipeline(device, device->pipeline_flash_attn_f32_f16_D ## D ## SUFFIX[TYPE][1][0][0], "flash_attn_f32_f16_D" #D "_f32acc" #NAMELC #SUFFIX, flash_attn_f32_f16_ ## NAMELC ## SUFFIX ## _len, flash_attn_f32_f16_ ## NAMELC ## SUFFIX ## _data, "main", 5, sizeof(vk_flash_attn_push_constants), fa_wg_denoms(SCALAR, D,1,TYPE,false), fa_spec_constants(SCALAR, D,1,TYPE,false), 1); \
|
||||||
|
ggml_vk_create_pipeline(device, device->pipeline_flash_attn_f32_f16_D ## D ## SUFFIX[TYPE][1][0][1], "flash_attn_f32_f16_D" #D "_aligned_f32acc" #NAMELC #SUFFIX, flash_attn_f32_f16_ ## NAMELC ## SUFFIX ## _len, flash_attn_f32_f16_ ## NAMELC ## SUFFIX ## _data, "main", 5, sizeof(vk_flash_attn_push_constants), fa_wg_denoms(SCALAR, D,0,TYPE,false), fa_spec_constants(SCALAR, D,0,TYPE,false), fa_rows_cols(SCALAR,D,0,TYPE,false)[1]); \
|
||||||
|
ggml_vk_create_pipeline(device, device->pipeline_flash_attn_f32_f16_D ## D ## SUFFIX[TYPE][0][1][0], "flash_attn_f32_f16_D" #D "_f16acc_smallrows" #NAMELC #SUFFIX, flash_attn_f32_f16_ ## NAMELC ## _f16acc ## SUFFIX ## _len, flash_attn_f32_f16_ ## NAMELC ## _f16acc ## SUFFIX ## _data, "main", 5, sizeof(vk_flash_attn_push_constants), fa_wg_denoms(SCALAR, D,1,TYPE,true), fa_spec_constants(SCALAR, D,1,TYPE,true), 1); \
|
||||||
|
ggml_vk_create_pipeline(device, device->pipeline_flash_attn_f32_f16_D ## D ## SUFFIX[TYPE][0][1][1], "flash_attn_f32_f16_D" #D "_aligned_f16acc_smallrows" #NAMELC #SUFFIX, flash_attn_f32_f16_ ## NAMELC ## _f16acc ## SUFFIX ## _len, flash_attn_f32_f16_ ## NAMELC ## _f16acc ## SUFFIX ## _data, "main", 5, sizeof(vk_flash_attn_push_constants), fa_wg_denoms(SCALAR, D,0,TYPE,true), fa_spec_constants(SCALAR, D,0,TYPE,true), fa_rows_cols(SCALAR,D,0,TYPE,true)[1]); \
|
||||||
|
ggml_vk_create_pipeline(device, device->pipeline_flash_attn_f32_f16_D ## D ## SUFFIX[TYPE][1][1][0], "flash_attn_f32_f16_D" #D "_f32acc_smallrows" #NAMELC #SUFFIX, flash_attn_f32_f16_ ## NAMELC ## SUFFIX ## _len, flash_attn_f32_f16_ ## NAMELC ## SUFFIX ## _data, "main", 5, sizeof(vk_flash_attn_push_constants), fa_wg_denoms(SCALAR, D,1,TYPE,true), fa_spec_constants(SCALAR, D,1,TYPE,true), 1); \
|
||||||
|
ggml_vk_create_pipeline(device, device->pipeline_flash_attn_f32_f16_D ## D ## SUFFIX[TYPE][1][1][1], "flash_attn_f32_f16_D" #D "_aligned_f32acc_smallrows" #NAMELC #SUFFIX, flash_attn_f32_f16_ ## NAMELC ## SUFFIX ## _len, flash_attn_f32_f16_ ## NAMELC ## SUFFIX ## _data, "main", 5, sizeof(vk_flash_attn_push_constants), fa_wg_denoms(SCALAR, D,0,TYPE,true), fa_spec_constants(SCALAR, D,0,TYPE,true), fa_rows_cols(SCALAR,D,0,TYPE,true)[1]); \
|
||||||
|
|
||||||
|
#define CREATE_FA(TYPE, NAMELC, SCALAR, SUFFIX) \
|
||||||
|
CREATE_FA2(TYPE, NAMELC, SCALAR, SUFFIX, 64) \
|
||||||
|
CREATE_FA2(TYPE, NAMELC, SCALAR, SUFFIX, 80) \
|
||||||
|
CREATE_FA2(TYPE, NAMELC, SCALAR, SUFFIX, 96) \
|
||||||
|
CREATE_FA2(TYPE, NAMELC, SCALAR, SUFFIX, 112) \
|
||||||
|
CREATE_FA2(TYPE, NAMELC, SCALAR, SUFFIX, 128) \
|
||||||
|
CREATE_FA2(TYPE, NAMELC, SCALAR, SUFFIX, 256)
|
||||||
|
|
||||||
|
CREATE_FA(GGML_TYPE_F16, f16, true, )
|
||||||
#if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT)
|
#if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT)
|
||||||
if (device->coopmat2) {
|
if (device->coopmat2) {
|
||||||
|
CREATE_FA(GGML_TYPE_F16, f16, false, _cm2)
|
||||||
auto const &fa_wg_denoms = [&](uint32_t D, uint32_t clamp, ggml_type type, bool small_rows) -> std::array<uint32_t, 3> {
|
CREATE_FA(GGML_TYPE_Q4_0, q4_0, false, _cm2)
|
||||||
return {fa_rows_cols(D, clamp, type, small_rows)[0], 1, 1};
|
CREATE_FA(GGML_TYPE_Q4_1, q4_1, false, _cm2)
|
||||||
};
|
CREATE_FA(GGML_TYPE_Q5_0, q5_0, false, _cm2)
|
||||||
|
CREATE_FA(GGML_TYPE_Q5_1, q5_1, false, _cm2)
|
||||||
auto const &fa_spec_constants = [&](uint32_t D, uint32_t clamp, ggml_type type, bool small_rows) -> std::vector<uint32_t> {
|
CREATE_FA(GGML_TYPE_Q8_0, q8_0, false, _cm2)
|
||||||
// For large number of rows, 128 invocations seems to work best.
|
CREATE_FA(GGML_TYPE_IQ4_NL, iq4_nl, false, _cm2)
|
||||||
// For small number of rows (e.g. N==1), 256 works better. But matrix granularity for 256 is 32, so we
|
}
|
||||||
// can't use 256 for D==80.
|
#endif
|
||||||
uint32_t wg_size = (small_rows && (D % 32) == 0) ? 256 : 128;
|
#undef CREATE_FA2
|
||||||
auto rows_cols = fa_rows_cols(D, clamp, type, small_rows);
|
|
||||||
// mask dim1 is padded to 64, we rely on this to avoid clamping mask loads
|
|
||||||
GGML_ASSERT((GGML_KQ_MASK_PAD % rows_cols[0]) == 0);
|
|
||||||
return {wg_size, rows_cols[0], rows_cols[1], (D), clamp};
|
|
||||||
};
|
|
||||||
|
|
||||||
#define CREATE_FA2(TYPE, NAMELC, D) \
|
|
||||||
ggml_vk_create_pipeline(device, device->pipeline_flash_attn_f32_f16_D ## D[TYPE][0][0][0], "flash_attn_f32_f16_D" #D "_f16acc" #NAMELC, flash_attn_f32_f16_ ## NAMELC ## _f16acc_cm2_len, flash_attn_f32_f16_ ## NAMELC ## _f16acc_cm2_data, "main", 5, sizeof(vk_flash_attn_push_constants), fa_wg_denoms(D,1,TYPE,false), fa_spec_constants(D,1,TYPE,false), 1); \
|
|
||||||
ggml_vk_create_pipeline(device, device->pipeline_flash_attn_f32_f16_D ## D[TYPE][0][0][1], "flash_attn_f32_f16_D" #D "_aligned_f16acc" #NAMELC, flash_attn_f32_f16_ ## NAMELC ## _f16acc_cm2_len, flash_attn_f32_f16_ ## NAMELC ## _f16acc_cm2_data, "main", 5, sizeof(vk_flash_attn_push_constants), fa_wg_denoms(D,0,TYPE,false), fa_spec_constants(D,0,TYPE,false), fa_rows_cols(D,0,TYPE,false)[1]); \
|
|
||||||
ggml_vk_create_pipeline(device, device->pipeline_flash_attn_f32_f16_D ## D[TYPE][1][0][0], "flash_attn_f32_f16_D" #D "_f32acc" #NAMELC, flash_attn_f32_f16_ ## NAMELC ## _cm2_len, flash_attn_f32_f16_ ## NAMELC ## _cm2_data, "main", 5, sizeof(vk_flash_attn_push_constants), fa_wg_denoms(D,1,TYPE,false), fa_spec_constants(D,1,TYPE,false), 1); \
|
|
||||||
ggml_vk_create_pipeline(device, device->pipeline_flash_attn_f32_f16_D ## D[TYPE][1][0][1], "flash_attn_f32_f16_D" #D "_aligned_f32acc" #NAMELC, flash_attn_f32_f16_ ## NAMELC ## _cm2_len, flash_attn_f32_f16_ ## NAMELC ## _cm2_data, "main", 5, sizeof(vk_flash_attn_push_constants), fa_wg_denoms(D,0,TYPE,false), fa_spec_constants(D,0,TYPE,false), fa_rows_cols(D,0,TYPE,false)[1]); \
|
|
||||||
ggml_vk_create_pipeline(device, device->pipeline_flash_attn_f32_f16_D ## D[TYPE][0][1][0], "flash_attn_f32_f16_D" #D "_f16acc_smallrows" #NAMELC, flash_attn_f32_f16_ ## NAMELC ## _f16acc_cm2_len, flash_attn_f32_f16_ ## NAMELC ## _f16acc_cm2_data, "main", 5, sizeof(vk_flash_attn_push_constants), fa_wg_denoms(D,1,TYPE,true), fa_spec_constants(D,1,TYPE,true), 1); \
|
|
||||||
ggml_vk_create_pipeline(device, device->pipeline_flash_attn_f32_f16_D ## D[TYPE][0][1][1], "flash_attn_f32_f16_D" #D "_aligned_f16acc_smallrows" #NAMELC, flash_attn_f32_f16_ ## NAMELC ## _f16acc_cm2_len, flash_attn_f32_f16_ ## NAMELC ## _f16acc_cm2_data, "main", 5, sizeof(vk_flash_attn_push_constants), fa_wg_denoms(D,0,TYPE,true), fa_spec_constants(D,0,TYPE,true), fa_rows_cols(D,0,TYPE,true)[1]); \
|
|
||||||
ggml_vk_create_pipeline(device, device->pipeline_flash_attn_f32_f16_D ## D[TYPE][1][1][0], "flash_attn_f32_f16_D" #D "_f32acc_smallrows" #NAMELC, flash_attn_f32_f16_ ## NAMELC ## _cm2_len, flash_attn_f32_f16_ ## NAMELC ## _cm2_data, "main", 5, sizeof(vk_flash_attn_push_constants), fa_wg_denoms(D,1,TYPE,true), fa_spec_constants(D,1,TYPE,true), 1); \
|
|
||||||
ggml_vk_create_pipeline(device, device->pipeline_flash_attn_f32_f16_D ## D[TYPE][1][1][1], "flash_attn_f32_f16_D" #D "_aligned_f32acc_smallrows" #NAMELC, flash_attn_f32_f16_ ## NAMELC ## _cm2_len, flash_attn_f32_f16_ ## NAMELC ## _cm2_data, "main", 5, sizeof(vk_flash_attn_push_constants), fa_wg_denoms(D,0,TYPE,true), fa_spec_constants(D,0,TYPE,true), fa_rows_cols(D,0,TYPE,true)[1]); \
|
|
||||||
|
|
||||||
#define CREATE_FA(TYPE, NAMELC) \
|
|
||||||
CREATE_FA2(TYPE, NAMELC, 64) \
|
|
||||||
CREATE_FA2(TYPE, NAMELC, 80) \
|
|
||||||
CREATE_FA2(TYPE, NAMELC, 96) \
|
|
||||||
CREATE_FA2(TYPE, NAMELC, 112) \
|
|
||||||
CREATE_FA2(TYPE, NAMELC, 128) \
|
|
||||||
CREATE_FA2(TYPE, NAMELC, 256)
|
|
||||||
|
|
||||||
CREATE_FA(GGML_TYPE_F16, f16)
|
|
||||||
CREATE_FA(GGML_TYPE_Q4_0, q4_0)
|
|
||||||
CREATE_FA(GGML_TYPE_Q4_1, q4_1)
|
|
||||||
CREATE_FA(GGML_TYPE_Q5_0, q5_0)
|
|
||||||
CREATE_FA(GGML_TYPE_Q5_1, q5_1)
|
|
||||||
CREATE_FA(GGML_TYPE_Q8_0, q8_0)
|
|
||||||
// K dequants currently disabled because D dimension is rounded up to 256 and runs inefficiently
|
|
||||||
//CREATE_FA(GGML_TYPE_Q2_K, q2_k)
|
|
||||||
//CREATE_FA(GGML_TYPE_Q3_K, q3_k)
|
|
||||||
//CREATE_FA(GGML_TYPE_Q4_K, q4_k)
|
|
||||||
//CREATE_FA(GGML_TYPE_Q5_K, q5_k)
|
|
||||||
//CREATE_FA(GGML_TYPE_Q6_K, q6_k)
|
|
||||||
//CREATE_FA(GGML_TYPE_IQ1_S, iq1_s)
|
|
||||||
//CREATE_FA(GGML_TYPE_IQ1_M, iq1_m)
|
|
||||||
//CREATE_FA(GGML_TYPE_IQ2_XXS, iq2_xxs)
|
|
||||||
//CREATE_FA(GGML_TYPE_IQ2_XS, iq2_xs)
|
|
||||||
//CREATE_FA(GGML_TYPE_IQ2_S, iq2_s)
|
|
||||||
//CREATE_FA(GGML_TYPE_IQ3_XXS, iq3_xxs)
|
|
||||||
//CREATE_FA(GGML_TYPE_IQ3_S, iq3_s)
|
|
||||||
//CREATE_FA(GGML_TYPE_IQ4_XS, iq4_xs)
|
|
||||||
CREATE_FA(GGML_TYPE_IQ4_NL, iq4_nl)
|
|
||||||
#undef CREATE_FA
|
#undef CREATE_FA
|
||||||
|
|
||||||
|
#if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT)
|
||||||
|
if (device->coopmat2) {
|
||||||
|
|
||||||
// Create 6 variants, {s,m,l}x{unaligned,aligned}
|
// Create 6 variants, {s,m,l}x{unaligned,aligned}
|
||||||
#define CREATE_MM(PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT) \
|
#define CREATE_MM(PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT) \
|
||||||
ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC #F16ACC "_l", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, l_ ## WARPTILE, 1); \
|
ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC #F16ACC "_l", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, l_ ## WARPTILE, 1); \
|
||||||
|
|
@ -2837,6 +2850,9 @@ static vk_device ggml_vk_get_device(size_t idx) {
|
||||||
device->subgroup_add = (vk11_props.subgroupSupportedStages & vk::ShaderStageFlagBits::eCompute) &&
|
device->subgroup_add = (vk11_props.subgroupSupportedStages & vk::ShaderStageFlagBits::eCompute) &&
|
||||||
(vk11_props.subgroupSupportedOperations & vk::SubgroupFeatureFlagBits::eArithmetic);
|
(vk11_props.subgroupSupportedOperations & vk::SubgroupFeatureFlagBits::eArithmetic);
|
||||||
|
|
||||||
|
device->subgroup_shuffle = (vk11_props.subgroupSupportedStages & vk::ShaderStageFlagBits::eCompute) &&
|
||||||
|
(vk11_props.subgroupSupportedOperations & vk::SubgroupFeatureFlagBits::eShuffle);
|
||||||
|
|
||||||
const bool force_disable_f16 = getenv("GGML_VK_DISABLE_F16") != nullptr;
|
const bool force_disable_f16 = getenv("GGML_VK_DISABLE_F16") != nullptr;
|
||||||
|
|
||||||
device->fp16 = !force_disable_f16 && fp16_storage && fp16_compute;
|
device->fp16 = !force_disable_f16 && fp16_storage && fp16_compute;
|
||||||
|
|
@ -5712,17 +5728,33 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx
|
||||||
vk_pipeline *pipelines;
|
vk_pipeline *pipelines;
|
||||||
// XXX TODO other backends may be changing accumulator precision to default to f32 soon
|
// XXX TODO other backends may be changing accumulator precision to default to f32 soon
|
||||||
bool f32acc = dst->op_params[3] == GGML_PREC_F32;
|
bool f32acc = dst->op_params[3] == GGML_PREC_F32;
|
||||||
bool small_rows = N <= flash_attention_num_small_rows;
|
bool scalar = !ctx->device->coopmat2;
|
||||||
switch (D) {
|
bool small_rows = N <= get_fa_num_small_rows(scalar);
|
||||||
case 64: pipelines = &ctx->device->pipeline_flash_attn_f32_f16_D64[k->type][f32acc][small_rows][0]; break;
|
|
||||||
case 80: pipelines = &ctx->device->pipeline_flash_attn_f32_f16_D80[k->type][f32acc][small_rows][0]; break;
|
if (scalar) {
|
||||||
case 96: pipelines = &ctx->device->pipeline_flash_attn_f32_f16_D96[k->type][f32acc][small_rows][0]; break;
|
switch (D) {
|
||||||
case 112: pipelines = &ctx->device->pipeline_flash_attn_f32_f16_D112[k->type][f32acc][small_rows][0]; break;
|
case 64: pipelines = &ctx->device->pipeline_flash_attn_f32_f16_D64[k->type][f32acc][small_rows][0]; break;
|
||||||
case 128: pipelines = &ctx->device->pipeline_flash_attn_f32_f16_D128[k->type][f32acc][small_rows][0]; break;
|
case 80: pipelines = &ctx->device->pipeline_flash_attn_f32_f16_D80[k->type][f32acc][small_rows][0]; break;
|
||||||
case 256: pipelines = &ctx->device->pipeline_flash_attn_f32_f16_D256[k->type][f32acc][small_rows][0]; break;
|
case 96: pipelines = &ctx->device->pipeline_flash_attn_f32_f16_D96[k->type][f32acc][small_rows][0]; break;
|
||||||
default:
|
case 112: pipelines = &ctx->device->pipeline_flash_attn_f32_f16_D112[k->type][f32acc][small_rows][0]; break;
|
||||||
assert(!"unsupported D value");
|
case 128: pipelines = &ctx->device->pipeline_flash_attn_f32_f16_D128[k->type][f32acc][small_rows][0]; break;
|
||||||
return;
|
case 256: pipelines = &ctx->device->pipeline_flash_attn_f32_f16_D256[k->type][f32acc][small_rows][0]; break;
|
||||||
|
default:
|
||||||
|
GGML_ASSERT(!"unsupported D value");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
switch (D) {
|
||||||
|
case 64: pipelines = &ctx->device->pipeline_flash_attn_f32_f16_D64_cm2[k->type][f32acc][small_rows][0]; break;
|
||||||
|
case 80: pipelines = &ctx->device->pipeline_flash_attn_f32_f16_D80_cm2[k->type][f32acc][small_rows][0]; break;
|
||||||
|
case 96: pipelines = &ctx->device->pipeline_flash_attn_f32_f16_D96_cm2[k->type][f32acc][small_rows][0]; break;
|
||||||
|
case 112: pipelines = &ctx->device->pipeline_flash_attn_f32_f16_D112_cm2[k->type][f32acc][small_rows][0]; break;
|
||||||
|
case 128: pipelines = &ctx->device->pipeline_flash_attn_f32_f16_D128_cm2[k->type][f32acc][small_rows][0]; break;
|
||||||
|
case 256: pipelines = &ctx->device->pipeline_flash_attn_f32_f16_D256_cm2[k->type][f32acc][small_rows][0]; break;
|
||||||
|
default:
|
||||||
|
GGML_ASSERT(!"unsupported D value");
|
||||||
|
return;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
assert(pipelines);
|
assert(pipelines);
|
||||||
|
|
||||||
|
|
@ -5746,7 +5778,9 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx
|
||||||
uint32_t workgroups_y = (uint32_t)neq2;
|
uint32_t workgroups_y = (uint32_t)neq2;
|
||||||
uint32_t workgroups_z = (uint32_t)neq3;
|
uint32_t workgroups_z = (uint32_t)neq3;
|
||||||
|
|
||||||
if (N == 1 && qk_ratio > 1 && gqa_ratio <= flash_attention_num_small_rows &&
|
const uint32_t max_gqa = get_fa_num_small_rows(scalar);
|
||||||
|
|
||||||
|
if (N == 1 && qk_ratio > 1 && qk_ratio <= max_gqa &&
|
||||||
qk_ratio * nek2 == neq2 && nek2 == nev2 && neq3 == 1 && nek3 == 1 && nev3 == 1) {
|
qk_ratio * nek2 == neq2 && nek2 == nev2 && neq3 == 1 && nek3 == 1 && nev3 == 1) {
|
||||||
// grouped query attention - make the N dimension equal to gqa_ratio, reduce
|
// grouped query attention - make the N dimension equal to gqa_ratio, reduce
|
||||||
// workgroups proportionally in y dimension. The shader will detect gqa_ratio > 1
|
// workgroups proportionally in y dimension. The shader will detect gqa_ratio > 1
|
||||||
|
|
@ -5759,8 +5793,11 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx
|
||||||
uint32_t split_kv = KV;
|
uint32_t split_kv = KV;
|
||||||
uint32_t split_k = 1;
|
uint32_t split_k = 1;
|
||||||
|
|
||||||
|
// Use a placeholder core count if one isn't available. split_k is a big help for perf.
|
||||||
|
const uint32_t shader_core_count = ctx->device->shader_core_count ? ctx->device->shader_core_count : 16;
|
||||||
|
|
||||||
// Try to use split_k when KV is large enough to be worth the overhead
|
// Try to use split_k when KV is large enough to be worth the overhead
|
||||||
if (workgroups_x == 1 && ctx->device->shader_core_count > 0 && KV >= 512) {
|
if (workgroups_x == 1 && shader_core_count > 0 && KV >= 512) {
|
||||||
// Try to run two workgroups per SM.
|
// Try to run two workgroups per SM.
|
||||||
split_k = ctx->device->shader_core_count * 2 / workgroups_y;
|
split_k = ctx->device->shader_core_count * 2 / workgroups_y;
|
||||||
if (split_k > 1) {
|
if (split_k > 1) {
|
||||||
|
|
@ -9530,9 +9567,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
|
||||||
case GGML_OP_FLASH_ATTN_EXT:
|
case GGML_OP_FLASH_ATTN_EXT:
|
||||||
{
|
{
|
||||||
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
|
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
|
||||||
if (!ggml_vk_get_device(ctx->device)->coopmat2) {
|
auto device = ggml_vk_get_device(ctx->device);
|
||||||
return false;
|
bool coopmat2 = device->coopmat2;
|
||||||
}
|
|
||||||
switch (op->src[0]->ne[0]) {
|
switch (op->src[0]->ne[0]) {
|
||||||
case 64:
|
case 64:
|
||||||
case 80:
|
case 80:
|
||||||
|
|
@ -9540,7 +9576,6 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
|
||||||
case 112:
|
case 112:
|
||||||
case 128:
|
case 128:
|
||||||
case 256:
|
case 256:
|
||||||
case 575: // DeepSeek MLA
|
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
return false;
|
return false;
|
||||||
|
|
@ -9589,6 +9624,13 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
|
||||||
default:
|
default:
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
if (!coopmat2 && op->src[1]->type != GGML_TYPE_F16) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (!coopmat2 && !device->subgroup_shuffle) {
|
||||||
|
// scalar FA uses subgroupShuffle
|
||||||
|
return false;
|
||||||
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
case GGML_OP_GET_ROWS:
|
case GGML_OP_GET_ROWS:
|
||||||
|
|
|
||||||
413
ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp
Normal file
413
ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp
Normal file
|
|
@ -0,0 +1,413 @@
|
||||||
|
#version 450
|
||||||
|
|
||||||
|
#extension GL_EXT_control_flow_attributes : enable
|
||||||
|
#extension GL_EXT_shader_16bit_storage : require
|
||||||
|
|
||||||
|
#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require
|
||||||
|
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
|
||||||
|
|
||||||
|
#extension GL_KHR_shader_subgroup_shuffle : enable
|
||||||
|
|
||||||
|
#include "types.comp"
|
||||||
|
|
||||||
|
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
||||||
|
|
||||||
|
layout (constant_id = 1) const uint32_t Br = 1;
|
||||||
|
layout (constant_id = 2) const uint32_t Bc = 32;
|
||||||
|
layout (constant_id = 3) const uint32_t D = 32;
|
||||||
|
|
||||||
|
layout (constant_id = 5) const uint32_t D_split = 16;
|
||||||
|
const uint32_t D_per_thread = D / D_split;
|
||||||
|
|
||||||
|
const uint32_t cols_per_iter = gl_WorkGroupSize.x / D_split;
|
||||||
|
const uint32_t cols_per_thread = Bc / cols_per_iter;
|
||||||
|
|
||||||
|
layout (push_constant) uniform parameter {
|
||||||
|
uint32_t N;
|
||||||
|
uint32_t KV;
|
||||||
|
|
||||||
|
uint32_t ne1;
|
||||||
|
uint32_t ne2;
|
||||||
|
uint32_t ne3;
|
||||||
|
|
||||||
|
uint32_t neq2;
|
||||||
|
uint32_t neq3;
|
||||||
|
uint32_t nek2;
|
||||||
|
uint32_t nek3;
|
||||||
|
uint32_t nev2;
|
||||||
|
uint32_t nev3;
|
||||||
|
uint32_t nem1;
|
||||||
|
|
||||||
|
uint32_t nb01;
|
||||||
|
uint32_t nb02;
|
||||||
|
uint32_t nb03;
|
||||||
|
uint32_t nb11;
|
||||||
|
uint32_t nb12;
|
||||||
|
uint32_t nb13;
|
||||||
|
uint32_t nb21;
|
||||||
|
uint32_t nb22;
|
||||||
|
uint32_t nb23;
|
||||||
|
uint32_t nb31;
|
||||||
|
|
||||||
|
float scale;
|
||||||
|
float max_bias;
|
||||||
|
float logit_softcap;
|
||||||
|
|
||||||
|
uint32_t mask;
|
||||||
|
uint32_t n_head_log2;
|
||||||
|
float m0;
|
||||||
|
float m1;
|
||||||
|
|
||||||
|
uint32_t gqa_ratio;
|
||||||
|
uint32_t split_kv;
|
||||||
|
uint32_t k_num;
|
||||||
|
} p;
|
||||||
|
|
||||||
|
layout (binding = 0) readonly buffer Q {float data_q[];};
|
||||||
|
layout (binding = 1) readonly buffer K {float16_t data_k[];};
|
||||||
|
layout (binding = 2) readonly buffer V {float16_t data_v[];};
|
||||||
|
layout (binding = 3) readonly buffer M {float16_t data_m[];};
|
||||||
|
layout (binding = 4) writeonly buffer O {D_TYPE data_o[];};
|
||||||
|
|
||||||
|
#define CEIL_DIV(a, b) (((a) + (b) - 1) / (b))
|
||||||
|
|
||||||
|
// Store the output when doing grouped query attention.
|
||||||
|
// Rows index by Q's dimension 2, and the first N rows are valid.
|
||||||
|
D_TYPE perElemOpGqaStore(const in uint32_t r, const in uint32_t c, const in D_TYPE elem, const in uint32_t o_offset, const in uint32_t iq2, const in uint32_t N)
|
||||||
|
{
|
||||||
|
uint32_t offset = (iq2 + r) * D + c;
|
||||||
|
data_o[o_offset + offset] = D_TYPE(elem);
|
||||||
|
return elem;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store column zero. This is used to save per-row m and L values for split_k.
|
||||||
|
ACC_TYPE perElemOpStoreCol0(const in uint32_t r, const in uint32_t c, const in ACC_TYPE elem, const in uint32_t o_offset, const in uint32_t iq2, const in uint32_t N)
|
||||||
|
{
|
||||||
|
if (r < N && c == 0) {
|
||||||
|
uint32_t offset = iq2 + r;
|
||||||
|
data_o[o_offset + offset] = D_TYPE(elem);
|
||||||
|
}
|
||||||
|
return elem;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load the slope matrix, indexed by Q's dimension 2.
|
||||||
|
ACC_TYPE perElemOpComputeSlope(const in uint32_t r, const in uint32_t c, const in ACC_TYPE elem, const in uint32_t iq2)
|
||||||
|
{
|
||||||
|
const uint32_t h = iq2 + (r % p.gqa_ratio);
|
||||||
|
|
||||||
|
const ACC_TYPE base = ACC_TYPE(h < p.n_head_log2 ? p.m0 : p.m1);
|
||||||
|
const int exph = int(h < p.n_head_log2 ? h + 1 : 2*(h - p.n_head_log2) + 1);
|
||||||
|
|
||||||
|
return ACC_TYPE(pow(base, ACC_TYPE(exph)));
|
||||||
|
}
|
||||||
|
|
||||||
|
shared FLOAT_TYPE tmpsh[gl_WorkGroupSize.x];
|
||||||
|
|
||||||
|
void main() {
|
||||||
|
#ifdef NEEDS_INIT_IQ_SHMEM
|
||||||
|
init_iq_shmem(gl_WorkGroupSize);
|
||||||
|
#endif
|
||||||
|
|
||||||
|
const uint32_t tid = gl_LocalInvocationIndex;
|
||||||
|
const uint32_t N = p.N;
|
||||||
|
const uint32_t KV = p.KV;
|
||||||
|
|
||||||
|
const uint32_t d_tid = gl_LocalInvocationIndex % D_split;
|
||||||
|
const uint32_t col_tid = gl_LocalInvocationIndex / D_split;
|
||||||
|
|
||||||
|
uint32_t i = gl_WorkGroupID.x;
|
||||||
|
uint32_t split_k_index = 0;
|
||||||
|
|
||||||
|
if (p.k_num > 1) {
|
||||||
|
i = 0;
|
||||||
|
split_k_index = gl_WorkGroupID.x;
|
||||||
|
}
|
||||||
|
|
||||||
|
const uint32_t Tr = CEIL_DIV(N, Br);
|
||||||
|
|
||||||
|
const uint32_t start_j = split_k_index * p.split_kv / Bc;
|
||||||
|
const uint32_t end_j = CEIL_DIV(min(KV, (split_k_index + 1) * p.split_kv), Bc);
|
||||||
|
|
||||||
|
// When not using grouped query attention, all rows share the same iq2, equal to gl_WorkGroupID.y.
|
||||||
|
// When using grouped query attention, each workgroup does gqa_ratio consecutive values of iq2.
|
||||||
|
const uint32_t iq2 = gl_WorkGroupID.y * p.gqa_ratio;
|
||||||
|
const uint32_t iq3 = gl_WorkGroupID.z;
|
||||||
|
|
||||||
|
// broadcast factors
|
||||||
|
const uint32_t rk2 = p.neq2/p.nek2;
|
||||||
|
const uint32_t rk3 = p.neq3/p.nek3;
|
||||||
|
|
||||||
|
const uint32_t rv2 = p.neq2/p.nev2;
|
||||||
|
const uint32_t rv3 = p.neq3/p.nev3;
|
||||||
|
|
||||||
|
// k indices
|
||||||
|
const uint32_t ik3 = iq3 / rk3;
|
||||||
|
const uint32_t ik2 = iq2 / rk2;
|
||||||
|
|
||||||
|
// v indices
|
||||||
|
const uint32_t iv3 = iq3 / rv3;
|
||||||
|
const uint32_t iv2 = iq2 / rv2;
|
||||||
|
|
||||||
|
// nb?1 are already divided by the type size and are in units of elements.
|
||||||
|
// When using grouped query attention, Q is indexed by iq2, so the stride
|
||||||
|
// should be nb02 (which is in bytes).
|
||||||
|
uint32_t q_stride = p.gqa_ratio > 1 ? (p.nb02 / 4) : p.nb01;
|
||||||
|
uint32_t k_stride = p.nb11;
|
||||||
|
uint32_t v_stride = p.nb21;
|
||||||
|
// When using grouped query attention, all rows use the same mask (stride 0).
|
||||||
|
// "p.gqa_ratio >> 16" is just a roundabout way of writing zero
|
||||||
|
// that prevents the compiler from folding the "&" through the select
|
||||||
|
// and breaking the alignment detection.
|
||||||
|
uint32_t m_stride = (p.gqa_ratio > 1) ? (p.gqa_ratio >> 16) : KV;
|
||||||
|
|
||||||
|
uint32_t q_offset = (iq2*p.nb02+iq3*p.nb03) / 4;
|
||||||
|
float Qf[Br][D_per_thread];
|
||||||
|
[[unroll]] for (uint32_t r = 0; r < Br; ++r) {
|
||||||
|
if (i * Br + r < N) {
|
||||||
|
[[unroll]] for (uint32_t d = 0; d < D_per_thread; ++d) {
|
||||||
|
Qf[r][d] = float(data_q[q_offset + (i * Br + r) * q_stride + d * D_split + d_tid]) * p.scale;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
float Of[Br][D_per_thread];
|
||||||
|
[[unroll]] for (uint32_t d = 0; d < D_per_thread; ++d) {
|
||||||
|
[[unroll]] for (uint32_t r = 0; r < Br; ++r) {
|
||||||
|
Of[r][d] = 0.0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
float Lf[Br], Mf[Br];
|
||||||
|
|
||||||
|
// Use -FLT_MAX/2 rather than -inf to reduce the possibility of NaNs, e.g. when computing Mold-M.
|
||||||
|
const float NEG_FLT_MAX_OVER_2 = uintBitsToFloat(0xFEFFFFFF);
|
||||||
|
|
||||||
|
[[unroll]] for (uint32_t r = 0; r < Br; ++r) {
|
||||||
|
Lf[r] = 0;
|
||||||
|
Mf[r] = NEG_FLT_MAX_OVER_2;
|
||||||
|
}
|
||||||
|
|
||||||
|
float slope[Br];
|
||||||
|
[[unroll]] for (uint32_t r = 0; r < Br; ++r) {
|
||||||
|
slope[r] = 1.0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ALiBi
|
||||||
|
if (p.max_bias > 0.0f) {
|
||||||
|
[[unroll]] for (uint32_t r = 0; r < Br; ++r) {
|
||||||
|
slope[r] = perElemOpComputeSlope(r, col_tid, ACC_TYPE(0), iq2);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[[dont_unroll]]
|
||||||
|
for (uint32_t j = start_j; j < end_j; ++j) {
|
||||||
|
|
||||||
|
float Sf[Br][cols_per_thread];
|
||||||
|
[[unroll]] for (uint32_t r = 0; r < Br; ++r) {
|
||||||
|
[[unroll]] for (uint32_t c = 0; c < cols_per_thread; ++c) {
|
||||||
|
Sf[r][c] = 0.0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
uint32_t k_offset = (ik2*p.nb12 + ik3*p.nb13) / 2;
|
||||||
|
|
||||||
|
[[unroll]] for (uint32_t c = 0; c < cols_per_thread; ++c) {
|
||||||
|
[[unroll]] for (uint32_t d = 0; d < D_per_thread; ++d) {
|
||||||
|
float K_Tf = float(data_k[k_offset + (j * Bc + c * cols_per_iter + col_tid) * k_stride + d * D_split + d_tid]);
|
||||||
|
[[unroll]] for (uint32_t r = 0; r < Br; ++r) {
|
||||||
|
Sf[r][c] += Qf[r][d] * K_Tf;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[[unroll]] for (uint32_t c = 0; c < cols_per_thread; ++c) {
|
||||||
|
// Compute sum across the D_split
|
||||||
|
[[unroll]] for (uint s = D_split / 2; s > 0; s >>= 1) {
|
||||||
|
[[unroll]] for (uint32_t r = 0; r < Br; ++r) {
|
||||||
|
Sf[r][c] += subgroupShuffleXor(Sf[r][c], s);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (p.logit_softcap != 0.0f) {
|
||||||
|
[[unroll]] for (uint32_t r = 0; r < Br; ++r) {
|
||||||
|
[[unroll]] for (uint32_t c = 0; c < cols_per_thread; ++c) {
|
||||||
|
Sf[r][c] = p.logit_softcap * tanh(Sf[r][c]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (p.mask != 0) {
|
||||||
|
[[unroll]] for (uint32_t c = 0; c < cols_per_thread; ++c) {
|
||||||
|
[[unroll]] for (uint32_t r = 0; r < Br; ++r) {
|
||||||
|
float mvf = data_m[(i * Br + r) * m_stride + (j * Bc + c * cols_per_iter + col_tid)];
|
||||||
|
|
||||||
|
Sf[r][c] += slope[r]*mvf;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
float rowmaxf[Br], Pf[Br][cols_per_thread], rowsumf[Br], eMf[Br], Moldf[Br];
|
||||||
|
[[unroll]] for (uint32_t r = 0; r < Br; ++r) {
|
||||||
|
rowmaxf[r] = Sf[r][0];
|
||||||
|
[[unroll]] for (uint32_t c = 0; c < cols_per_thread; ++c) {
|
||||||
|
rowmaxf[r] = max(rowmaxf[r], Sf[r][c]);
|
||||||
|
}
|
||||||
|
Moldf[r] = Mf[r];
|
||||||
|
|
||||||
|
// M = max(rowmax, Mold)
|
||||||
|
// P = e^(S - M)
|
||||||
|
// eM = e^(Mold - M)
|
||||||
|
Mf[r] = max(rowmaxf[r], Moldf[r]);
|
||||||
|
[[unroll]] for (uint32_t c = 0; c < cols_per_thread; ++c) {
|
||||||
|
Pf[r][c] = exp(Sf[r][c] - Mf[r]);
|
||||||
|
}
|
||||||
|
eMf[r] = exp(Moldf[r] - Mf[r]);
|
||||||
|
|
||||||
|
// Compute sum across row of P
|
||||||
|
rowsumf[r] = 0.0;
|
||||||
|
[[unroll]] for (uint32_t c = 0; c < cols_per_thread; ++c) {
|
||||||
|
rowsumf[r] += Pf[r][c];
|
||||||
|
}
|
||||||
|
|
||||||
|
Lf[r] = eMf[r]*Lf[r] + rowsumf[r];
|
||||||
|
}
|
||||||
|
|
||||||
|
uint32_t v_offset = (iv2*p.nb22 + iv3*p.nb23) / 2;
|
||||||
|
|
||||||
|
float PVf[Br][D_per_thread];
|
||||||
|
[[unroll]] for (uint32_t d = 0; d < D_per_thread; ++d) {
|
||||||
|
[[unroll]] for (uint32_t r = 0; r < Br; ++r) {
|
||||||
|
PVf[r][d] = 0.0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
[[unroll]] for (uint32_t c = 0; c < cols_per_thread; ++c) {
|
||||||
|
[[unroll]] for (uint32_t d = 0; d < D_per_thread; ++d) {
|
||||||
|
float Vf = float(data_v[v_offset + (j * Bc + c * cols_per_iter + col_tid) * v_stride + d * D_split + d_tid]);
|
||||||
|
[[unroll]] for (uint32_t r = 0; r < Br; ++r) {
|
||||||
|
PVf[r][d] += Pf[r][c] * Vf;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
[[unroll]] for (uint32_t d = 0; d < D_per_thread; ++d) {
|
||||||
|
[[unroll]] for (uint32_t r = 0; r < Br; ++r) {
|
||||||
|
Of[r][d] = eMf[r] * Of[r][d] + PVf[r][d];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
barrier();
|
||||||
|
}
|
||||||
|
|
||||||
|
// reduce across threads
|
||||||
|
|
||||||
|
[[unroll]] for (uint32_t r = 0; r < Br; ++r) {
|
||||||
|
float rowmaxf, eMf;
|
||||||
|
|
||||||
|
tmpsh[tid] = Mf[r];
|
||||||
|
// Compute max across the row
|
||||||
|
barrier();
|
||||||
|
[[unroll]] for (int s = int(gl_WorkGroupSize.x) / 2; s >= D_split; s >>= 1) {
|
||||||
|
if (tid < s) {
|
||||||
|
tmpsh[tid] = max(tmpsh[tid], tmpsh[tid + s]);
|
||||||
|
}
|
||||||
|
barrier();
|
||||||
|
}
|
||||||
|
rowmaxf = tmpsh[d_tid];
|
||||||
|
barrier();
|
||||||
|
|
||||||
|
float Moldf = Mf[r];
|
||||||
|
|
||||||
|
// M = max(rowmax, Mold)
|
||||||
|
// eM = e^(Mold - M)
|
||||||
|
Mf[r] = max(rowmaxf, Moldf);
|
||||||
|
eMf = exp(Moldf - Mf[r]);
|
||||||
|
|
||||||
|
Lf[r] = eMf*Lf[r];
|
||||||
|
|
||||||
|
tmpsh[tid] = Lf[r];
|
||||||
|
|
||||||
|
// Compute sum across the row
|
||||||
|
barrier();
|
||||||
|
[[unroll]] for (int s = int(gl_WorkGroupSize.x) / 2; s >= D_split; s >>= 1) {
|
||||||
|
if (tid < s) {
|
||||||
|
tmpsh[tid] = tmpsh[tid] + tmpsh[tid + s];
|
||||||
|
}
|
||||||
|
barrier();
|
||||||
|
}
|
||||||
|
Lf[r] = tmpsh[d_tid];
|
||||||
|
barrier();
|
||||||
|
|
||||||
|
[[unroll]] for (uint32_t d = 0; d < D_per_thread; ++d) {
|
||||||
|
|
||||||
|
Of[r][d] = eMf * Of[r][d];
|
||||||
|
tmpsh[tid] = Of[r][d];
|
||||||
|
|
||||||
|
barrier();
|
||||||
|
[[unroll]] for (int s = int(gl_WorkGroupSize.x) / 2; s >= D_split; s >>= 1) {
|
||||||
|
if (tid < s) {
|
||||||
|
Of[r][d] += tmpsh[tid + s];
|
||||||
|
tmpsh[tid] = Of[r][d];
|
||||||
|
}
|
||||||
|
barrier();
|
||||||
|
}
|
||||||
|
Of[r][d] = tmpsh[d_tid];
|
||||||
|
barrier();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// If there is split_k, then the split_k resolve shader does the final
|
||||||
|
// division by L. Store the intermediate O value and per-row m and L values.
|
||||||
|
if (p.k_num > 1) {
|
||||||
|
uint32_t o_offset = D * p.ne1 * split_k_index;
|
||||||
|
|
||||||
|
[[unroll]] for (uint32_t r = 0; r < Br; ++r) {
|
||||||
|
if (r < N) {
|
||||||
|
for (uint32_t d = 0; d < D_per_thread; ++d) {
|
||||||
|
perElemOpGqaStore(r, d * D_split + d_tid, Of[r][d], o_offset, iq2, N);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
o_offset = D * p.ne1 * p.k_num + p.ne1 * split_k_index * 2;
|
||||||
|
[[unroll]] for (uint32_t r = 0; r < Br; ++r) {
|
||||||
|
if (r < N) {
|
||||||
|
perElemOpStoreCol0(r, 0u, ACC_TYPE(Lf[r]), o_offset, iq2, N);
|
||||||
|
perElemOpStoreCol0(r, 0u, ACC_TYPE(Mf[r]), o_offset + p.ne1, iq2, N);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
float Lfrcp[Br];
|
||||||
|
[[unroll]] for (uint32_t r = 0; r < Br; ++r) {
|
||||||
|
Lfrcp[r] = 1.0 / Lf[r];
|
||||||
|
}
|
||||||
|
|
||||||
|
[[unroll]] for (uint32_t d = 0; d < D_per_thread; ++d) {
|
||||||
|
[[unroll]] for (uint32_t r = 0; r < Br; ++r) {
|
||||||
|
Of[r][d] *= Lfrcp[r];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
uint32_t o_offset = iq3*p.ne2*p.ne1;
|
||||||
|
|
||||||
|
if (p.gqa_ratio > 1) {
|
||||||
|
[[unroll]] for (uint32_t r = 0; r < Br; ++r) {
|
||||||
|
if (r < N) {
|
||||||
|
for (uint32_t d = 0; d < D_per_thread; ++d) {
|
||||||
|
perElemOpGqaStore(r, d * D_split + d_tid, Of[r][d], o_offset, iq2, N);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
[[unroll]] for (uint32_t r = 0; r < Br; ++r) {
|
||||||
|
if (i * Br + r < N) {
|
||||||
|
for (uint32_t d = 0; d < D_per_thread; ++d) {
|
||||||
|
data_o[o_offset + iq2 * D + (i * Br + r) * p.ne1 * D + d * D_split + d_tid] = D_TYPE(Of[r][d]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -421,7 +421,6 @@ void process_shaders() {
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
#if defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT)
|
|
||||||
// flash attention
|
// flash attention
|
||||||
for (const auto& f16acc : {false, true}) {
|
for (const auto& f16acc : {false, true}) {
|
||||||
std::string acctype = f16acc ? "float16_t" : "float";
|
std::string acctype = f16acc ? "float16_t" : "float";
|
||||||
|
|
@ -432,6 +431,7 @@ void process_shaders() {
|
||||||
}
|
}
|
||||||
if (tname == "bf16") continue;
|
if (tname == "bf16") continue;
|
||||||
|
|
||||||
|
#if defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT)
|
||||||
if (tname == "f16") {
|
if (tname == "f16") {
|
||||||
string_to_spv("flash_attn_f32_f16_" + tname, "flash_attn_cm2.comp",
|
string_to_spv("flash_attn_f32_f16_" + tname, "flash_attn_cm2.comp",
|
||||||
merge_maps(base_dict, {{"Q_TYPE", "float"}, {"D_TYPE", "float"}, {"ACC_TYPE", acctype}}), true, false, true, f16acc);
|
merge_maps(base_dict, {{"Q_TYPE", "float"}, {"D_TYPE", "float"}, {"ACC_TYPE", acctype}}), true, false, true, f16acc);
|
||||||
|
|
@ -440,9 +440,13 @@ void process_shaders() {
|
||||||
string_to_spv("flash_attn_f32_f16_" + tname, "flash_attn_cm2.comp",
|
string_to_spv("flash_attn_f32_f16_" + tname, "flash_attn_cm2.comp",
|
||||||
merge_maps(base_dict, {{data_a_key, "1"}, {"Q_TYPE", "float"}, {"D_TYPE", "float"}, {"ACC_TYPE", acctype}, {"DEQUANTFUNC", "dequantFunc"+to_uppercase(tname) }, {"BLOCK_SIZE", "QUANT_K_"+to_uppercase(tname) }}), true, false, true, f16acc);
|
merge_maps(base_dict, {{data_a_key, "1"}, {"Q_TYPE", "float"}, {"D_TYPE", "float"}, {"ACC_TYPE", acctype}, {"DEQUANTFUNC", "dequantFunc"+to_uppercase(tname) }, {"BLOCK_SIZE", "QUANT_K_"+to_uppercase(tname) }}), true, false, true, f16acc);
|
||||||
}
|
}
|
||||||
|
#endif
|
||||||
|
if (tname == "f16") {
|
||||||
|
string_to_spv("flash_attn_f32_f16_" + tname, "flash_attn.comp",
|
||||||
|
merge_maps(base_dict, {{"Q_TYPE", "float"}, {"D_TYPE", "float"}, {"ACC_TYPE", acctype}}), true, false, false, f16acc);
|
||||||
|
} // quants not supported yet
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endif
|
|
||||||
|
|
||||||
for (const auto& tname : type_names) {
|
for (const auto& tname : type_names) {
|
||||||
// mul mat vec
|
// mul mat vec
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue