diff --git a/common/arg.cpp b/common/arg.cpp index 6b8df6667..bf2b62460 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -497,13 +497,15 @@ void common_models_handler_apply(common_models_handler & handler, common_params } // handle hf_plan tasks - auto add_tasks = [&opts, &tasks](const hf_cache::hf_files & model_files, common_params_model & model) { + auto add_tasks = [&opts, &tasks](const hf_cache::hf_files & model_files, + const hf_cache::hf_file & primary, + common_params_model & model) { for (size_t i = 0; i < model_files.size(); ++i) { auto & model_file = model_files[i]; - bool is_first = (i == 0); - tasks.emplace_back(model_file, opts, [&, is_first]() { - if (is_first) { - // only use first part as model path + bool is_primary = (model_file.path == primary.path); + tasks.emplace_back(model_file, opts, [&, is_primary]() { + if (is_primary) { + // the primary file is the first split (00001-of), use it as model path model.path = hf_cache::finalize_file(model_file); } else { hf_cache::finalize_file(model_file); @@ -512,7 +514,7 @@ void common_models_handler_apply(common_models_handler & handler, common_params } }; if (!plan.model_files.empty()) { - add_tasks(plan.model_files, params.model); + add_tasks(plan.model_files, plan.primary, params.model); } if (!plan.mmproj.local_path.empty()) { tasks.emplace_back(plan.mmproj, opts, [&]() { @@ -540,12 +542,12 @@ void common_models_handler_apply(common_models_handler & handler, common_params // handle plan_spec (e.g. --spec-draft-hf) if (!plan_spec.model_files.empty()) { - add_tasks(plan_spec.model_files, params.speculative.draft.mparams); + add_tasks(plan_spec.model_files, plan_spec.primary, params.speculative.draft.mparams); } // handle vocoder plan (e.g. --hf-repo-v) if (!plan_voc.model_files.empty()) { - add_tasks(plan_voc.model_files, params.vocoder.model); + add_tasks(plan_voc.model_files, plan_voc.primary, params.vocoder.model); } // run all tasks in parallel diff --git a/common/http.h b/common/http.h index d3daccd6b..e88bc6a5e 100644 --- a/common/http.h +++ b/common/http.h @@ -11,6 +11,11 @@ struct common_http_url { std::string path; }; +// bracket an IPv6 literal host for a URL authority (RFC 3986) +static std::string common_http_format_host(const std::string & host) { + return host.find(':') != std::string::npos ? "[" + host + "]" : host; +} + static common_http_url common_http_parse_url(const std::string & url) { common_http_url parts; auto scheme_end = url.find("://"); @@ -49,11 +54,28 @@ static common_http_url common_http_parse_url(const std::string & url) { parts.path = "/"; } - auto colon_pos = parts.host.find(':'); + // split the authority into host and optional port, a bracketed IPv6 literal keeps its inner colons (RFC 3986) + std::string port_str; + if (!parts.host.empty() && parts.host.front() == '[') { + auto close = parts.host.find(']'); + if (close == std::string::npos) { + throw std::runtime_error("invalid IPv6 URL authority: " + parts.host); + } + auto after = parts.host.substr(close + 1); + if (!after.empty() && after.front() == ':') { + port_str = after.substr(1); + } + parts.host = parts.host.substr(1, close - 1); + } else { + auto colon_pos = parts.host.find(':'); + if (colon_pos != std::string::npos) { + port_str = parts.host.substr(colon_pos + 1); + parts.host = parts.host.substr(0, colon_pos); + } + } - if (colon_pos != std::string::npos) { - parts.port = std::stoi(parts.host.substr(colon_pos + 1)); - parts.host = parts.host.substr(0, colon_pos); + if (!port_str.empty()) { + parts.port = std::stoi(port_str); } else if (parts.scheme == "http") { parts.port = 80; } else if (parts.scheme == "https") { @@ -83,7 +105,7 @@ static std::pair common_http_client(const std: } #endif - httplib::Client cli(parts.scheme + "://" + parts.host + ":" + std::to_string(parts.port)); + httplib::Client cli(parts.scheme + "://" + common_http_format_host(parts.host) + ":" + std::to_string(parts.port)); if (!parts.user.empty()) { cli.set_basic_auth(parts.user, parts.password); @@ -95,5 +117,5 @@ static std::pair common_http_client(const std: } static std::string common_http_show_masked_url(const common_http_url & parts) { - return parts.scheme + "://" + (parts.user.empty() ? "" : "****:****@") + parts.host + parts.path; + return parts.scheme + "://" + (parts.user.empty() ? "" : "****:****@") + common_http_format_host(parts.host) + parts.path; } diff --git a/ggml/src/ggml-common.h b/ggml/src/ggml-common.h index f05683b44..a51b37dcd 100644 --- a/ggml/src/ggml-common.h +++ b/ggml/src/ggml-common.h @@ -1111,11 +1111,12 @@ GGML_TABLE_BEGIN(int8_t, kvalues_iq4nl, 16) -127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113, GGML_TABLE_END() -// e2m1 values (doubled) +// e2m1 values (doubled), shared by MXFP4 and NVFP4 // ref: https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf -GGML_TABLE_BEGIN(int8_t, kvalues_mxfp4, 16) +GGML_TABLE_BEGIN(int8_t, kvalues_fp4, 16) 0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12, GGML_TABLE_END() +#define kvalues_mxfp4 kvalues_fp4 #define NGRID_IQ1S 2048 #define IQ1S_DELTA 0.125f diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index 11e721733..167ff47f5 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -72,7 +72,6 @@ #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K #elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64) // quants.c -#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 // repack.cpp #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 diff --git a/ggml/src/ggml-cpu/arch/x86/quants.c b/ggml/src/ggml-cpu/arch/x86/quants.c index 62873639f..d14e4983c 100644 --- a/ggml/src/ggml-cpu/arch/x86/quants.c +++ b/ggml/src/ggml-cpu/arch/x86/quants.c @@ -935,7 +935,7 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo #if defined __AVX2__ - const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_mxfp4); + const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_fp4); const __m128i m4b = _mm_set1_epi8(0x0f); const __m256i mone = _mm256_set1_epi16(1); @@ -964,7 +964,7 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo sumf = hsum_float_8(_mm256_add_ps(accum1, accum2)); #elif defined __AVX__ - const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_mxfp4); + const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_fp4); const __m128i m4b = _mm_set1_epi8(0x0f); __m256 accum = _mm256_setzero_ps(); @@ -994,14 +994,152 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo int sumi1 = 0; int sumi2 = 0; for (int j = 0; j < QK_MXFP4/2; ++j) { - sumi1 += y[ib].qs[j + 0] * kvalues_mxfp4[x[ib].qs[j] & 0xf]; - sumi2 += y[ib].qs[j + QK_MXFP4/2] * kvalues_mxfp4[x[ib].qs[j] >> 4]; + sumi1 += y[ib].qs[j + 0] * kvalues_fp4[x[ib].qs[j] & 0xf]; + sumi2 += y[ib].qs[j + QK_MXFP4/2] * kvalues_fp4[x[ib].qs[j] >> 4]; } sumf += d * (sumi1 + sumi2); } *s = sumf; } +void ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + assert(n % QK_NVFP4 == 0); + + const block_nvfp4 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + + const int nb = n / QK_NVFP4; + int ib = 0; + float sumf = 0; + +#if defined(__AVX2__) + + const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_fp4); + const __m128i m4b = _mm_set1_epi8(0x0f); + const __m256i mone = _mm256_set1_epi16(1); + + __m256 accum = _mm256_setzero_ps(); + for(; ib < nb; ib++){ + + const __m128i q4bits_01 = _mm_loadu_si128((const __m128i *)(x[ib].qs + 0)); + const __m128i q4bits_23 = _mm_loadu_si128((const __m128i *)(x[ib].qs + 16)); + + const __m256i q8_01 = _mm256_loadu_si256((const __m256i *)y[2*ib + 0].qs); + const __m256i q8_23 = _mm256_loadu_si256((const __m256i *)y[2*ib + 1].qs); + + const __m128i q4_01_lo = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_01, m4b)); + const __m128i q4_01_hi = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_01, 4), m4b)); + const __m128i q4_23_lo = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_23, m4b)); + const __m128i q4_23_hi = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_23, 4), m4b)); + + //reordering + const __m256i q4_01 = MM256_SET_M128I(_mm_unpackhi_epi64(q4_01_lo,q4_01_hi), _mm_unpacklo_epi64(q4_01_lo,q4_01_hi)); + const __m256i q4_23 = MM256_SET_M128I(_mm_unpackhi_epi64(q4_23_lo,q4_23_hi),_mm_unpacklo_epi64(q4_23_lo,q4_23_hi)); + + const __m256i p01 = mul_add_epi8(q4_01,q8_01); + const __m256i p_1 = _mm256_madd_epi16(p01, mone); + + const __m256i p23 = mul_add_epi8(q4_23,q8_23); + const __m256i p_2 = _mm256_madd_epi16(p23, mone); + + const float dy0 = GGML_CPU_FP16_TO_FP32(y[2*ib].d); + const float dy1 = GGML_CPU_FP16_TO_FP32(y[2*ib+1].d); + + const float s0 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[0]) * dy0; + const float s1 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[1]) * dy0; + const float s2 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[2]) * dy1; + const float s3 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[3]) * dy1; + + const __m256 scales01 = _mm256_set_m128(_mm_set1_ps(s1), _mm_set1_ps(s0)); + const __m256 scales23 = _mm256_set_m128(_mm_set1_ps(s3), _mm_set1_ps(s2)); + + accum = _mm256_fmadd_ps(scales01, _mm256_cvtepi32_ps(p_1), accum); + accum = _mm256_fmadd_ps(scales23, _mm256_cvtepi32_ps(p_2), accum); + } + sumf = hsum_float_8(accum); + +#elif defined(__AVX__) + + const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_fp4); + const __m128i m4b = _mm_set1_epi8(0x0f); + + __m256 accum = _mm256_setzero_ps(); + for(; ib < nb; ib++){ + + const __m128i q4bits_01 = _mm_loadu_si128((const __m128i *)(x[ib].qs + 0)); + const __m128i q4bits_23 = _mm_loadu_si128((const __m128i *)(x[ib].qs + 16)); + + const __m128i q8_0 = _mm_loadu_si128((const __m128i *)(y[2*ib + 0].qs + 0)); + const __m128i q8_1 = _mm_loadu_si128((const __m128i *)(y[2*ib + 0].qs + 16)); + const __m128i q8_2 = _mm_loadu_si128((const __m128i *)(y[2*ib + 1].qs + 0)); + const __m128i q8_3 = _mm_loadu_si128((const __m128i *)(y[2*ib + 1].qs + 16)); + + const __m128i q4_01_lo = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_01, m4b)); + const __m128i q4_01_hi = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_01, 4), m4b)); + const __m128i q4_23_lo = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_23, m4b)); + const __m128i q4_23_hi = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_23, 4), m4b)); + + const __m128i q4_0 = _mm_unpacklo_epi64(q4_01_lo, q4_01_hi); + const __m128i q4_1 = _mm_unpackhi_epi64(q4_01_lo, q4_01_hi); + const __m128i q4_2 = _mm_unpacklo_epi64(q4_23_lo, q4_23_hi); + const __m128i q4_3 = _mm_unpackhi_epi64(q4_23_lo, q4_23_hi); + + const __m128i p0_i32 = mul_sum_i8_pairs(q4_0, q8_0); + const __m128i p1_i32 = mul_sum_i8_pairs(q4_1, q8_1); + const __m128i p2_i32 = mul_sum_i8_pairs(q4_2, q8_2); + const __m128i p3_i32 = mul_sum_i8_pairs(q4_3, q8_3); + + const __m128 p0 = _mm_cvtepi32_ps(p0_i32); + const __m128 p1 = _mm_cvtepi32_ps(p1_i32); + const __m128 p2 = _mm_cvtepi32_ps(p2_i32); + const __m128 p3 = _mm_cvtepi32_ps(p3_i32); + + const __m256 p01 = _mm256_set_m128(p1, p0); + const __m256 p23 = _mm256_set_m128(p3, p2); + + const float dy0 = GGML_CPU_FP16_TO_FP32(y[2*ib].d); + const float dy1 = GGML_CPU_FP16_TO_FP32(y[2*ib+1].d); + + const float s0 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[0]) * dy0; + const float s1 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[1]) * dy0; + const float s2 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[2]) * dy1; + const float s3 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[3]) * dy1; + + const __m256 scales01 = _mm256_set_m128(_mm_set1_ps(s1), _mm_set1_ps(s0)); + const __m256 scales23 = _mm256_set_m128(_mm_set1_ps(s3), _mm_set1_ps(s2)); + + accum = _mm256_add_ps(accum, _mm256_mul_ps(p01, scales01)); + accum = _mm256_add_ps(accum, _mm256_mul_ps(p23, scales23)); + } + sumf = hsum_float_8(accum); + +#endif + + for (;ib < nb; ++ib) { + for (int s_idx = 0; s_idx < 4; ++s_idx) { + const float d = GGML_CPU_UE4M3_TO_FP32(x[ib].d[s_idx]); + const int q8_block = s_idx / 2; + const int q8_off = (s_idx % 2) * QK_NVFP4_SUB; + const float dy = GGML_CPU_FP16_TO_FP32(y[2*ib + q8_block].d); + + int sumi_lo = 0, sumi_hi = 0; + for (int j = 0; j < QK_NVFP4_SUB/2; ++j) { + const uint8_t qv = x[ib].qs[s_idx*(QK_NVFP4_SUB/2) + j]; + sumi_lo += y[2*ib + q8_block].qs[q8_off + j + 0] * kvalues_fp4[qv & 0xf]; + sumi_hi += y[2*ib + q8_block].qs[q8_off + j + QK_NVFP4_SUB/2] * kvalues_fp4[qv >> 4]; + } + + sumf += dy * d * (sumi_lo + sumi_hi); + } + } + *s = sumf; +} + void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK8_0; const int nb = n / qk; diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 70e9d8660..0d110b87d 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -83,6 +83,9 @@ float ggml_table_f32_f16[1 << 16]; // precomputed f32 table for e8m0 half (1 KB) (simd-mappings.h) float ggml_table_f32_e8m0_half[1 << 8]; +// precomputed f32 table for ue4m3 (1 KB) (simd-mappings.h) +float ggml_table_f32_ue4m3[1 << 8]; + #if defined(__ARM_ARCH) struct ggml_arm_arch_features_type { int sve_cnt; @@ -4647,6 +4650,11 @@ void ggml_cpu_init(void) { ggml_table_f32_e8m0_half[i] = GGML_E8M0_TO_FP32_HALF(i); } + // initialize UE4M3 table (256 entries) + for (int i = 0; i < (1 << 8); ++i) { + ggml_table_f32_ue4m3[i] = ggml_ue4m3_to_fp32(i); + } + const uint64_t t_end = ggml_time_us(); UNUSED(t_end); GGML_PRINT_DEBUG("%s: GELU, Quick GELU, SILU and EXP tables initialized in %f ms\n", __func__, (t_end - t_start)/1000.0); diff --git a/ggml/src/ggml-cpu/simd-mappings.h b/ggml/src/ggml-cpu/simd-mappings.h index 62e687201..be50c25c0 100644 --- a/ggml/src/ggml-cpu/simd-mappings.h +++ b/ggml/src/ggml-cpu/simd-mappings.h @@ -120,6 +120,10 @@ extern float ggml_table_f32_f16[1 << 16]; // defined in ggml-cpu.c, initialized in ggml_cpu_init() extern float ggml_table_f32_e8m0_half[1 << 8]; +// precomputed f32 table for ue4m3 (1 KB) +// defined in ggml-cpu.c, initialized in ggml_cpu_init() +extern float ggml_table_f32_ue4m3[1 << 8]; + // Use lookup table for E8M0 on x86 (faster than bit manipulation) #if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) #define GGML_CPU_E8M0_TO_FP32_HALF(x) ggml_table_f32_e8m0_half[(uint8_t)(x)] @@ -127,6 +131,13 @@ extern float ggml_table_f32_e8m0_half[1 << 8]; #define GGML_CPU_E8M0_TO_FP32_HALF(x) GGML_E8M0_TO_FP32_HALF(x) #endif +// Use lookup table for UE4M3 on x86 (faster than bit manipulation) +#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) +#define GGML_CPU_UE4M3_TO_FP32(x) ggml_table_f32_ue4m3[(uint8_t)(x)] +#else +#define GGML_CPU_UE4M3_TO_FP32(x) ggml_ue4m3_to_fp32(x) +#endif + // On ARM NEON, it's quicker to directly convert x -> x instead of calling into ggml_lookup_fp16_to_fp32, // so we define GGML_CPU_FP16_TO_FP32 and GGML_CPU_FP32_TO_FP16 elsewhere for NEON. // This is also true for POWER9. diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuh index 8dfa51ad1..e67cc7fdf 100644 --- a/ggml/src/ggml-cuda/fattn-common.cuh +++ b/ggml/src/ggml-cuda/fattn-common.cuh @@ -664,7 +664,10 @@ constexpr __device__ dequantize_V_t get_dequantize_V() { template __launch_bounds__(FATTN_KQ_STRIDE/2, 1) static __global__ void flash_attn_mask_to_KV_max( - const half2 * __restrict__ mask, int * __restrict__ KV_max, const int ne30, const int s31, const int s33) { + const half2 * mask_ptr, int * KV_max_ptr, const int ne30, const int64_t s31, const int64_t s33) { + const half2 * GGML_CUDA_RESTRICT mask = mask_ptr; + int * GGML_CUDA_RESTRICT KV_max = KV_max_ptr; + const int ne31 = gridDim.x; const int tid = threadIdx.x; const int sequence = blockIdx.y; @@ -1089,8 +1092,8 @@ void launch_fattn( // Only worth the overhead if there is at lease one FATTN_KQ_STRIDE x FATTN_KQ_STRIDE square to be skipped or // multiple sequences of possibly different lengths. if (mask && K->ne[1] % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1)) { - const int s31 = mask->nb[1] / sizeof(half2); - const int s33 = mask->nb[3] / sizeof(half2); + const int64_t s31 = mask->nb[1] / sizeof(half2); + const int64_t s33 = mask->nb[3] / sizeof(half2); const dim3 blocks_num_KV_max(ntiles_x, Q->ne[3], 1); const dim3 block_dim_KV_max(FATTN_KQ_STRIDE/2, 1, 1); @@ -1099,8 +1102,9 @@ void launch_fattn( const int iter_k = K->ne[1] / FATTN_KQ_STRIDE; KV_max.alloc(ne_KV_max); - flash_attn_mask_to_KV_max<<>> - ((const half2 *) mask->data, KV_max.ptr, iter_k, s31, s33); + ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks_num_KV_max, block_dim_KV_max, 0, main_stream); + ggml_cuda_kernel_launch(flash_attn_mask_to_KV_max, launch_params, + (const half2 *) mask->data, KV_max.ptr, iter_k, s31, s33); CUDA_CHECK(cudaGetLastError()); } diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuh index 83478a02c..7f4cfd551 100644 --- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh +++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh @@ -2003,6 +2003,10 @@ DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 112, 64) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 128, 64) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 256, 64) +extern DECL_FATTN_MMA_F16_CASE(512, 512, 4, 2); +extern DECL_FATTN_MMA_F16_CASE(512, 512, 8, 2); +extern DECL_FATTN_MMA_F16_CASE(512, 512, 16, 2); +extern DECL_FATTN_MMA_F16_CASE(512, 512, 32, 2); extern DECL_FATTN_MMA_F16_CASE(512, 512, 2, 4); extern DECL_FATTN_MMA_F16_CASE(512, 512, 4, 4); extern DECL_FATTN_MMA_F16_CASE(512, 512, 8, 4); diff --git a/ggml/src/ggml-cuda/fattn-tile.cuh b/ggml/src/ggml-cuda/fattn-tile.cuh index 0a099810e..3e07a9f7e 100644 --- a/ggml/src/ggml-cuda/fattn-tile.cuh +++ b/ggml/src/ggml-cuda/fattn-tile.cuh @@ -76,6 +76,7 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_nv GGML_CUDA_FATTN_TILE_CONFIG_CASE(320, 256, 16, 256, 2, 64, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 2, 64, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 4, 128, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 8, 256, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 16, 256, 2, 64, 64) @@ -144,6 +145,7 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_nv GGML_CUDA_FATTN_TILE_CONFIG_CASE(320, 256, 16, 256, 2, 32, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 2, 64, 2, 32, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 4, 128, 2, 32, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 8, 256, 2, 32, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 16, 256, 2, 32, 64) @@ -219,6 +221,7 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_am GGML_CUDA_FATTN_TILE_CONFIG_CASE(320, 256, 32, 512, 1, 128, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 2, 64, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 4, 128, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 8, 256, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 16, 256, 2, 64, 64) @@ -296,6 +299,7 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_am GGML_CUDA_FATTN_TILE_CONFIG_CASE(320, 256, 32, 256, 2, 128, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 2, 64, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 4, 128, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 8, 256, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 16, 256, 4, 64, 64) @@ -1308,12 +1312,12 @@ static void launch_fattn_tile_switch_ncols2(ggml_backend_cuda_context & ctx, ggm return; } - if constexpr (DV <= 256) { - if (use_gqa_opt && gqa_ratio % 2 == 0) { - launch_fattn_tile_switch_ncols1(ctx, dst); - return; - } + if (use_gqa_opt && gqa_ratio % 2 == 0) { + launch_fattn_tile_switch_ncols1(ctx, dst); + return; + } + if constexpr (DV <= 256) { launch_fattn_tile_switch_ncols1(ctx, dst); return; } diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index e5ce27d54..9c6584e64 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -99,12 +99,12 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2(ggml_backend_cuda_con return; } - if constexpr (DKQ <= 256) { - if (use_gqa_opt && gqa_ratio > 1) { - ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); - return; - } + if (use_gqa_opt && gqa_ratio > 1) { + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); + return; + } + if constexpr (DKQ <= 256) { ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); } else { GGML_ABORT("fatal error"); diff --git a/ggml/src/ggml-cuda/getrows.cu b/ggml/src/ggml-cuda/getrows.cu index eb157b8ba..0e1570709 100644 --- a/ggml/src/ggml-cuda/getrows.cu +++ b/ggml/src/ggml-cuda/getrows.cu @@ -78,26 +78,29 @@ static __global__ void k_get_rows_float( template static __global__ void k_get_rows_back_float( - const grad_t * __restrict__ grad, const int32_t * __restrict__ rows, dst_t * __restrict__ dst, const int64_t ncols, const int64_t nrows_grad) { + const grad_t * __restrict__ grad, const int32_t * __restrict__ rows, dst_t * __restrict__ dst, + const int64_t ncols, const int64_t nrows_grad, const int64_t nrows_dst) { const int col = blockIdx.x*blockDim.x + threadIdx.x; if (col >= ncols) { return; } - const int dst_row = blockIdx.y*blockDim.y + threadIdx.y; - - float sum = 0.0f; - ggml_cuda_pdl_sync(); - for (int64_t i = 0; i < nrows_grad; ++i) { - if (rows[i] != dst_row) { - continue; - } - sum += grad[i*ncols + col]; - } - dst[dst_row*ncols + col] = sum; + // grid.y is clamped to the CUDA grid limit, so stride over the destination rows + for (int64_t dst_row = blockIdx.y; dst_row < nrows_dst; dst_row += gridDim.y) { + float sum = 0.0f; + + for (int64_t i = 0; i < nrows_grad; ++i) { + if (rows[i] != dst_row) { + continue; + } + sum += grad[i*ncols + col]; + } + + dst[dst_row*ncols + col] = sum; + } } template @@ -302,7 +305,7 @@ void ggml_cuda_op_get_rows_back(ggml_backend_cuda_context & ctx, ggml_tensor * d const dim3 block_dims(CUDA_GET_ROWS_BACK_BLOCK_SIZE, 1, 1); const int block_num_x = (ne00 + CUDA_GET_ROWS_BACK_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BACK_BLOCK_SIZE; - const dim3 block_nums(block_num_x, ne1, 1); + const dim3 block_nums(block_num_x, MIN(ne1, (int64_t)UINT16_MAX), 1); - k_get_rows_back_float<<>>(src0_d, src1_d, dst_d, ne00, ne10); + k_get_rows_back_float<<>>(src0_d, src1_d, dst_d, ne00, ne10, ne1); } diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu index 2e1883af4..0c1a37fd0 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu @@ -8,3 +8,4 @@ DECL_FATTN_MMA_F16_CASE(96, 96, 16, 2); DECL_FATTN_MMA_F16_CASE(112, 112, 16, 2); DECL_FATTN_MMA_F16_CASE(128, 128, 16, 2); DECL_FATTN_MMA_F16_CASE(256, 256, 16, 2); +DECL_FATTN_MMA_F16_CASE(512, 512, 16, 2); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu index 407b6cf4c..d5a4d160a 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu @@ -8,3 +8,4 @@ DECL_FATTN_MMA_F16_CASE(96, 96, 32, 2); DECL_FATTN_MMA_F16_CASE(112, 112, 32, 2); DECL_FATTN_MMA_F16_CASE(128, 128, 32, 2); DECL_FATTN_MMA_F16_CASE(256, 256, 32, 2); +DECL_FATTN_MMA_F16_CASE(512, 512, 32, 2); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu index 5e4668502..d8e7588cf 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu @@ -8,3 +8,4 @@ DECL_FATTN_MMA_F16_CASE(96, 96, 4, 2); DECL_FATTN_MMA_F16_CASE(112, 112, 4, 2); DECL_FATTN_MMA_F16_CASE(128, 128, 4, 2); DECL_FATTN_MMA_F16_CASE(256, 256, 4, 2); +DECL_FATTN_MMA_F16_CASE(512, 512, 4, 2); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu index a6e6f093d..e48e28c4e 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu @@ -8,3 +8,4 @@ DECL_FATTN_MMA_F16_CASE(96, 96, 8, 2); DECL_FATTN_MMA_F16_CASE(112, 112, 8, 2); DECL_FATTN_MMA_F16_CASE(128, 128, 8, 2); DECL_FATTN_MMA_F16_CASE(256, 256, 8, 2); +DECL_FATTN_MMA_F16_CASE(512, 512, 8, 2); diff --git a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py index af05a9eff..614b1566c 100755 --- a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +++ b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py @@ -92,7 +92,7 @@ for ncols in [8, 16, 32, 64]: continue if head_size_kq == 320 and ncols2 != 32: # Mistral Small 4 continue - if head_size_kq == 512 and ncols2 not in (4, 8): # Gemma 4 + if head_size_kq == 512 and ncols2 not in (2, 4, 8): # Gemma 4 (+ MTP) continue if head_size_kq == 576 and ncols2 not in (4, 16, 32): # Deepseek, GLM 4.7 Flash continue diff --git a/ggml/src/ggml-hexagon/htp-opnode.h b/ggml/src/ggml-hexagon/htp-opnode.h index 6fe23b0d6..19a2504c7 100644 --- a/ggml/src/ggml-hexagon/htp-opnode.h +++ b/ggml/src/ggml-hexagon/htp-opnode.h @@ -11,6 +11,7 @@ #include #include "htp-ops.h" #include "htp/matmul-ops.h" +#include "htp/flash-attn-ops.h" struct htp_opnode { ggml_tensor * node = nullptr; @@ -335,7 +336,8 @@ struct htp_opformat { } void format_kernel_params(char * str, size_t max_size, const htp_opnode & node) { if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID || - node.opcode == HTP_OP_MUL_MAT_QKV || node.opcode == HTP_OP_MUL_MAT_FFN) { + node.opcode == HTP_OP_MUL_MAT_QKV || node.opcode == HTP_OP_MUL_MAT_FFN || + node.opcode == HTP_OP_MUL_MAT_ADD) { const auto * kparams = (const struct htp_mm_kernel_params *) node.kernel_params; const char * path = "unknown"; int32_t type = kparams->kernel_type; @@ -350,6 +352,16 @@ struct htp_opformat { path = "hvx-flat"; } snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) { + const auto * kparams = (const struct htp_fa_kernel_params *) node.kernel_params; + const char * path = "unknown"; + int32_t type = kparams->kernel_type; + if (type == HTP_FA_KERNEL_HMX) { + path = kparams->u.hmx.pipeline ? "hmx-pipe" : "hmx-seq"; + } else if (type == HTP_FA_KERNEL_HVX) { + path = "hvx"; + } + snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size); } else { snprintf(str, max_size, "----"); } diff --git a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h index b7fba22a8..4a0ca7885 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +++ b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h @@ -712,7 +712,17 @@ static inline void hmx_matmul_job_init(hmx_matmul_job_t * job, // output : fp16 -> f32p -static void transfer_output_chunk_fp16_to_fp32(float *restrict dst, const __fp16 *restrict vtcm_src, uint32_t start_row, uint32_t n_rows, uint32_t n_cols, uint32_t dst_stride, uint32_t dst_cols) { +static void transfer_output_chunk_fp16_to_fp32( + float *restrict dst, + const float *restrict src2, + const __fp16 *restrict vtcm_src, + uint32_t start_row, + uint32_t n_rows, + uint32_t n_cols, + uint32_t dst_stride, + uint32_t src2_stride, + uint32_t dst_cols +) { assert(n_cols % HTP_MM_HMX_TILE_N_COLS == 0); const size_t tile_row_stride = (n_cols / HTP_MM_HMX_TILE_N_COLS) * HTP_MM_HMX_TILE_N_ELMS; @@ -727,6 +737,7 @@ static void transfer_output_chunk_fp16_to_fp32(float *restrict dst, const __fp16 const size_t r1 = (r_idx0 % HTP_MM_HMX_TILE_N_ROWS) / 2; // index of the row pair within the tile const __fp16 *row_base = vtcm_src + r0 * tile_row_stride; float *output_row_base = dst + r * dst_stride; // global memory row base for row r (and r+1) + const float *src2_row_base = src2 ? (src2 + r * src2_stride) : NULL; #pragma unroll(4) for (size_t c = 0; c < limit_c_aligned; c += HTP_MM_HMX_TILE_N_COLS) { @@ -738,9 +749,20 @@ static void transfer_output_chunk_fp16_to_fp32(float *restrict dst, const __fp16 HVX_Vector *pv_out0 = (HVX_Vector *) (output_row_base + c + 0); HVX_Vector *pv_out1 = (HVX_Vector *) (output_row_base + c + dst_stride); - *pv_out0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp)); + HVX_Vector v_out0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp)); + if (src2_row_base) { + HVX_Vector v_src2_0 = hvx_vmemu(src2_row_base + c + 0); + v_out0 = hvx_vec_add_f32_f32(v_out0, v_src2_0); + } + *pv_out0 = v_out0; + if (r + 1 < n_rows) { - *pv_out1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp)); + HVX_Vector v_out1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp)); + if (src2_row_base) { + HVX_Vector v_src2_1 = hvx_vmemu(src2_row_base + c + src2_stride); + v_out1 = hvx_vec_add_f32_f32(v_out1, v_src2_1); + } + *pv_out1 = v_out1; } } @@ -752,9 +774,20 @@ static void transfer_output_chunk_fp16_to_fp32(float *restrict dst, const __fp16 HVX_Vector v = ((const HVX_Vector *) tile)[r1]; HVX_VectorPair vp = Q6_Wqf32_vmpy_VhfVhf(v, one); - hvx_vec_store_u(output_row_base + c, valid_c * sizeof(float), Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp))); + HVX_Vector v_out0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp)); + if (src2_row_base) { + HVX_Vector v_src2_0 = hvx_vmemu(src2_row_base + c + 0); + v_out0 = hvx_vec_add_f32_f32(v_out0, v_src2_0); + } + hvx_vec_store_u(output_row_base + c, valid_c * sizeof(float), v_out0); + if (r + 1 < n_rows) { - hvx_vec_store_u(output_row_base + c + dst_stride, valid_c * sizeof(float), Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp))); + HVX_Vector v_out1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp)); + if (src2_row_base) { + HVX_Vector v_src2_1 = hvx_vmemu(src2_row_base + c + src2_stride); + v_out1 = hvx_vec_add_f32_f32(v_out1, v_src2_1); + } + hvx_vec_store_u(output_row_base + c + dst_stride, valid_c * sizeof(float), v_out1); } } } @@ -763,11 +796,13 @@ static void transfer_output_chunk_fp16_to_fp32(float *restrict dst, const __fp16 typedef struct { const __fp16 *vtcm_src; float *dst; + const float *src2; uint32_t n_tasks; uint32_t n_tot_chunks; uint32_t n_chunks_per_task; uint32_t n_cols; uint32_t dst_stride; // DDR row stride + uint32_t src2_stride; // DDR row stride for residual uint32_t dst_cols; // Actual output columns struct htp_thread_trace * traces; } output_transfer_task_state_t; diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h index 52351b103..328a83118 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h @@ -256,7 +256,7 @@ static inline void quantize_f16_f16_flat_kernel( // Dot kernels that consume flat (non-tiled) activations -static void flat_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows) { +static void flat_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -312,10 +312,14 @@ static void flat_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const v v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); } - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } } -static void flat_vec_dot_q4_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows) { +static void flat_vec_dot_q4_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y0_q = vy0; const uint8_t * restrict y1_q = vy1; @@ -397,11 +401,19 @@ static void flat_vec_dot_q4_0_32x2(const uint32_t n, float * restrict s0, float v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); } - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } } -static void flat_vec_dot_q4_1_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows) { +static void flat_vec_dot_q4_1_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -464,10 +476,14 @@ static void flat_vec_dot_q4_1_32x1(const uint32_t n, float * restrict s, const v v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); } - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } } -static void flat_vec_dot_q4_1_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows) { +static void flat_vec_dot_q4_1_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y0_q = vy0; const uint8_t * restrict y1_q = vy1; @@ -561,11 +577,19 @@ static void flat_vec_dot_q4_1_32x2(const uint32_t n, float * restrict s0, float v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); } - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } } -static void flat_vec_dot_q8_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows) { +static void flat_vec_dot_q8_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -620,10 +644,14 @@ static void flat_vec_dot_q8_0_32x1(const uint32_t n, float * restrict s, const v v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); } - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } } -static void flat_vec_dot_q8_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows) { +static void flat_vec_dot_q8_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y0_q = vy0; const uint8_t * restrict y1_q = vy1; @@ -704,11 +732,19 @@ static void flat_vec_dot_q8_0_32x2(const uint32_t n, float * restrict s0, float v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); } - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } } -static void flat_vec_dot_iq4nl_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows) { +static void flat_vec_dot_iq4nl_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -765,10 +801,14 @@ static void flat_vec_dot_iq4nl_32x1(const uint32_t n, float * restrict s, const v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); } - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } } -static void flat_vec_dot_iq4nl_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows) { +static void flat_vec_dot_iq4nl_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y0_q = vy0; const uint8_t * restrict y1_q = vy1; @@ -851,11 +891,19 @@ static void flat_vec_dot_iq4nl_32x2(const uint32_t n, float * restrict s0, float v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); } - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } } -static void flat_vec_dot_mxfp4_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows) { +static void flat_vec_dot_mxfp4_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -921,10 +969,14 @@ static void flat_vec_dot_mxfp4_32x1(const uint32_t n, float * restrict s, const v_sum_float = hvx_vec_mul_f32_f32(v_sum_float, hvx_vec_splat_f32(0.5f)); - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } } -static void flat_vec_dot_mxfp4_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows) { +static void flat_vec_dot_mxfp4_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y0_q = vy0; const uint8_t * restrict y1_q = vy1; @@ -1019,6 +1071,441 @@ static void flat_vec_dot_mxfp4_32x2(const uint32_t n, float * restrict s0, float v_sum_float_c0 = hvx_vec_mul_f32_f32(v_sum_float_c0, hvx_vec_splat_f32(0.5f)); v_sum_float_c1 = hvx_vec_mul_f32_f32(v_sum_float_c1, hvx_vec_splat_f32(0.5f)); - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } } + +#if __HVX_ARCH__ < 79 +#define HVX_OP_ADD_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(a, b)) +#define HVX_OP_MUL_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a, b)) +#else +#define HVX_OP_ADD_F32(a, b) Q6_Vsf_vadd_VsfVsf(a, b) +#define HVX_OP_MUL_F32(a, b) Q6_Vsf_vmpy_VsfVsf(a, b) +#endif + +static inline void vec_dot_f32_f32_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) { + const HVX_Vector * restrict x = (const HVX_Vector *) vx; + const HVX_Vector * restrict y = (const HVX_Vector *) vy; + + uint32_t nvec = n / VLEN_FP32; // num full fp32 hvx vectors + uint32_t nloe = n % VLEN_FP32; // leftover elements + + HVX_Vector rsum = Q6_V_vzero(); + + uint32_t i = 0; + + #pragma unroll(4) + for (i = 0; i < nvec; i++) { + HVX_Vector prod = HVX_OP_MUL_F32(x[i], y[i]); + rsum = HVX_OP_ADD_F32(rsum, prod); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + HVX_Vector x_sf = Q6_V_vand_QV(bmask, x[i]); + HVX_Vector y_sf = Q6_V_vand_QV(bmask, y[i]); + HVX_Vector prod = HVX_OP_MUL_F32(x_sf, y_sf); + rsum = HVX_OP_ADD_F32(rsum, prod); + } + + *s = hvx_vec_get_f32(hvx_vec_reduce_sum_f32(rsum)); +} + +static inline void vec_dot_f32_f32_aa_2x1(const uint32_t n, float * restrict s0, + const void * restrict vx0, const void * restrict vx1, + const void * restrict vy0) { + const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; + const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; + const HVX_Vector * restrict y = (const HVX_Vector *) vy0; + + uint32_t nvec = n / VLEN_FP32; + uint32_t nloe = n % VLEN_FP32; + + HVX_Vector rsum0 = Q6_V_vzero(); + HVX_Vector rsum1 = Q6_V_vzero(); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_Vector y_sf = y[i]; + HVX_Vector prod0 = HVX_OP_MUL_F32(x0[i], y_sf); + HVX_Vector prod1 = HVX_OP_MUL_F32(x1[i], y_sf); + rsum0 = HVX_OP_ADD_F32(rsum0, prod0); + rsum1 = HVX_OP_ADD_F32(rsum1, prod1); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + HVX_Vector y_sf = Q6_V_vand_QV(bmask, y[i]); + HVX_Vector x0_sf = Q6_V_vand_QV(bmask, x0[i]); + HVX_Vector x1_sf = Q6_V_vand_QV(bmask, x1[i]); + HVX_Vector prod0 = HVX_OP_MUL_F32(x0_sf, y_sf); + HVX_Vector prod1 = HVX_OP_MUL_F32(x1_sf, y_sf); + rsum0 = HVX_OP_ADD_F32(rsum0, prod0); + rsum1 = HVX_OP_ADD_F32(rsum1, prod1); + } + + HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(rsum0, rsum1); + hvx_vec_store_u(s0, 8, rsum); +} + +static inline void vec_dot_f32_f32_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1, + const void * restrict vx0, const void * restrict vx1, + const void * restrict vy0, const void * restrict vy1) { + const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; + const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; + const HVX_Vector * restrict y0 = (const HVX_Vector *) vy0; + const HVX_Vector * restrict y1 = (const HVX_Vector *) vy1; + + uint32_t nvec = n / VLEN_FP32; + uint32_t nloe = n % VLEN_FP32; + + HVX_Vector r0_c0_sum = Q6_V_vzero(); + HVX_Vector r0_c1_sum = Q6_V_vzero(); + HVX_Vector r1_c0_sum = Q6_V_vzero(); + HVX_Vector r1_c1_sum = Q6_V_vzero(); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_Vector r0_sf = x0[i]; + HVX_Vector r1_sf = x1[i]; + HVX_Vector c0_sf = y0[i]; + HVX_Vector c1_sf = y1[i]; + + r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(r0_sf, c0_sf)); + r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(r0_sf, c1_sf)); + r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(r1_sf, c0_sf)); + r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(r1_sf, c1_sf)); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + + HVX_Vector r0_sf = Q6_V_vand_QV(bmask, x0[i]); + HVX_Vector r1_sf = Q6_V_vand_QV(bmask, x1[i]); + HVX_Vector c0_sf = Q6_V_vand_QV(bmask, y0[i]); + HVX_Vector c1_sf = Q6_V_vand_QV(bmask, y1[i]); + + r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(r0_sf, c0_sf)); + r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(r0_sf, c1_sf)); + r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(r1_sf, c0_sf)); + r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(r1_sf, c1_sf)); + } + + // Reduce and store results + HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum); + HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum); + + hvx_vec_store_u(s0, 8, r0_r1_c0_sum); + hvx_vec_store_u(s1, 8, r0_r1_c1_sum); +} + +static inline void vec_dot_f32_f32_uu_1x1(const uint32_t n, float * restrict s, const void * restrict x, const void * restrict y) { + const HVX_UVector * restrict vx = (const HVX_UVector * restrict) x; + const HVX_UVector * restrict vy = (const HVX_UVector * restrict) y; + + uint32_t nvec = n / VLEN_FP32; // num full fp32 hvx vectors + uint32_t nloe = n % VLEN_FP32; // leftover elements + + HVX_Vector rsum = Q6_V_vzero(); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_Vector x_sf = vx[i]; + HVX_Vector y_sf = vy[i]; + + rsum = HVX_OP_ADD_F32(rsum, HVX_OP_MUL_F32(x_sf, y_sf)); + } + + if (nloe) { + HVX_Vector x_sf = vx[i]; + HVX_Vector y_sf = vy[i]; + + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + x_sf = Q6_V_vand_QV(bmask, x_sf); + y_sf = Q6_V_vand_QV(bmask, y_sf); + + rsum = HVX_OP_ADD_F32(rsum, HVX_OP_MUL_F32(x_sf, y_sf)); + } + + rsum = hvx_vec_reduce_sum_f32(rsum); + hvx_vec_store_u(&s[0], 4, rsum); +} + +#undef HVX_OP_ADD_F32 +#undef HVX_OP_MUL_F32 + +static inline void vec_dot_f16_f16_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) { + const HVX_Vector * restrict x = (const HVX_Vector *) vx; + const HVX_Vector * restrict y = (const HVX_Vector *) vy; + + uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors + uint32_t nloe = n % VLEN_FP16; // leftover elements + + HVX_VectorPair rsum_p = Q6_W_vzero(); + + uint32_t i = 0; + + #pragma unroll(4) + for (i = 0; i < nvec; i++) { + rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x[i], y[i]); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); + HVX_Vector x_hf = Q6_V_vand_QV(bmask, x[i]); + HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]); + rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x_hf, y_hf); + } + + HVX_Vector rsum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum_p), Q6_V_hi_W(rsum_p))); + hvx_vec_store_u(s, 4, hvx_vec_reduce_sum_f32(rsum)); +} + +static inline void vec_dot_f16_f16_aa_2x1(const uint32_t n, float * restrict s0, + const void * restrict vx0, const void * restrict vx1, + const void * restrict vy0) { + const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; + const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; + const HVX_Vector * restrict y = (const HVX_Vector *) vy0; + + uint32_t nvec = n / VLEN_FP16; + uint32_t nloe = n % VLEN_FP16; + + HVX_VectorPair rsum0_p = Q6_W_vzero(); + HVX_VectorPair rsum1_p = Q6_W_vzero(); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_Vector y_hf = y[i]; + rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0[i], y_hf); + rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1[i], y_hf); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); + HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]); + HVX_Vector x0_hf = Q6_V_vand_QV(bmask, x0[i]); + HVX_Vector x1_hf = Q6_V_vand_QV(bmask, x1[i]); + rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0_hf, y_hf); + rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1_hf, y_hf); + } + + HVX_Vector rsum0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum0_p), Q6_V_hi_W(rsum0_p))); + HVX_Vector rsum1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum1_p), Q6_V_hi_W(rsum1_p))); + HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(rsum0, rsum1); + hvx_vec_store_u(s0, 8, rsum); +} + +static inline void vec_dot_f16_f16_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1, + const void * restrict vx0, const void * restrict vx1, + const void * restrict vy0, const void * restrict vy1) { + const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; + const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; + const HVX_Vector * restrict y0 = (const HVX_Vector *) vy0; + const HVX_Vector * restrict y1 = (const HVX_Vector *) vy1; + + uint32_t nvec = n / VLEN_FP16; + uint32_t nloe = n % VLEN_FP16; + + // Row sums (sf) - 4 accumulators for 2x2 tile + HVX_VectorPair r0_c0_sum_p = Q6_W_vzero(); + HVX_VectorPair r0_c1_sum_p = Q6_W_vzero(); + HVX_VectorPair r1_c0_sum_p = Q6_W_vzero(); + HVX_VectorPair r1_c1_sum_p = Q6_W_vzero(); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_Vector r0_hf = x0[i]; + HVX_Vector r1_hf = x1[i]; + HVX_Vector c0_hf = y0[i]; + HVX_Vector c1_hf = y1[i]; + + // Compute 4 dot products: r0xc0, r0xc1, r1xc0, r1xc1 + r0_c0_sum_p = hvx_vec_mpyacc_f32_f16(r0_c0_sum_p, r0_hf, c0_hf); + r0_c1_sum_p = hvx_vec_mpyacc_f32_f16(r0_c1_sum_p, r0_hf, c1_hf); + r1_c0_sum_p = hvx_vec_mpyacc_f32_f16(r1_c0_sum_p, r1_hf, c0_hf); + r1_c1_sum_p = hvx_vec_mpyacc_f32_f16(r1_c1_sum_p, r1_hf, c1_hf); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); + + HVX_Vector r0_hf = Q6_V_vand_QV(bmask, x0[i]); + HVX_Vector r1_hf = Q6_V_vand_QV(bmask, x1[i]); + HVX_Vector c0_hf = Q6_V_vand_QV(bmask, y0[i]); + HVX_Vector c1_hf = Q6_V_vand_QV(bmask, y1[i]); + + r0_c0_sum_p = hvx_vec_mpyacc_f32_f16(r0_c0_sum_p, r0_hf, c0_hf); + r0_c1_sum_p = hvx_vec_mpyacc_f32_f16(r0_c1_sum_p, r0_hf, c1_hf); + r1_c0_sum_p = hvx_vec_mpyacc_f32_f16(r1_c0_sum_p, r1_hf, c0_hf); + r1_c1_sum_p = hvx_vec_mpyacc_f32_f16(r1_c1_sum_p, r1_hf, c1_hf); + } + + HVX_Vector r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r0_c0_sum_p), Q6_V_hi_W(r0_c0_sum_p))); + HVX_Vector r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r0_c1_sum_p), Q6_V_hi_W(r0_c1_sum_p))); + HVX_Vector r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r1_c0_sum_p), Q6_V_hi_W(r1_c0_sum_p))); + HVX_Vector r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r1_c1_sum_p), Q6_V_hi_W(r1_c1_sum_p))); + + // Reduce and store results + HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum); + HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum); + + hvx_vec_store_u(&s0[0], 8, r0_r1_c0_sum); // row0,col0 row1,col0 + hvx_vec_store_u(&s1[0], 8, r0_r1_c1_sum); // row0,col1 row1,col1 +} + +static inline void vec_dot_f16_f16_uu_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) { + const HVX_UVector * restrict x = (const HVX_UVector *) vx; + const HVX_UVector * restrict y = (const HVX_UVector *) vy; + + uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors + uint32_t nloe = n % VLEN_FP16; // leftover elements + + HVX_Vector rsum = Q6_V_vzero(); + + uint32_t i = 0; + + #pragma unroll(4) + for (i = 0; i < nvec; i++) { + HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x[i], y[i]); + rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf))); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); + HVX_Vector x_hf = Q6_V_vand_QV(bmask, x[i]); + HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]); + + HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x_hf, y_hf); + rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf))); + } + + rsum = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(rsum)); + hvx_vec_store_u(&s[0], 4, rsum); +} + +static inline void vec_dot_f16_f32_uu_1x1(const uint32_t n, float * restrict s, const void * restrict x, const void * restrict y) { + const HVX_UVector * restrict vx = (const HVX_UVector * restrict) x; + const HVX_UVector * restrict vy = (const HVX_UVector * restrict) y; + + uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors + uint32_t nloe = n % VLEN_FP16; // leftover elements + + const HVX_Vector zero = Q6_V_vzero(); + + HVX_Vector rsum = Q6_V_vzero(); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + // Load y (fp32) and convert into fp16 + HVX_Vector y0_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+0], zero); // 32 elements + HVX_Vector y1_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+1], zero); // 32 elements + HVX_Vector y_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(y1_qf, y0_qf))); + + // Load x (fp16) + HVX_Vector x_hf = vx[i]; + + HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x_hf, y_hf); + + rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf))); + } + + if (nloe) { + // Load y (fp32) and convert into fp16 + HVX_Vector y0_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+0], zero); // 32 elements + HVX_Vector y1_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+1], zero); // 32 elements + HVX_Vector y_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(y1_qf, y0_qf))); + + // Load x (fp16) + HVX_Vector x_hf = vx[i]; + + // Zero-out unused elements + // Note that we need to clear both x and y because they may contain NANs + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); + x_hf = Q6_V_vand_QV(bmask, x_hf); + y_hf = Q6_V_vand_QV(bmask, y_hf); + + HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x_hf, y_hf); + + rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf))); + } + + // Convert into fp32 and reduce + rsum = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(rsum)); + hvx_vec_store_u(&s[0], 4, rsum); +} + +static inline void hvx_tensor_add_f32_grid( + const struct htp_tensor * restrict dst, + const struct htp_tensor * restrict src2, + uint32_t start_row, + uint32_t end_row, + uint32_t start_col, + uint32_t end_col, + const struct fastdiv_values * div_ne11_12, + const struct fastdiv_values * div_ne11 +) { + if (start_row >= end_row || start_col >= end_col) return; + const uint32_t nb1 = dst->nb[1]; // row stride in bytes + + const uint32_t ne11 = dst->ne[1]; + const uint32_t ne12 = dst->ne[2]; + const uint32_t ne11_12 = ne11 * ne12; + + const bool is_broadcast1 = (src2->ne[1] == 1); + const bool is_broadcast2 = (src2->ne[2] == 1); + const bool is_broadcast3 = (src2->ne[3] == 1); + + for (uint32_t r = start_row; r < end_row; r++) { + float * dst_row = (float *) ((uint8_t *) dst->data + r * nb1); + + uint32_t i13 = fastdiv(r, div_ne11_12); + uint32_t i12 = fastdiv(r - i13 * ne11_12, div_ne11); + uint32_t i11 = r - i13 * ne11_12 - i12 * ne11; + + uint32_t i23 = is_broadcast3 ? 0 : i13; + uint32_t i22 = is_broadcast2 ? 0 : i12; + uint32_t i21 = is_broadcast1 ? 0 : i11; + + const float * src2_row = (const float *) ((const uint8_t *) src2->data + + i21 * src2->nb[1] + i22 * src2->nb[2] + i23 * src2->nb[3]); + + float * dst_ptr = &dst_row[start_col]; + const float * src2_ptr = &src2_row[start_col]; + int remaining = end_col - start_col; + while (remaining >= 32) { + HVX_Vector v_out = hvx_vmemu(dst_ptr); + HVX_Vector v_z = hvx_vmemu(src2_ptr); + hvx_vmemu(dst_ptr) = hvx_vec_add_f32_f32(v_out, v_z); + dst_ptr += 32; + src2_ptr += 32; + remaining -= 32; + } + if (remaining > 0) { + HVX_Vector v_out = hvx_vmemu(dst_ptr); + HVX_Vector v_z = hvx_vmemu(src2_ptr); + hvx_vec_store_u(dst_ptr, remaining * sizeof(float), hvx_vec_add_f32_f32(v_out, v_z)); + } + } +} + diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h index bcb0b8f9e..40b65aa3b 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h @@ -378,7 +378,7 @@ static inline HVX_VectorPair accum_q8_0_32x2( return Q6_W_vcombine_VV(v_sum1, v_sum0); } -static void tiled_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows) { +static void tiled_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -401,10 +401,14 @@ static void tiled_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); } - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } } -static void tiled_vec_dot_q4_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows) { +static void tiled_vec_dot_q4_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y0_q = vy0; const uint8_t * restrict y1_q = vy1; @@ -484,11 +488,19 @@ static void tiled_vec_dot_q4_0_32x2(const uint32_t n, float * restrict s0, float v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); } - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } } -static void tiled_vec_dot_q4_1_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows) { +static void tiled_vec_dot_q4_1_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -519,10 +531,14 @@ static void tiled_vec_dot_q4_1_32x1(const uint32_t n, float * restrict s, const v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); } - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } } -static void tiled_vec_dot_q4_1_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows) { +static void tiled_vec_dot_q4_1_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y0_q = vy0; const uint8_t * restrict y1_q = vy1; @@ -637,11 +653,19 @@ static void tiled_vec_dot_q4_1_32x2(const uint32_t n, float * restrict s0, float v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); } - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } } -static void tiled_vec_dot_q8_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows) { +static void tiled_vec_dot_q8_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -663,10 +687,14 @@ static void tiled_vec_dot_q8_0_32x1(const uint32_t n, float * restrict s, const v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); } - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } } -static void tiled_vec_dot_q8_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows) { +static void tiled_vec_dot_q8_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y0_q = vy0; const uint8_t * restrict y1_q = vy1; @@ -745,11 +773,19 @@ static void tiled_vec_dot_q8_0_32x2(const uint32_t n, float * restrict s0, float v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); } - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } } -static void tiled_vec_dot_iq4nl_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows) { +static void tiled_vec_dot_iq4nl_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -773,10 +809,14 @@ static void tiled_vec_dot_iq4nl_32x1(const uint32_t n, float * restrict s, const v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); } - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } } -static void tiled_vec_dot_iq4nl_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows) { +static void tiled_vec_dot_iq4nl_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y0_q = vy0; const uint8_t * restrict y1_q = vy1; @@ -857,11 +897,19 @@ static void tiled_vec_dot_iq4nl_32x2(const uint32_t n, float * restrict s0, floa v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); } - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } } -static void tiled_vec_dot_mxfp4_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows) { +static void tiled_vec_dot_mxfp4_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -896,10 +944,14 @@ static void tiled_vec_dot_mxfp4_32x1(const uint32_t n, float * restrict s, const v_sum_float = hvx_vec_mul_f32_f32(v_sum_float, hvx_vec_splat_f32(0.5f)); - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } } -static void tiled_vec_dot_mxfp4_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows) { +static void tiled_vec_dot_mxfp4_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y0_q = vy0; const uint8_t * restrict y1_q = vy1; @@ -1013,8 +1065,16 @@ static void tiled_vec_dot_mxfp4_32x2(const uint32_t n, float * restrict s0, floa v_sum_float_c0 = hvx_vec_mul_f32_f32(v_sum_float_c0, hvx_vec_splat_f32(0.5f)); v_sum_float_c1 = hvx_vec_mul_f32_f32(v_sum_float_c1, hvx_vec_splat_f32(0.5f)); - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } } static inline void quantize_f32_q8_0_tiled_kernel( diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.h b/ggml/src/ggml-hexagon/htp/matmul-ops.h index a94d5430d..96369825a 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.h +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.h @@ -392,56 +392,49 @@ static inline size_t htp_mm_hvx_get_vtcm_sizes( case HTP_MM_KERNEL_HVX_QUANT_ROW: { size_t q_src1_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); - vtcm_dst_size = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) : 0; vtcm_src0_size = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256); vtcm_src1_size = htp_mm_round_up(q_src1_row_size * src1_nrows, 256); - // src0 spad is also used in dynamic quantizer to store padded src1 rows - size_t src1_row_size_padded = htp_mm_round_up(q_src1_row_size, QK_Q8_0_TILED * sizeof(float)); - if (vtcm_src0_size < src1_row_size_padded) { - vtcm_src0_size = src1_row_size_padded; - } - vtcm_src0_size = vtcm_src0_size * n_threads; - vtcm_dst_size = vtcm_dst_size * n_threads; if (is_repack) { uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); uint32_t n_k_tiles = ne10 / 32; uint32_t tile_row_size = n_k_tiles * aligned_tile_size; size_t repacked_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256); - if (repacked_vtcm_size < src1_row_size_padded) { - repacked_vtcm_size = src1_row_size_padded; - } vtcm_src0_size = repacked_vtcm_size * n_threads; } + + size_t quant_scratch_size_per_thread = htp_mm_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float)); + size_t dst_size_per_thread = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) : 0; + if (dst_size_per_thread < quant_scratch_size_per_thread) { + dst_size_per_thread = quant_scratch_size_per_thread; + } + vtcm_dst_size = dst_size_per_thread * n_threads; break; } case HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT: { size_t q_src1_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10); - vtcm_dst_size = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) : 0; vtcm_src0_size = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256); vtcm_src1_size = htp_mm_round_up(q_src1_row_size * src1_nrows, 256); - size_t src1_row_size_padded = htp_mm_round_up(q_src1_row_size, 256); - if (vtcm_src0_size < src1_row_size_padded) { - vtcm_src0_size = src1_row_size_padded; - } - vtcm_src0_size = vtcm_src0_size * n_threads; - vtcm_dst_size = vtcm_dst_size * n_threads; if (is_repack) { uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); uint32_t n_k_tiles = ne10 / 32; uint32_t tile_row_size = n_k_tiles * aligned_tile_size; size_t repacked_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256); - if (repacked_vtcm_size < src1_row_size_padded) { - repacked_vtcm_size = src1_row_size_padded; - } vtcm_src0_size = repacked_vtcm_size * n_threads; } + + size_t quant_scratch_size_per_thread = htp_mm_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float)); + size_t dst_size_per_thread = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) : 0; + if (dst_size_per_thread < quant_scratch_size_per_thread) { + dst_size_per_thread = quant_scratch_size_per_thread; + } + vtcm_dst_size = dst_size_per_thread * n_threads; break; } default: @@ -463,7 +456,8 @@ static inline size_t htp_mm_hvx_id_get_vtcm_sizes( size_t src0_row_size, // nb01 uint32_t n_prefetch, size_t * vtcm_src0_size_out, - size_t * vtcm_src1_size_out + size_t * vtcm_src1_size_out, + size_t * vtcm_dst_size_out ) { const bool is_repack = (wtype == HTP_TYPE_Q4_0 || wtype == HTP_TYPE_Q4_1 || wtype == HTP_TYPE_Q8_0 || wtype == HTP_TYPE_IQ4_NL || @@ -476,29 +470,22 @@ static inline size_t htp_mm_hvx_id_get_vtcm_sizes( size_t src0_sz_per_thread = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256); size_t src1_sz = htp_mm_round_up(src1_row_size * src1_nrows, 256); - // src0 spad also holds temporary transposed src1 columns during dynamic quantization. - const size_t src1_row_size_padded = htp_mm_round_up(src1_row_size, QK_Q8_0_TILED * sizeof(float)); - if (src0_sz_per_thread < src1_row_size_padded) { - src0_sz_per_thread = src1_row_size_padded; - } - if (is_repack) { const uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); const uint32_t n_k_tiles = ne10 / 32; const uint32_t tile_row_size = n_k_tiles * aligned_tile_size; size_t repacked_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256); - if (repacked_vtcm_size < src1_row_size_padded) { - repacked_vtcm_size = src1_row_size_padded; - } src0_sz_per_thread = repacked_vtcm_size; } const size_t vtcm_src0_size = src0_sz_per_thread * n_threads; + const size_t vtcm_dst_size = htp_mm_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float)) * n_threads; *vtcm_src0_size_out = vtcm_src0_size; *vtcm_src1_size_out = src1_sz; + *vtcm_dst_size_out = vtcm_dst_size; - return vtcm_src0_size + src1_sz; + return vtcm_src0_size + src1_sz + vtcm_dst_size; } #ifdef __cplusplus diff --git a/koboldcpp.py b/koboldcpp.py index 9a06da35b..4261eeeb4 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -78,7 +78,7 @@ dry_seq_break_max = 128 extra_images_max = 4 # for kontext/qwen img # global vars -KcppVersion = "1.116.2" +KcppVersion = "1.117" showdebug = True kcpp_instance = None #global running instance global_memory = {"tunnel_url": "", "restart_target":"", "input_to_exit":False, "load_complete":False, "restart_override_base_config":"", "last_active_timestamp":datetime.now(), "triggered_sleeping":False, "current_model":"initial_model", "base_config":"", "swapReqType": None, "autoswapmode": False} diff --git a/src/models/qwen3next.cpp b/src/models/qwen3next.cpp index 4505c92b0..f6a21cf40 100644 --- a/src/models/qwen3next.cpp +++ b/src/models/qwen3next.cpp @@ -121,6 +121,8 @@ llama_model_qwen3next::graph::graph(const llama_model & model, const llm_graph_p ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { + res->t_layer_inp[il] = inpL; + ggml_tensor * inpSA = inpL; cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); diff --git a/tools/server/server-cors-proxy.h b/tools/server/server-cors-proxy.h index 53a6909ed..3600a33ec 100644 --- a/tools/server/server-cors-proxy.h +++ b/tools/server/server-cors-proxy.h @@ -39,7 +39,7 @@ static server_http_res_ptr proxy_request(const server_http_req & req, std::strin throw std::runtime_error("unsupported URL scheme in target URL: " + parsed_url.scheme); } - SRV_INF("proxying %s request to %s://%s:%i%s\n", method.c_str(), parsed_url.scheme.c_str(), parsed_url.host.c_str(), parsed_url.port, parsed_url.path.c_str()); + SRV_INF("proxying %s request to %s://%s:%i%s\n", method.c_str(), parsed_url.scheme.c_str(), common_http_format_host(parsed_url.host).c_str(), parsed_url.port, parsed_url.path.c_str()); std::map headers; const std::string proxy_header_prefix = "x-llama-server-proxy-header-"; diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp index 21bed64c9..bb88dda21 100644 --- a/tools/server/server-http.cpp +++ b/tools/server/server-http.cpp @@ -1,4 +1,5 @@ #include "common.h" +#include "http.h" #include "server-http.h" #include "server-stream.h" #include "server-common.h" @@ -441,7 +442,7 @@ bool server_http_context::start() { srv->wait_until_ready(); listening_address = is_sock ? string_format("unix://%s", hostname.c_str()) - : string_format("%s://%s:%d", is_ssl ? "https" : "http", hostname.c_str(), port); + : string_format("%s://%s:%d", is_ssl ? "https" : "http", common_http_format_host(hostname).c_str(), port); return true; } diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 81da00c0e..2ee318f14 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1,4 +1,5 @@ #include "server-common.h" +#include "http.h" #include "server-models.h" #include "server-context.h" #include "server-stream.h" @@ -2263,7 +2264,8 @@ server_http_proxy::server_http_proxy( } if (lowered == "host") { bool is_default_port = (scheme == "https" && port == 443) || (scheme == "http" && port == 80); - req.set_header(key, is_default_port ? host : host + ":" + std::to_string(port)); + const std::string url_host = common_http_format_host(host); + req.set_header(key, is_default_port ? url_host : url_host + ":" + std::to_string(port)); } else { req.set_header(key, value); } diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte index dadcae0c4..7189ce1c7 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte @@ -43,7 +43,7 @@ assistantMessages: number; messageTypes: string[]; } | null>(null); - let editedContent = $derived(message.content); + let editedContent = $state(message.content); let rawEditContent = $derived.by(() => { if (message.role !== MessageRole.ASSISTANT) return undefined; diff --git a/tools/ui/src/lib/constants/pwa.ts b/tools/ui/src/lib/constants/pwa.ts index 80505fabd..b9171d4bb 100644 --- a/tools/ui/src/lib/constants/pwa.ts +++ b/tools/ui/src/lib/constants/pwa.ts @@ -288,9 +288,7 @@ export const API_CACHING_PATTERNS = { } as const; // SvelteKit PWA plugin options -export const PWA_KIT_OPTIONS = { - NAVIGATE_FALLBACK: './' -} as const; +export const PWA_KIT_OPTIONS = {} as const; export const APPLE_META_TAGS = { MOBILE_WEB_APP_CAPABLE: { name: 'apple-mobile-web-app-capable', content: 'yes' }, @@ -322,6 +320,14 @@ export const SVELTEKIT_PWA_OPTIONS: SvelteKitPWAOptions = { globIgnores: GLOB_IGNORES, maximumFileSizeToCacheInBytes: CACHE_SETTINGS.MAX_FILE_SIZE_BYTES, + // Prevent @vite-pwa/sveltekit from auto-adding a NavigationRoute by + // setting navigateFallback to empty string. This keeps the service + // worker from intercepting direct browser navigation to server API + // endpoints (e.g. /slots, /models, /v1/models) which should return + // JSON, not the SPA HTML shell. The server's own static-file fallback + // handles non-API navigation to index.html for the SPA router. + navigateFallback: '', + // Runtime caching for API calls - use NetworkFirst so APIs are always fresh runtimeCaching: [ { @@ -351,10 +357,7 @@ export const SVELTEKIT_PWA_OPTIONS: SvelteKitPWAOptions = { devOptions: { enabled: true, - suppressWarnings: true, - // Use PWA_KIT_OPTIONS.NAVIGATE_FALLBACK to match production SW behaviour - // (navigateFallback defaults to the configured base path, which is '/' for this SPA). - navigateFallback: PWA_KIT_OPTIONS.NAVIGATE_FALLBACK + suppressWarnings: true }, // SvelteKit-specific options diff --git a/tools/ui/src/lib/stores/chat.svelte.ts b/tools/ui/src/lib/stores/chat.svelte.ts index 43f9e72c7..6949047b5 100644 --- a/tools/ui/src/lib/stores/chat.svelte.ts +++ b/tools/ui/src/lib/stores/chat.svelte.ts @@ -1083,6 +1083,11 @@ class ChatStore { let resolvedModel: string | null = null; let modelPersisted = false; const convId = assistantMessage.convId; + // Tracks the last message created in this flow. Used as the parent for the next + // turn's assistant message so createAssistantMessage does not have to read + // conversationsStore.activeMessages, which may belong to a different conversation + // after the user navigates while the loop is still running. + let lastCreatedInFlow = currentMessageId; // freeze the POST identity from t0 so a stop cancels with the exact session key, // never a stale or empty model resolved later this.setChatStreaming(convId, streamedContent, currentMessageId, effectiveModel); @@ -1208,8 +1213,15 @@ class ChatStore { }; if (timings) uiUpdate.timings = timings; if (resolvedModel) uiUpdate.model = resolvedModel; - conversationsStore.updateMessageAtIndex(idx, uiUpdate); - await conversationsStore.updateCurrentNode(currentMessageId); + // touch the active ui array and node pointer only when this conversation + // is displayed; otherwise persist the node move straight to the db so a + // foreign conv's currNode stays untouched + if (conversationsStore.activeConversation?.id === convId) { + conversationsStore.updateMessageAtIndex(idx, uiUpdate); + await conversationsStore.updateCurrentNode(currentMessageId); + } else { + await DatabaseService.updateCurrentNode(convId, currentMessageId); + } }, createToolResultMessage: async ( toolCallId: string, @@ -1230,8 +1242,16 @@ class ChatStore { }, currentMessageId ); - conversationsStore.addMessageToActive(msg); - await conversationsStore.updateCurrentNode(msg.id); + // mirror into the active store and move the node pointer only when this + // conversation is displayed; otherwise persist the node move straight to + // the db for the owning conv so a foreign conv's currNode stays untouched + if (conversationsStore.activeConversation?.id === convId) { + conversationsStore.addMessageToActive(msg); + await conversationsStore.updateCurrentNode(msg.id); + } else { + await DatabaseService.updateCurrentNode(convId, msg.id); + } + lastCreatedInFlow = msg.id; return msg; }, createAssistantMessage: async () => { @@ -1239,8 +1259,6 @@ class ChatStore { streamedContent = ''; streamedReasoningContent = ''; - const lastMsg = - conversationsStore.activeMessages[conversationsStore.activeMessages.length - 1]; const msg = await DatabaseService.createMessageBranch( { convId, @@ -1252,10 +1270,13 @@ class ChatStore { children: [], model: resolvedModel }, - lastMsg.id + lastCreatedInFlow ); - conversationsStore.addMessageToActive(msg); + if (conversationsStore.activeConversation?.id === convId) { + conversationsStore.addMessageToActive(msg); + } currentMessageId = msg.id; + lastCreatedInFlow = msg.id; return msg; }, onFlowComplete: (finalTimings?: ChatMessageTimings) => { diff --git a/tools/ui/tests/e2e/pwa.e2e.ts b/tools/ui/tests/e2e/pwa.e2e.ts index e21672239..c20eec276 100644 --- a/tools/ui/tests/e2e/pwa.e2e.ts +++ b/tools/ui/tests/e2e/pwa.e2e.ts @@ -43,7 +43,10 @@ test.describe('PWA Service Worker', () => { expect(swContent).toMatch(/"_app\/immutable\/assets\/bundle\.[a-zA-Z0-9_-]+\.css"/); expect(swContent).toMatch(/"manifest\.webmanifest"/); expect(swContent).toMatch(/"_app\/version\.json"/); - expect(swContent).toMatch(/NavigationRoute/); + // NavigationRoute is intentionally absent — server API endpoints + // (e.g. /slots, /models) must not be intercepted by the PWA and + // should return JSON directly from the server. + expect(swContent).not.toMatch(/NavigationRoute/); expect(swContent).toMatch(/api-cache/); }); diff --git a/tools/ui/tests/unit/pwa.spec.ts b/tools/ui/tests/unit/pwa.spec.ts index 0059e246c..17da27986 100644 --- a/tools/ui/tests/unit/pwa.spec.ts +++ b/tools/ui/tests/unit/pwa.spec.ts @@ -108,9 +108,11 @@ describe('PWA Build Output', () => { expect(swContent).toMatch(/"manifest\.webmanifest"/); }); - it('has navigation route registered', () => { + it('no navigation route — API endpoints bypass PWA', () => { expect(swContent).toBeTruthy(); - expect(swContent).toMatch(/NavigationRoute/); + // NavigationRoute is intentionally absent so direct browser + // navigation to server API endpoints returns JSON, not HTML. + expect(swContent).not.toMatch(/NavigationRoute/); }); it('has runtime caching for API routes', () => {