mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-05-22 11:16:08 +00:00
ggml-hexagon: add PAD op HVX kernel (#23078)
* ggml-hexagon: add PAD op HVX kernel Implements GGML_OP_PAD on the Hexagon HTP backend using HVX vectorized kernels. Supports zero-padding and circular padding across all 4 tensor dimensions. * hex-ggml: remove duplicate op cases (merge conflict) * hex-pad: fix editorconfig checks and macro alignment --------- Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
This commit is contained in:
Pranav Dhinakar
GitHub
parent
5cbaa5e69e
commit
b7340443d4
6 changed files with 569 additions and 0 deletions
|
|
@ -2744,6 +2744,18 @@ static bool ggml_hexagon_supported_ssm_conv(const struct ggml_hexagon_session *
|
|||
return true;
|
||||
}
|
||||
|
||||
static bool ggml_hexagon_supported_pad(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
|
||||
const struct ggml_tensor * src0 = op->src[0];
|
||||
const struct ggml_tensor * dst = op;
|
||||
|
||||
if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
|
||||
return false;
|
||||
}
|
||||
|
||||
GGML_UNUSED(sess);
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool ggml_hexagon_supported_cumsum(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
|
||||
const struct ggml_tensor * src0 = op->src[0];
|
||||
const struct ggml_tensor * dst = op;
|
||||
|
|
@ -2857,6 +2869,8 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) {
|
|||
case GGML_OP_FILL: return HTP_OP_FILL;
|
||||
case GGML_OP_DIAG: return HTP_OP_DIAG;
|
||||
case GGML_OP_SOLVE_TRI: return HTP_OP_SOLVE_TRI;
|
||||
case GGML_OP_PAD: return HTP_OP_PAD;
|
||||
|
||||
case GGML_OP_UNARY:
|
||||
switch (ggml_get_unary_op(t)) {
|
||||
case GGML_UNARY_OP_SILU: return HTP_OP_UNARY_SILU;
|
||||
|
|
@ -3416,6 +3430,10 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
|
|||
supp = ggml_hexagon_supported_solve_tri(sess, op);
|
||||
break;
|
||||
|
||||
case GGML_OP_PAD:
|
||||
supp = ggml_hexagon_supported_pad(sess, op);
|
||||
break;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ add_library(${HTP_LIB} SHARED
|
|||
diag-ops.c
|
||||
solve-tri-ops.c
|
||||
gated-delta-net-ops.c
|
||||
pad-ops.c
|
||||
)
|
||||
|
||||
target_compile_definitions(${HTP_LIB} PRIVATE
|
||||
|
|
|
|||
|
|
@ -107,5 +107,6 @@ int op_fill(struct htp_ops_context * octx);
|
|||
int op_diag(struct htp_ops_context * octx);
|
||||
int op_solve_tri(struct htp_ops_context * octx);
|
||||
int op_gated_delta_net(struct htp_ops_context * octx);
|
||||
int op_pad(struct htp_ops_context * octx);
|
||||
|
||||
#endif /* HTP_CTX_H */
|
||||
|
|
|
|||
|
|
@ -86,6 +86,7 @@ enum htp_op_code {
|
|||
HTP_OP_SOLVE_TRI,
|
||||
HTP_OP_L2_NORM,
|
||||
HTP_OP_GATED_DELTA_NET,
|
||||
HTP_OP_PAD,
|
||||
|
||||
HTP_OP_INVALID
|
||||
};
|
||||
|
|
|
|||
|
|
@ -595,6 +595,9 @@ static int execute_op(struct htp_ops_context * octx) {
|
|||
case HTP_OP_SOLVE_TRI:
|
||||
return op_solve_tri(octx);
|
||||
|
||||
case HTP_OP_PAD:
|
||||
return op_pad(octx);
|
||||
|
||||
case HTP_OP_GATED_DELTA_NET:
|
||||
return op_gated_delta_net(octx);
|
||||
|
||||
|
|
|
|||
545
ggml/src/ggml-hexagon/htp/pad-ops.c
Normal file
545
ggml/src/ggml-hexagon/htp/pad-ops.c
Normal file
|
|
@ -0,0 +1,545 @@
|
|||
#pragma clang diagnostic ignored "-Wunused-variable"
|
||||
#pragma clang diagnostic ignored "-Wunused-function"
|
||||
#pragma clang diagnostic ignored "-Wunused-but-set-variable"
|
||||
|
||||
#include <HAP_farf.h>
|
||||
#include <HAP_perf.h>
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include "hex-dma.h"
|
||||
#include "hvx-utils.h"
|
||||
|
||||
#define GGML_COMMON_DECL_C
|
||||
#include "ggml-common.h"
|
||||
#include "htp-ctx.h"
|
||||
#include "htp-ops.h"
|
||||
|
||||
/* Circular wrap: maps any integer x into [0, n) */
|
||||
static inline uint32_t wrap_around(int32_t x, uint32_t n) {
|
||||
return (uint32_t)(((x % (int32_t)n) + (int32_t)n) % (int32_t)n);
|
||||
}
|
||||
|
||||
/* Decompose a flat dst row index into (i1, i2, i3) */
|
||||
static inline void pad_decompose_row(uint32_t ir, uint32_t ne1, uint32_t ne2,
|
||||
uint32_t *i1, uint32_t *i2, uint32_t *i3) {
|
||||
*i1 = ir % ne1;
|
||||
*i2 = (ir / ne1) % ne2;
|
||||
*i3 = ir / (ne1 * ne2);
|
||||
}
|
||||
|
||||
/* Return non-zero if row (i1,i2,i3) falls in the non-padded interior */
|
||||
static inline int pad_is_interior(uint32_t i1, uint32_t i2, uint32_t i3,
|
||||
int32_t lp1, int32_t rp1, uint32_t ne1,
|
||||
int32_t lp2, int32_t rp2, uint32_t ne2,
|
||||
int32_t lp3, int32_t rp3, uint32_t ne3) {
|
||||
return ((int32_t)i1 >= lp1 && (int32_t)i1 < (int32_t)ne1 - rp1) &&
|
||||
((int32_t)i2 >= lp2 && (int32_t)i2 < (int32_t)ne2 - rp2) &&
|
||||
((int32_t)i3 >= lp3 && (int32_t)i3 < (int32_t)ne3 - rp3);
|
||||
}
|
||||
|
||||
/* Compute the DDR src row pointer for a zero-pad interior row */
|
||||
static inline const uint8_t * pad_src_row_ptr(const struct htp_tensor * src,
|
||||
uint32_t i1, uint32_t i2, uint32_t i3,
|
||||
int32_t lp1, int32_t lp2, int32_t lp3) {
|
||||
return (const uint8_t *) src->data
|
||||
+ (i1 - (uint32_t)lp1) * src->nb[1]
|
||||
+ (i2 - (uint32_t)lp2) * src->nb[2]
|
||||
+ (i3 - (uint32_t)lp3) * src->nb[3];
|
||||
}
|
||||
|
||||
/* Compute the DDR src row pointer for a circular row (wrap-around indexing) */
|
||||
static inline const uint8_t * pad_circ_src_row_ptr(const struct htp_tensor * src,
|
||||
uint32_t i1, uint32_t i2, uint32_t i3,
|
||||
int32_t lp1, int32_t lp2, int32_t lp3) {
|
||||
return (const uint8_t *) src->data
|
||||
+ wrap_around((int32_t)i1 - lp1, src->ne[1]) * src->nb[1]
|
||||
+ wrap_around((int32_t)i2 - lp2, src->ne[2]) * src->nb[2]
|
||||
+ wrap_around((int32_t)i3 - lp3, src->ne[3]) * src->nb[3];
|
||||
}
|
||||
|
||||
struct htp_pad_context {
|
||||
struct htp_ops_context * octx;
|
||||
|
||||
int32_t lp0, rp0;
|
||||
int32_t lp1, rp1;
|
||||
int32_t lp2, rp2;
|
||||
int32_t lp3, rp3;
|
||||
|
||||
uint32_t nrows_per_thread;
|
||||
uint32_t total_dst_rows;
|
||||
|
||||
size_t type_size;
|
||||
|
||||
// Row sizes for DMA kernel (populated when VTCM is available)
|
||||
size_t src_row_size;
|
||||
size_t src_row_size_aligned;
|
||||
size_t dst_row_size;
|
||||
size_t dst_row_size_aligned;
|
||||
};
|
||||
|
||||
#define htp_pad_preamble \
|
||||
const struct htp_tensor * src = octx->src[0]; \
|
||||
const struct htp_tensor * dst = octx->dst; \
|
||||
\
|
||||
const uint32_t ne00 = src->ne[0]; \
|
||||
const uint32_t nb00 = src->nb[0]; \
|
||||
\
|
||||
const uint32_t ne0 = dst->ne[0]; \
|
||||
const uint32_t ne1 = dst->ne[1]; \
|
||||
const uint32_t ne2 = dst->ne[2]; \
|
||||
const uint32_t ne3 = dst->ne[3]; \
|
||||
\
|
||||
const uint32_t nb1 = dst->nb[1]; \
|
||||
const uint32_t nb2 = dst->nb[2]; \
|
||||
const uint32_t nb3 = dst->nb[3]; \
|
||||
\
|
||||
const int32_t lp0 = pctx->lp0, rp0 = pctx->rp0; \
|
||||
const int32_t lp1 = pctx->lp1, rp1 = pctx->rp1; \
|
||||
const int32_t lp2 = pctx->lp2, rp2 = pctx->rp2; \
|
||||
const int32_t lp3 = pctx->lp3, rp3 = pctx->rp3; \
|
||||
\
|
||||
const size_t type_size = pctx->type_size; \
|
||||
\
|
||||
const uint32_t row_start = pctx->nrows_per_thread * ith; \
|
||||
const uint32_t row_end = MIN(row_start + pctx->nrows_per_thread, pctx->total_dst_rows);
|
||||
|
||||
|
||||
#define htp_pad_dma_preamble \
|
||||
const size_t src_row_size = pctx->src_row_size; \
|
||||
const size_t src_row_size_aligned = pctx->src_row_size_aligned; \
|
||||
const size_t dst_row_size = pctx->dst_row_size; \
|
||||
const size_t dst_row_size_aligned = pctx->dst_row_size_aligned; \
|
||||
\
|
||||
uint8_t * src_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; \
|
||||
uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread; \
|
||||
\
|
||||
dma_queue * dma = octx->ctx->dma[ith];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// HVX vectorized PAD kernel
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
static void pad_job_per_thread_hvx(unsigned int nth, unsigned int ith, void * data) {
|
||||
const struct htp_pad_context * pctx = (const struct htp_pad_context *) data;
|
||||
struct htp_ops_context * octx = pctx->octx;
|
||||
htp_pad_preamble;
|
||||
|
||||
uint64_t t1, t2;
|
||||
t1 = HAP_perf_get_qtimer_count();
|
||||
|
||||
for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) {
|
||||
uint32_t i1, i2, i3;
|
||||
pad_decompose_row(dst_row, ne1, ne2, &i1, &i2, &i3);
|
||||
|
||||
uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;
|
||||
|
||||
const int interior = pad_is_interior(i1, i2, i3,
|
||||
lp1, rp1, ne1,
|
||||
lp2, rp2, ne2,
|
||||
lp3, rp3, ne3);
|
||||
|
||||
if (!interior) {
|
||||
hvx_splat_f32_u(dst_ptr, 0.0f, ne0);
|
||||
} else {
|
||||
const uint8_t * src_ptr = pad_src_row_ptr(src, i1, i2, i3, lp1, lp2, lp3);
|
||||
|
||||
if (lp0 > 0) {
|
||||
hvx_splat_f32_u(dst_ptr, 0.0f, (uint32_t)lp0);
|
||||
}
|
||||
|
||||
uint8_t * dst_row_start = dst_ptr + (size_t)lp0 * type_size;
|
||||
if (nb00 == type_size) {
|
||||
hvx_copy_f32_uu(dst_row_start, src_ptr, ne00);
|
||||
} else {
|
||||
for (uint32_t i = 0; i < ne00; i++) {
|
||||
memcpy(dst_row_start + i * type_size,
|
||||
src_ptr + (size_t)i * nb00,
|
||||
type_size);
|
||||
}
|
||||
}
|
||||
|
||||
if (rp0 > 0) {
|
||||
hvx_splat_f32_u(dst_ptr + ((size_t)lp0 + ne00) * type_size, 0.0f, (uint32_t)rp0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
t2 = HAP_perf_get_qtimer_count();
|
||||
|
||||
FARF(HIGH, "pad-hvx %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n",
|
||||
ith, nth,
|
||||
src->ne[0], src->ne[1], src->ne[2], src->ne[3],
|
||||
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
|
||||
row_start, row_end,
|
||||
(unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// HVX + DMA PAD kernel — aligned, double-buffered
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void * data) {
|
||||
const struct htp_pad_context * pctx = (const struct htp_pad_context *) data;
|
||||
struct htp_ops_context * octx = pctx->octx;
|
||||
htp_pad_preamble;
|
||||
htp_pad_dma_preamble;
|
||||
|
||||
uint64_t t1, t2;
|
||||
t1 = HAP_perf_get_qtimer_count();
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Priming phase: push 2 pairs of (dummy_dst_DMA, src_DMA) to seed the
|
||||
// double-buffer pipeline before the main loop begins.
|
||||
// -----------------------------------------------------------------------
|
||||
for (uint32_t ir = row_start, spad_idx = 0; ir < row_end && spad_idx < 2; ir++, spad_idx++) {
|
||||
uint8_t * src_spad_cur = src_spad_base + spad_idx * src_row_size_aligned;
|
||||
uint8_t * dst_spad_cur = dst_spad_base + spad_idx * dst_row_size_aligned;
|
||||
|
||||
dma_queue_push_vtcm_to_ddr(dma,
|
||||
dma_make_ptr((uint8_t *)dst->data, dst_spad_cur),
|
||||
dst_row_size, dst_row_size_aligned, 0);
|
||||
|
||||
uint32_t i1, i2, i3;
|
||||
pad_decompose_row(ir, ne1, ne2, &i1, &i2, &i3);
|
||||
const int interior = pad_is_interior(i1, i2, i3,
|
||||
lp1, rp1, ne1,
|
||||
lp2, rp2, ne2,
|
||||
lp3, rp3, ne3);
|
||||
|
||||
const uint8_t * src_ptr = interior
|
||||
? pad_src_row_ptr(src, i1, i2, i3, lp1, lp2, lp3) : NULL;
|
||||
|
||||
// Interior row: real DMA (1 row) from DDR to VTCM.
|
||||
// Border row: null DMA (nrows=0)
|
||||
dma_queue_push_ddr_to_vtcm(dma,
|
||||
dma_make_ptr(src_spad_cur,
|
||||
src_ptr ? src_ptr : (const uint8_t *)src_spad_cur),
|
||||
src_row_size_aligned, src_row_size, src_ptr ? 1 : 0);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Main loop: pop completed DMAs, compute in VTCM with aligned HVX ops,
|
||||
// push dst DMA and prefetch src for the next+1 row.
|
||||
// -----------------------------------------------------------------------
|
||||
for (uint32_t ir = row_start; ir < row_end; ir++) {
|
||||
uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma).src;
|
||||
uint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma).dst;
|
||||
|
||||
uint32_t i1, i2, i3;
|
||||
pad_decompose_row(ir, ne1, ne2, &i1, &i2, &i3);
|
||||
|
||||
uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;
|
||||
|
||||
const int interior = pad_is_interior(i1, i2, i3,
|
||||
lp1, rp1, ne1,
|
||||
lp2, rp2, ne2,
|
||||
lp3, rp3, ne3);
|
||||
|
||||
if (!interior) {
|
||||
hvx_splat_f32_a(dst_spad_cur, 0.0f, ne0);
|
||||
} else {
|
||||
hvx_splat_f32_a(dst_spad_cur, 0.0f, ne0);
|
||||
|
||||
uint8_t * dst_interior = dst_spad_cur + (size_t)lp0 * type_size;
|
||||
|
||||
if ((uintptr_t)dst_interior % VLEN == 0) {
|
||||
hvx_copy_f32_aa(dst_interior, src_spad_cur, ne00);
|
||||
} else {
|
||||
hvx_copy_f32_ua(dst_interior, src_spad_cur, ne00);
|
||||
}
|
||||
}
|
||||
|
||||
dma_queue_push_vtcm_to_ddr(dma,
|
||||
dma_make_ptr(dst_ptr, dst_spad_cur),
|
||||
dst_row_size, dst_row_size_aligned, 1);
|
||||
|
||||
const uint32_t next_row = ir + 2;
|
||||
if (next_row < row_end) {
|
||||
uint32_t ni1, ni2, ni3;
|
||||
pad_decompose_row(next_row, ne1, ne2, &ni1, &ni2, &ni3);
|
||||
const int next_interior = pad_is_interior(ni1, ni2, ni3,
|
||||
lp1, rp1, ne1,
|
||||
lp2, rp2, ne2,
|
||||
lp3, rp3, ne3);
|
||||
const uint8_t * next_src_ptr = next_interior
|
||||
? pad_src_row_ptr(src, ni1, ni2, ni3, lp1, lp2, lp3) : NULL;
|
||||
|
||||
dma_queue_push_ddr_to_vtcm(dma,
|
||||
dma_make_ptr(src_spad_cur,
|
||||
next_src_ptr ? next_src_ptr : (const uint8_t *)src_spad_cur),
|
||||
src_row_size_aligned, src_row_size, next_src_ptr ? 1 : 0);
|
||||
}
|
||||
}
|
||||
|
||||
dma_queue_flush(dma);
|
||||
|
||||
t2 = HAP_perf_get_qtimer_count();
|
||||
|
||||
FARF(HIGH, "pad-hvx-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n",
|
||||
ith, nth,
|
||||
src->ne[0], src->ne[1], src->ne[2], src->ne[3],
|
||||
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
|
||||
row_start, row_end,
|
||||
(unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// HVX circular PAD kernel
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
static void pad_job_per_thread_hvx_circular(unsigned int nth, unsigned int ith, void * data) {
|
||||
const struct htp_pad_context * pctx = (const struct htp_pad_context *) data;
|
||||
struct htp_ops_context * octx = pctx->octx;
|
||||
htp_pad_preamble;
|
||||
|
||||
uint64_t t1, t2;
|
||||
t1 = HAP_perf_get_qtimer_count();
|
||||
|
||||
for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) {
|
||||
uint32_t i1, i2, i3;
|
||||
pad_decompose_row(dst_row, ne1, ne2, &i1, &i2, &i3);
|
||||
|
||||
uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;
|
||||
const uint8_t * src_row = pad_circ_src_row_ptr(src, i1, i2, i3, lp1, lp2, lp3);
|
||||
|
||||
if (nb00 == type_size) {
|
||||
|
||||
if (lp0 > 0) {
|
||||
if ((uint32_t)lp0 < 32) {
|
||||
memcpy(dst_ptr,
|
||||
src_row + (size_t)(ne00 - (uint32_t)lp0) * type_size,
|
||||
(size_t)lp0 * type_size);
|
||||
} else {
|
||||
hvx_copy_f32_uu(dst_ptr,
|
||||
src_row + (size_t)(ne00 - (uint32_t)lp0) * type_size,
|
||||
(uint32_t)lp0);
|
||||
}
|
||||
}
|
||||
hvx_copy_f32_uu(dst_ptr + (size_t)lp0 * type_size, src_row, ne00);
|
||||
if (rp0 > 0) {
|
||||
if ((uint32_t)rp0 < 32) {
|
||||
memcpy(dst_ptr + ((size_t)lp0 + ne00) * type_size,
|
||||
src_row,
|
||||
(size_t)rp0 * type_size);
|
||||
} else {
|
||||
hvx_copy_f32_uu(dst_ptr + ((size_t)lp0 + ne00) * type_size,
|
||||
src_row,
|
||||
(uint32_t)rp0);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (uint32_t i = 0; i < (uint32_t)lp0; i++) {
|
||||
*(float *)(dst_ptr + i * type_size) =
|
||||
*(const float *)(src_row + (size_t)(ne00 - (uint32_t)lp0 + i) * nb00);
|
||||
}
|
||||
for (uint32_t i = 0; i < ne00; i++) {
|
||||
*(float *)(dst_ptr + ((size_t)lp0 + i) * type_size) =
|
||||
*(const float *)(src_row + (size_t)i * nb00);
|
||||
}
|
||||
for (uint32_t i = 0; i < (uint32_t)rp0; i++) {
|
||||
*(float *)(dst_ptr + ((size_t)lp0 + ne00 + i) * type_size) =
|
||||
*(const float *)(src_row + (size_t)i * nb00);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
t2 = HAP_perf_get_qtimer_count();
|
||||
|
||||
FARF(HIGH, "pad-hvx-circ %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n",
|
||||
ith, nth,
|
||||
src->ne[0], src->ne[1], src->ne[2], src->ne[3],
|
||||
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
|
||||
row_start, row_end,
|
||||
(unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// HVX + DMA circular PAD kernel — aligned, double-buffered
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int ith, void * data) {
|
||||
const struct htp_pad_context * pctx = (const struct htp_pad_context *) data;
|
||||
struct htp_ops_context * octx = pctx->octx;
|
||||
htp_pad_preamble;
|
||||
htp_pad_dma_preamble;
|
||||
|
||||
uint64_t t1, t2;
|
||||
t1 = HAP_perf_get_qtimer_count();
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Priming phase: push 2 pairs of (dummy_dst_DMA, src_DMA) to seed the
|
||||
// double-buffer pipeline. Every row is a real src DMA (no null DMAs).
|
||||
// -----------------------------------------------------------------------
|
||||
for (uint32_t ir = row_start, spad_idx = 0; ir < row_end && spad_idx < 2; ir++, spad_idx++) {
|
||||
uint8_t * src_spad_cur = src_spad_base + spad_idx * src_row_size_aligned;
|
||||
uint8_t * dst_spad_cur = dst_spad_base + spad_idx * dst_row_size_aligned;
|
||||
|
||||
dma_queue_push_vtcm_to_ddr(dma,
|
||||
dma_make_ptr((uint8_t *)dst->data, dst_spad_cur),
|
||||
dst_row_size, dst_row_size_aligned, 0);
|
||||
|
||||
uint32_t pi1, pi2, pi3;
|
||||
pad_decompose_row(ir, ne1, ne2, &pi1, &pi2, &pi3);
|
||||
dma_queue_push_ddr_to_vtcm(dma,
|
||||
dma_make_ptr(src_spad_cur, pad_circ_src_row_ptr(src, pi1, pi2, pi3, lp1, lp2, lp3)),
|
||||
src_row_size_aligned, src_row_size, 1);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Main loop: pop completed DMAs, assemble circular row in VTCM with
|
||||
// aligned HVX ops, push dst DMA and prefetch src for the next+1 row.
|
||||
// -----------------------------------------------------------------------
|
||||
for (uint32_t ir = row_start; ir < row_end; ir++) {
|
||||
uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma).src;
|
||||
uint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma).dst;
|
||||
|
||||
uint32_t i1, i2, i3;
|
||||
pad_decompose_row(ir, ne1, ne2, &i1, &i2, &i3);
|
||||
uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;
|
||||
|
||||
|
||||
if (lp0 > 0) {
|
||||
uint8_t * dst_left = dst_spad_cur;
|
||||
const uint8_t * src_left = src_spad_cur + (size_t)(ne00 - (uint32_t)lp0) * type_size;
|
||||
if ((uint32_t)lp0 < 32) {
|
||||
memcpy(dst_left, src_left, (size_t)lp0 * type_size);
|
||||
} else {
|
||||
hvx_copy_f32_uu(dst_left, src_left, (uint32_t)lp0);
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
uint8_t * dst_mid = dst_spad_cur + (size_t)lp0 * type_size;
|
||||
if ((uintptr_t)dst_mid % VLEN == 0) {
|
||||
hvx_copy_f32_aa(dst_mid, src_spad_cur, ne00);
|
||||
} else {
|
||||
hvx_copy_f32_ua(dst_mid, src_spad_cur, ne00);
|
||||
}
|
||||
}
|
||||
|
||||
if (rp0 > 0) {
|
||||
uint8_t * dst_right = dst_spad_cur + ((size_t)lp0 + ne00) * type_size;
|
||||
if ((uint32_t)rp0 < 32) {
|
||||
memcpy(dst_right, src_spad_cur, (size_t)rp0 * type_size);
|
||||
} else {
|
||||
if ((uintptr_t)dst_right % VLEN == 0) {
|
||||
hvx_copy_f32_aa(dst_right, src_spad_cur, (uint32_t)rp0);
|
||||
} else {
|
||||
hvx_copy_f32_ua(dst_right, src_spad_cur, (uint32_t)rp0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dma_queue_push_vtcm_to_ddr(dma,
|
||||
dma_make_ptr(dst_ptr, dst_spad_cur),
|
||||
dst_row_size, dst_row_size_aligned, 1);
|
||||
|
||||
const uint32_t next_row = ir + 2;
|
||||
if (next_row < row_end) {
|
||||
uint32_t nri1, nri2, nri3;
|
||||
pad_decompose_row(next_row, ne1, ne2, &nri1, &nri2, &nri3);
|
||||
dma_queue_push_ddr_to_vtcm(dma,
|
||||
dma_make_ptr(src_spad_cur,
|
||||
pad_circ_src_row_ptr(src, nri1, nri2, nri3, lp1, lp2, lp3)),
|
||||
src_row_size_aligned, src_row_size, 1);
|
||||
}
|
||||
}
|
||||
|
||||
dma_queue_flush(dma);
|
||||
|
||||
t2 = HAP_perf_get_qtimer_count();
|
||||
|
||||
FARF(HIGH, "pad-hvx-circ-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n",
|
||||
ith, nth,
|
||||
src->ne[0], src->ne[1], src->ne[2], src->ne[3],
|
||||
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
|
||||
row_start, row_end,
|
||||
(unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
|
||||
}
|
||||
|
||||
int op_pad(struct htp_ops_context * octx) {
|
||||
const struct htp_tensor * src0 = octx->src[0];
|
||||
const struct htp_tensor * dst = octx->dst;
|
||||
|
||||
// Only F32 supported
|
||||
size_t type_size;
|
||||
switch (src0->type) {
|
||||
case HTP_TYPE_F32: type_size = 4; break;
|
||||
default:
|
||||
FARF(ERROR, "pad-hvx: unsupported type %u\n", src0->type);
|
||||
return HTP_STATUS_NO_SUPPORT;
|
||||
}
|
||||
|
||||
if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) {
|
||||
return HTP_STATUS_OK;
|
||||
}
|
||||
|
||||
const int32_t lp0 = octx->op_params[0];
|
||||
const int32_t rp0 = octx->op_params[1];
|
||||
const int32_t lp1 = octx->op_params[2];
|
||||
const int32_t rp1 = octx->op_params[3];
|
||||
const int32_t lp2 = octx->op_params[4];
|
||||
const int32_t rp2 = octx->op_params[5];
|
||||
const int32_t lp3 = octx->op_params[6];
|
||||
const int32_t rp3 = octx->op_params[7];
|
||||
const int32_t circular = octx->op_params[8];
|
||||
|
||||
const uint32_t ne0 = dst->ne[0];
|
||||
const uint32_t ne00 = src0->ne[0];
|
||||
|
||||
const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3];
|
||||
const uint32_t n_threads = MIN(octx->n_threads, total_dst_rows > 0 ? total_dst_rows : 1);
|
||||
|
||||
const size_t src_row_size = (size_t)ne00 * type_size;
|
||||
const size_t dst_row_size = (size_t)ne0 * type_size;
|
||||
const size_t src_row_size_aligned = hex_round_up(src_row_size, VLEN);
|
||||
const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);
|
||||
|
||||
// Total VTCM needed: 2 buffers (ping+pong) for src and dst, per thread
|
||||
const size_t vtcm_needed = (size_t)n_threads * 2 * (src_row_size_aligned + dst_row_size_aligned);
|
||||
|
||||
const int use_dma = (src0->nb[0] == (uint32_t)type_size) &&
|
||||
(ne00 >= 512) &&
|
||||
(octx->ctx->vtcm_base != NULL) &&
|
||||
(octx->ctx->vtcm_size >= vtcm_needed);
|
||||
|
||||
if (use_dma) {
|
||||
octx->src0_spad.size_per_thread = 2 * src_row_size_aligned;
|
||||
octx->dst_spad.size_per_thread = 2 * dst_row_size_aligned;
|
||||
octx->src0_spad.size = n_threads * octx->src0_spad.size_per_thread;
|
||||
octx->dst_spad.size = n_threads * octx->dst_spad.size_per_thread;
|
||||
octx->src0_spad.data = octx->ctx->vtcm_base;
|
||||
octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size;
|
||||
}
|
||||
|
||||
struct htp_pad_context pctx = {
|
||||
.octx = octx,
|
||||
.lp0 = lp0, .rp0 = rp0,
|
||||
.lp1 = lp1, .rp1 = rp1,
|
||||
.lp2 = lp2, .rp2 = rp2,
|
||||
.lp3 = lp3, .rp3 = rp3,
|
||||
.nrows_per_thread = (total_dst_rows + n_threads - 1) / n_threads,
|
||||
.total_dst_rows = total_dst_rows,
|
||||
.type_size = type_size,
|
||||
.src_row_size = src_row_size,
|
||||
.src_row_size_aligned = src_row_size_aligned,
|
||||
.dst_row_size = dst_row_size,
|
||||
.dst_row_size_aligned = dst_row_size_aligned,
|
||||
};
|
||||
|
||||
FARF(HIGH, "pad-hvx%s%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) pads=(%d,%d,%d,%d,%d,%d,%d,%d)\n",
|
||||
circular ? "-circ" : "",
|
||||
use_dma ? "-dma" : "",
|
||||
src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],
|
||||
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
|
||||
lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3);
|
||||
|
||||
if (circular && use_dma) { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx_circular_dma, &pctx, n_threads); }
|
||||
else if (circular) { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx_circular, &pctx, n_threads); }
|
||||
else if (use_dma) { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx_dma, &pctx, n_threads); }
|
||||
else { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx, &pctx, n_threads); }
|
||||
|
||||
return HTP_STATUS_OK;
|
||||
}
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue